bc527b6264234c33824854e5b596fb6f790983d4 lrnassar Tue Aug 4 14:22:28 2026 -0700 Fix release blockers and data errors found during QA of the mouseDevTimecourse tracks. refs #37001 Add maxWindowToDraw 10000000 to the six bigBarChart subtracks. Without it a whole-chromosome view asked the track to draw 3445 items x 156 bars and took 71 seconds on mm10 and 69 on mm39. The superTrack is on by default, so any mouse user zooming out could hit it. Now 254 ms and 86 ms. Correct a 1-bp off-by-one in the bigBarChart chromStart. The hub builder wrote 1-based GTF gene starts into the 0-based BED chromStart field, so every gene sat one base right of its true start while chromEnd was correct. Measured against GENCODE on mm10 before the fix, 42081/42093 genes (VM21) and 35323/35333 (VM4) were start+1 with none exact. Added fixBarChartStarts.sh, which rebuilds the files and refuses to run on one that has already been corrected. Originals kept as *.bb.preStartFix. Reported upstream to the hub author. Correct the replicate numbers on the bigWig signal composite. The biosample TSV has no replicate column, so generateBigwigTrackDb.py had been deriving one by sorting biosample accessions alphabetically, which mislabeled 124 of the 312 subtracks and flipped the default-on state of 62 of them. Added fetchReplicateNumbers.py to read the real biological_replicate_number from the ENCODE portal. Rewrite the signal shortLabels. They had been hard-truncated at 20 characters, which left 20 subtracks with duplicate labels. The generator now emits a one-letter view code and errors out if two labels match within the 17 characters hgTracks draws in the left label area, rather than silently truncating. Make generateBigwigTrackDb.py reproduce the committed .ra. It now emits the two-digit tissue prefixes that give the author-requested biological order, and the html setting, instead of depending on a one-off patch applied afterwards. Add barChartMerge, barChartMetric, labelFields and defaultLabelFields, and Title Case all shortLabels. Update the nine description pages: replicate wording to match barChartMerge, GitHub source links to the makedoc, build scripts and trackDb in Methods, an mm39 liftOver accounting note, and remove a duplicated sentence from the shared Display include. Add curl -f to downloadBigwigs.sh so an HTTP error body is never saved as a bigWig and then skipped forever by the restart check. diff --git src/hg/makeDb/scripts/mouseDevTimecourse/generateBigwigTrackDb.py src/hg/makeDb/scripts/mouseDevTimecourse/generateBigwigTrackDb.py index fec338f09cb..cee34b025a5 100755 --- src/hg/makeDb/scripts/mouseDevTimecourse/generateBigwigTrackDb.py +++ src/hg/makeDb/scripts/mouseDevTimecourse/generateBigwigTrackDb.py @@ -1,199 +1,287 @@ #!/usr/bin/env python3 """ Read Diane's ENCSR574CRQ_biosample.tsv (#36998 attachment) and emit a view-based trackDb composite for the ENCODE bulk RNA-seq bigWig signal tracks on mm10. The structure mirrors the Wold Lab's own hg19 RNA-seq track (wgEncodeCaltechRnaSeq): the composite has two view containers ("Unique reads" and "All reads") with their own display settings, and tissue/age/rep stay as subGroups underneath. Per-(tissue, age) colors are read from the local .facets file and emitted as RGB on each subtrack, matching the bigBarChart gradient. +Replicate numbers come from ENCSR574CRQ_replicates.tsv, which +fetchReplicateNumbers.py builds from the ENCODE portal. The biosample TSV +has no replicate column, so that file has to be built first. + +Tissue subGroup tags carry a two-digit prefix (t01_thymus ... t17_neural_tube) +to give the biological order the data authors asked for (#36998 note-79). +sortOrder compares tag strings, so without the prefix the matrix sorts +alphabetically. + The composite is hidden by default. When the user enables it, rep1 + unique-reads subtracks turn on (one signal per (tissue, age) sample, 78 tracks); rep2 and all-reads stay off and can be enabled via the trackUi page. Output goes to stdout. Redirect to a file. """ import sys DEFAULT_TSV = '/hive/data/outside/woldlab/mouseDevTimecourse/mm10/ENCSR574CRQ_biosample.tsv' FACETS = '/hive/data/outside/woldlab/mouseDevTimecourse/mm10/mouse_development_M21.facets' - +REPLICATES = '/hive/data/outside/woldlab/mouseDevTimecourse/mm10/ENCSR574CRQ_replicates.tsv' + +# Biological order the data authors asked for, not alphabetical order. +TISSUE_ORDER = [ + 'thymus', + 'spleen', + 'liver', + 'heart', + 'skeletal muscle tissue', + 'urinary bladder', + 'adrenal gland', + 'kidney', + 'lung', + 'stomach', + 'intestine', + 'limb', + 'embryonic facial prominence', + 'forebrain', + 'midbrain', + 'hindbrain', + 'neural tube', +] + +# Tissue names abbreviated to fit the shortLabel budget (see MAX_SHORT_LABEL). +# The tissues sampled at embryonic ages get a 5-character age string, leaving +# only 6 characters for the name; the five P0-only tissues have room for 9. TISSUE_SHORT = { 'adrenal gland': 'adrenal', 'urinary bladder': 'bladder', 'embryonic facial prominence': 'face', 'skeletal muscle tissue': 'muscle', - 'neural tube': 'neuraltube', + 'neural tube': 'ntube', + 'forebrain': 'fbrain', + 'midbrain': 'mbrain', + 'hindbrain': 'hbrain', + 'intestine': 'intest', + 'stomach': 'stomch', } +# One-letter view code. A longer suffix does not fit, and truncating one gives +# sibling subtracks the same label. +VIEW_CODE = {'unique': 'U', 'all': 'A'} + +# A shortLabel a few characters over the nominal 17 is fine by house convention; +# 23 is the practical ceiling. What is NOT fine is two subtracks that differ only +# past column 17: hgTracks draws the left-hand label in a 17-character area by +# default (goldenPath/help/hgTracksHelp.html), so those render identically in the +# browser image even though their trackDb labels differ. Hence two separate +# checks: a length ceiling, and uniqueness of the first LEFT_LABEL_WIDTH +# characters, which is the invariant that actually protects the display. +MAX_SHORT_LABEL = 23 +LEFT_LABEL_WIDTH = 17 +MAX_LONG_LABEL = 80 + def tissue_short(name): return TISSUE_SHORT.get(name, name) def tissue_key(name): return name.replace(' ', '_') +def tissue_tag(name): + """subGroup tag carrying the biological sort order, e.g. t14_forebrain.""" + return 't%02d_%s' % (TISSUE_ORDER.index(name) + 1, tissue_key(name)) + + def age_label(life_stage, age): if life_stage == 'embryonic': return 'e' + str(age) if life_stage == 'postnatal': if float(age) == 0: return 'P0' return 'P' + str(age) return life_stage + '_' + str(age) def age_key(life_stage, age): return age_label(life_stage, age).replace('.', '_') def accession(url): return url.rsplit('/', 1)[-1].replace('.bigWig', '') def hex_to_rgb(h): h = h.lstrip('#') return ','.join(str(int(h[i:i + 2], 16)) for i in (0, 2, 4)) def load_colors(): """Build a (tissue, timepoint_label) -> hex color map from the .facets file.""" colors = {} with open(FACETS) as f: f.readline() for line in f: cols = line.rstrip('\n').split('\t') if len(cols) < 5: continue tissue, timepoint, hex_color = cols[2], cols[3], cols[4] colors[(tissue, timepoint)] = hex_color return colors +def load_replicates(): + """Build a file accession -> ENCODE biological replicate number map.""" + reps = {} + with open(REPLICATES) as f: + f.readline() + for line in f: + line = line.rstrip('\n') + if not line: + continue + acc, rep = line.split('\t') + reps[acc] = int(rep) + return reps + + def main(): tsv_path = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_TSV colors = load_colors() + replicates = load_replicates() + seen_labels = {} rows = [] with open(tsv_path) as f: header = f.readline().rstrip('\n').split('\t') for line in f: line = line.rstrip('\n') if not line: continue rows.append(dict(zip(header, line.split('\t')))) - # Replicate numbers (1 or 2) per experiment by biosample order. - exp_biosamples = {} - for row in rows: - exp = row['experiment'] - if exp not in exp_biosamples: - exp_biosamples[exp] = [] - exp_biosamples[exp].append(row['biosample']) - for exp in exp_biosamples: - exp_biosamples[exp].sort() - - tissues = sorted({row['biosample_term_name'] for row in rows}) + unknown = sorted({t for t in (r['biosample_term_name'] for r in rows) + if t not in TISSUE_ORDER}) + if unknown: + sys.exit('tissue missing from TISSUE_ORDER: %s' % ', '.join(unknown)) + ages = sorted( {(row['mouse_life_stage'], row['age']) for row in rows}, key=lambda p: (p[0], float(p[1])), ) # Composite parent stanza print(' track developmentTimecourseSignalMm10') print(' parent mouseDevTimecourse') print(' compositeTrack on') print(' type bigWig') - print(' shortLabel timecourse signal') + print(' shortLabel Timecourse Signal') print(' longLabel ENCODE mouse development time course bulk RNA-seq signal') print(' visibility hide') print(' group regulation') + print(' html developmentTimecourseSignalMm10') print(' subGroup1 view Views unique=Unique_reads all=All_reads') - tissue_grp = ' '.join(tissue_key(t) + '=' + tissue_key(t) for t in tissues) + tissue_grp = ' '.join(tissue_tag(t) + '=' + tissue_key(t) for t in TISSUE_ORDER) print(' subGroup2 tissue Tissue ' + tissue_grp) age_grp = ' '.join(age_key(ls, age) + '=' + age_label(ls, age) for ls, age in ages) print(' subGroup3 age Age ' + age_grp) print(' subGroup4 rep Replicate rep1=Rep_1 rep2=Rep_2') print(' dimensions dimX=age dimY=tissue dimA=rep') print(' dimensionAchecked rep1') print(' sortOrder view=+ tissue=+ age=+ rep=+') print(' dragAndDrop subTracks') print(' noInherit on') print() views = ( ('unique', 'Unique', 'Unique reads', 'signal_of_unique_reads', 'full'), ('all', 'All', 'All reads', 'signal_of_all_reads', 'hide'), ) for view_key, view_cap, view_label, url_col, view_visibility in views: view_container = 'developmentTimecourseSignalMm10View' + view_cap print(' track ' + view_container) print(' view ' + view_key) print(' parent developmentTimecourseSignalMm10') print(' shortLabel ' + view_label) print(' type bigWig') print(' visibility ' + view_visibility) print(' autoScale on') print(' maxHeightPixels 100:32:8') print() for row in rows: - exp = row['experiment'] - bs = row['biosample'] tissue = row['biosample_term_name'] ls = row['mouse_life_stage'] age = row['age'] - rep_num = exp_biosamples[exp].index(bs) + 1 - t_key = tissue_key(tissue) + t_tag = tissue_tag(tissue) a_key = age_key(ls, age) a_lbl = age_label(ls, age) t_short = tissue_short(tissue) url = row[url_col] acc = accession(url) big_data_url = '/gbdb/mm10/mouseDevTimecourse/' + acc + '.bigWig' - hex_color = colors.get((tissue, a_lbl), '#888888') + if acc not in replicates: + sys.exit('%s has no replicate number in %s; rerun ' + 'fetchReplicateNumbers.py' % (acc, REPLICATES)) + rep_num = replicates[acc] + + hex_color = colors.get((tissue, a_lbl)) + if hex_color is None: + sys.exit('no color in %s for (%s, %s)' % (FACETS, tissue, a_lbl)) rgb = hex_to_rgb(hex_color) track = 'developmentTimecourseSignalMm10_' + acc # Default on for rep1 + unique-reads (78 subtracks visible when the # composite is enabled). rep2 and all-reads remain off; users can # enable them from the trackUi page. parent_state = 'on' if (rep_num == 1 and view_key == 'unique') else 'off' - short = t_short + ' ' + a_lbl + ' r' + str(rep_num) + ' ' + view_key[:3] - short = short[:20] + # Title Case on the tissue and the replicate marker; e14.5 / P0 stay + # as written since they are standard developmental stage notation. + short = (t_short.capitalize() + ' ' + a_lbl + ' R' + str(rep_num) + + ' ' + VIEW_CODE[view_key]) + if len(short) > MAX_SHORT_LABEL: + sys.exit('shortLabel %d chars, limit %d: %s' + % (len(short), MAX_SHORT_LABEL, short)) + clipped = short[:LEFT_LABEL_WIDTH] + if clipped in seen_labels and seen_labels[clipped] != short: + sys.exit('shortLabels "%s" and "%s" are identical in the first %d ' + 'characters, so hgTracks draws them the same. Shorten the ' + 'tissue abbreviation in TISSUE_SHORT.' + % (seen_labels[clipped], short, LEFT_LABEL_WIDTH)) + seen_labels[clipped] = short long_ = ('ENCODE mouse ' + tissue + ' ' + a_lbl + ' rep' + str(rep_num) + ' ' + view_label + ' (' + acc + ')') - long_ = long_[:80] + if len(long_) > MAX_LONG_LABEL: + sys.exit('longLabel %d chars, limit %d: %s' + % (len(long_), MAX_LONG_LABEL, long_)) print(' track ' + track) print(' parent ' + view_container + ' ' + parent_state) - print(' subGroups view=' + view_key + ' tissue=' + t_key + print(' subGroups view=' + view_key + ' tissue=' + t_tag + ' age=' + a_key + ' rep=rep' + str(rep_num)) print(' type bigWig') print(' shortLabel ' + short) print(' longLabel ' + long_) print(' bigDataUrl ' + big_data_url) print(' color ' + rgb) print() if __name__ == '__main__': main()