bc527b6264234c33824854e5b596fb6f790983d4 lrnassar Tue Aug 4 14:22:28 2026 -0700 Fix release blockers and data errors found during QA of the mouseDevTimecourse tracks. refs #37001 Add maxWindowToDraw 10000000 to the six bigBarChart subtracks. Without it a whole-chromosome view asked the track to draw 3445 items x 156 bars and took 71 seconds on mm10 and 69 on mm39. The superTrack is on by default, so any mouse user zooming out could hit it. Now 254 ms and 86 ms. Correct a 1-bp off-by-one in the bigBarChart chromStart. The hub builder wrote 1-based GTF gene starts into the 0-based BED chromStart field, so every gene sat one base right of its true start while chromEnd was correct. Measured against GENCODE on mm10 before the fix, 42081/42093 genes (VM21) and 35323/35333 (VM4) were start+1 with none exact. Added fixBarChartStarts.sh, which rebuilds the files and refuses to run on one that has already been corrected. Originals kept as *.bb.preStartFix. Reported upstream to the hub author. Correct the replicate numbers on the bigWig signal composite. The biosample TSV has no replicate column, so generateBigwigTrackDb.py had been deriving one by sorting biosample accessions alphabetically, which mislabeled 124 of the 312 subtracks and flipped the default-on state of 62 of them. Added fetchReplicateNumbers.py to read the real biological_replicate_number from the ENCODE portal. Rewrite the signal shortLabels. They had been hard-truncated at 20 characters, which left 20 subtracks with duplicate labels. The generator now emits a one-letter view code and errors out if two labels match within the 17 characters hgTracks draws in the left label area, rather than silently truncating. Make generateBigwigTrackDb.py reproduce the committed .ra. It now emits the two-digit tissue prefixes that give the author-requested biological order, and the html setting, instead of depending on a one-off patch applied afterwards. Add barChartMerge, barChartMetric, labelFields and defaultLabelFields, and Title Case all shortLabels. Update the nine description pages: replicate wording to match barChartMerge, GitHub source links to the makedoc, build scripts and trackDb in Methods, an mm39 liftOver accounting note, and remove a duplicated sentence from the shared Display include. Add curl -f to downloadBigwigs.sh so an HTTP error body is never saved as a bigWig and then skipped forever by the restart check. diff --git src/hg/makeDb/scripts/mouseDevTimecourse/fetchReplicateNumbers.py src/hg/makeDb/scripts/mouseDevTimecourse/fetchReplicateNumbers.py new file mode 100755 index 00000000000..0da2ec9b6ff --- /dev/null +++ src/hg/makeDb/scripts/mouseDevTimecourse/fetchReplicateNumbers.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +""" +Look up the ENCODE biological replicate number for each bigWig in the mouse +developmental time course and write it to a TSV that +generateBigwigTrackDb.py reads. + +Diane's ENCSR574CRQ_biosample.tsv (#36998 attachment) has no replicate +column, so the replicate number has to come from the ENCODE portal. Without +this step generateBigwigTrackDb.py has to guess, and guessing by biosample +accession order mislabels 124 of the 312 subtracks (#37001). + +Reads the file accessions out of the biosample TSV, queries the portal +search endpoint in batches, and writes accession<TAB>replicate to +ENCSR574CRQ_replicates.tsv in the same directory. + +Output goes to stdout. Redirect to a file. +""" + +import json +import sys +import time +import urllib.request + +DEFAULT_TSV = '/hive/data/outside/woldlab/mouseDevTimecourse/mm10/ENCSR574CRQ_biosample.tsv' + +SEARCH = ('https://www.encodeproject.org/search/?type=File&limit=all&format=json' + '&field=accession&field=biological_replicates&field=status') +BATCH = 40 + + +def accessions(tsv_path): + """File accessions from the signal_of_unique_reads and signal_of_all_reads URLs.""" + accs = [] + with open(tsv_path) as f: + header = f.readline().rstrip('\n').split('\t') + for line in f: + line = line.rstrip('\n') + if not line: + continue + row = dict(zip(header, line.split('\t'))) + for col in ('signal_of_unique_reads', 'signal_of_all_reads'): + accs.append(row[col].rsplit('/', 1)[-1].replace('.bigWig', '')) + return accs + + +def fetch(accs): + """accession -> biological replicate number, from the ENCODE portal.""" + reps = {} + for i in range(0, len(accs), BATCH): + group = accs[i:i + BATCH] + url = SEARCH + ''.join('&accession=' + a for a in group) + for attempt in range(4): + try: + req = urllib.request.Request(url, headers={'Accept': 'application/json'}) + results = json.load(urllib.request.urlopen(req, timeout=90)) + break + except Exception as e: + sys.stderr.write('retry %d for batch at %d: %s\n' % (attempt, i, e)) + time.sleep(3) + else: + sys.exit('ENCODE portal query failed for batch starting at %d' % i) + + for f in results.get('@graph', []): + if f['status'] != 'released': + sys.stderr.write('warning: %s status is %s, not released\n' + % (f['accession'], f['status'])) + bio = f['biological_replicates'] + if len(bio) != 1: + sys.exit('%s covers %d biological replicates; expected exactly one' + % (f['accession'], len(bio))) + reps[f['accession']] = bio[0] + sys.stderr.write('fetched %d of %d\n' % (len(reps), len(accs))) + + missing = [a for a in accs if a not in reps] + if missing: + sys.exit('no replicate number returned for: %s' % ' '.join(missing)) + return reps + + +def main(): + tsv_path = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_TSV + accs = accessions(tsv_path) + reps = fetch(accs) + print('accession\treplicate') + for acc in accs: + print('%s\t%d' % (acc, reps[acc])) + + +if __name__ == '__main__': + main()