fe02c55d26614f1d5059c884fb2594f9b4d5ad50 mspeir Wed Aug 12 08:02:19 2026 -0700 singleCellSignalsPeaks: point at the hub build's new home, fix the file-copy script The hub build now lives in the cellBrowser repo under ucsc/allTracksHub, so the makeDocs link there instead of a personal work dir. HUB_BUILD in these scripts is the build's output dir, not its code; comments say so now, paths unchanged. copySingleCellSignalsPeaksFiles.py was silently copying nothing: it compared a whole line against "parent ", but stanzas are now indented and read "parent off". Now dedents, matches the first token, and refuses to run on zero subtracks. Dry runs give 925 hg38 / 587 mm10, none missing. refs #37914 diff --git src/hg/makeDb/scripts/singleCellSignalsPeaks/makeSingleCellSignalsPeaksRa.py src/hg/makeDb/scripts/singleCellSignalsPeaks/makeSingleCellSignalsPeaksRa.py index cfe8c684937..2c2b9f7756f 100644 --- src/hg/makeDb/scripts/singleCellSignalsPeaks/makeSingleCellSignalsPeaksRa.py +++ src/hg/makeDb/scripts/singleCellSignalsPeaks/makeSingleCellSignalsPeaksRa.py @@ -1,201 +1,203 @@ #!/usr/bin/env python3 """ Regenerate the native singleCellSignalsPeaks trackDb .ra for a genome assembly from the Cell Browser all-tracks-hub build (Redmine #37820 for hg38, #37914 for mm10). The native "singleCellSignalsPeaks" faceted composite is the Genome Browser version of the hub's main signal-&-peaks composite for that assembly (cellBrowserHg38 for hg38, cellBrowserMm10 for mm10). This script takes that composite's stanzas from the hub build and rewrites them into a native track: - the composite is renamed cellBrowser -> singleCellSignalsPeaks - each subtrack bigDataUrl is repointed to the local /gbdb copy - subtrack colors / labels / types are carried through unchanged (so the harmonized cell-type labels and any per-track colors come along for free) The data files themselves are copied into /hive/data/genomes//bed/singleCellSignalsPeaks/ (see copySingleCellSignalsPeaksFiles.py) and served via the /gbdb//bbi/singleCellSignalsPeaks symlink; this script only (re)writes the trackDb .ra. Usage: makeSingleCellSignalsPeaksRa.py [--assembly hg38|mm10] [--stanzas STANZAS] [--out OUT] """ import re, os, sys, argparse from urllib.parse import urlparse -# Where the hub build (build_manifest.py / build_stanzas.py) writes its stanzas and -# metadata. That machinery builds the whole Cell Browser super hub, not just this track, -# so it lives outside the kent tree; override with HUB_BUILD when it moves. +# Where the hub build writes its stanzas and metadata -- its OUTPUT dir, not its code. +# The build itself lives in the cellBrowser repo (ucsc/allTracksHub), since it builds the +# whole Cell Browser super hub and not just this track: +# https://github.com/ucscGenomeBrowser/cellBrowser/tree/develop/ucsc/allTracksHub +# Its output dir is set there by CBHUB_OUT; keep this default in step with it. HUB_BUILD = os.environ.get( "HUB_BUILD", "/hive/users/mspeir/claude/cell-browser/all-tracks-hub-build") TRACK = "singleCellSignalsPeaks" GROUP = "regulation" # ATAC-seq signal/peaks live with the ENCODE # regulatory tracks, not under singleCell ORG = {"hg38": "human", "mm10": "mouse"} # trackDb organism subdir per assembly def main(): ap = argparse.ArgumentParser() ap.add_argument("--assembly", default="hg38", choices=sorted(ORG)) ap.add_argument("--stanzas") ap.add_argument("--out") args = ap.parse_args() asm = args.assembly hub_composite = "cellBrowser" + asm.capitalize() # cellBrowserHg38 / cellBrowserMm10 gbdb = "/gbdb/%s/bbi/%s" % (asm, TRACK) stanzas = args.stanzas or os.path.join(HUB_BUILD, "stanzas/%s.trackDb.txt" % asm) out = args.out or os.path.join( os.path.dirname(os.path.abspath(__file__)), "../../trackDb/%s/%s/%s.ra" % (ORG[asm], asm, TRACK)) header = "\n".join([ "track " + TRACK, "compositeTrack faceted", "group " + GROUP, "visibility hide", "type bigBed 3", "shortLabel Single-cell ATAC-seq", "longLabel Single-cell ATAC-seq Peaks and Signals for UCSC Cell Browser datasets", "metaDataUrl %s/%s_metadata.tsv" % (gbdb, TRACK), "primaryKey Track", "subtrackUrls Dataset=https://cells.ucsc.edu/?ds=$$", "defaultSortField Cell_class", "maxCheckboxes 200", ]) # class ordering for subtrack priority: palette line order (neurons, glia, # vascular, immune, ...) so same-class tracks group together in the display, # with the source (hub/dataset) order preserved within a class. The subtrack's # broad class is recovered from its color (palette is 1:1 class<->color). color_rank = {} # prefer the palette archived alongside this script (the copy of record, written by # build_celltype_crosswalks.py); fall back to the hub build dir _palf = os.path.join(os.path.dirname(os.path.abspath(__file__)), "celltype-crosswalks", "celltype-palette.tsv") if not os.path.isfile(_palf): _palf = os.path.join(HUB_BUILD, "celltype-crosswalks", "celltype-palette.tsv") for _i, _l in enumerate(open(_palf)): _pp = _l.rstrip("\n").split("\t") if len(_pp) >= 2: color_rank[_pp[1]] = _i class_seq = {} # rank -> running counter within that class # a source path segment of "old" / "*.old" / "*_old" marks deprecated data # (e.g. cortex-atac/hub/interact.old/); the hub may keep it, but the native # track must not carry it. OLD_SEG = re.compile(r"(^|/)[^/]*(\.old|_old|\bold)($|/)", re.I) out_stanzas = [header] n = skipped_old = n_stanzas = n_parented = 0 bad_src = [] # source track names not in the cellBrowser_ namespace for s in re.split(r"\n\s*\n", open(stanzas).read().strip()): # Dedent: the hub stanzas are indented to show their hierarchy, and this script # re-indents on write. Parsing them indented broke the "track " rename below # without tripping any count check -- subtracks kept their cellBrowser names # and no longer matched their facet metadata rows. lines = [l.lstrip() for l in s.splitlines()] n_stanzas += 1 # Match on the parent's first token rather than the whole line. The exact-string # compare this replaces would have silently skipped every stanza if the hub ever # emitted "parent off" or changed its spacing, leaving a header-only # .ra and a zero exit status. parent = next((l for l in lines if l.strip().startswith("parent ")), "") ptoks = parent.split() if len(ptoks) >= 2: n_parented += 1 if len(ptoks) < 2 or ptoks[1] != hub_composite: continue bdu = next((l for l in lines if l.strip().startswith("bigDataUrl ")), "") rel_check = urlparse(bdu.split(None, 1)[1].strip()).path if bdu else "" if OLD_SEG.search(rel_check): skipped_old += 1 continue n += 1 # priority groups tracks by broad class (via color), source order within color = next((l.split(None, 1)[1].strip() for l in lines if l.strip().startswith("color ")), "") rank = color_rank.get(color, len(color_rank)) # unknown/uncolored last seq = class_seq.get(rank, 0); class_seq[rank] = seq + 1 priority = rank * 100000 + seq newl = [] for l in lines: if l.startswith("track "): # Validate the SOURCE name before slicing it. This used to chop a fixed # number of characters off whatever it was given, so a track not actually # named cellBrowser_* came out silently mangled but well-formed # (BOGUS_allen_basal_ganglia_atac__dorsal -> ..._ia_atac__dorsal), which no # check on the output could catch. src_name = l.split(None, 1)[1] if not src_name.startswith(hub_composite + "_"): bad_src.append(src_name) continue suffix = src_name[len(hub_composite) + 1:] newl.append("track %s_%s" % (TRACK, suffix)) elif l.strip().startswith("parent ") and l.split()[1] == hub_composite: # "off" so every subtrack is unchecked by default; the user turns # on individual tracks via the faceted selector newl.append("parent " + TRACK + " off") newl.append("priority " + str(priority)) elif l.strip().startswith("bigDataUrl "): rel = urlparse(l.split(None, 1)[1].strip()).path.lstrip("/") newl.append("bigDataUrl %s/%s" % (gbdb, rel)) else: newl.append(l) out_stanzas.append("\n".join(newl)) # Every emitted subtrack must be "track _". Checking only the # prefix was not enough: the rename always prepends it, so a source name that did not # start with the hub composite still passed, just with an empty suffix. if bad_src: sys.exit("ERROR: %d source track name(s) are not %s_* , e.g. %r. Renaming them " "would silently mangle the name. Has the hub stanza layout changed?" % (len(bad_src), hub_composite, bad_src[0])) _ok = re.compile(r"^track %s_\S+$" % re.escape(TRACK)) bad = [x.split("\n", 1)[0] for x in out_stanzas[1:] if not _ok.match(x.split("\n", 1)[0])] if bad: sys.exit("ERROR: %d subtrack(s) were not renamed into the %s namespace, e.g. %r. " "Has the hub stanza layout changed?" % (len(bad), TRACK, bad[0])) # Sanity checks: fail loudly rather than write a truncated .ra. A hub-format change # that stops the parent line matching would otherwise produce a header-only file and # exit 0, and the next trackDb load would quietly drop every subtrack. if n == 0: sys.exit("ERROR: no subtracks matched composite '%s' in %s " "(%d stanzas, %d with a parent line). Has the hub stanza format " "changed?" % (hub_composite, stanzas, n_stanzas, n_parented)) # The facet metadata is the parallel artifact: build_stanzas writes one row per # subtrack of this composite, so the counts must agree once the old-dir skips are # added back. A mismatch means the .ra and the metadata disagree, which shows up in # the browser as subtracks with no facet row (or facet rows with no track). meta = os.path.join(HUB_BUILD, "meta", "%s.metadata.tsv" % asm) if os.path.isfile(meta): with open(meta) as fh: meta_rows = sum(1 for _ in fh) - 1 # minus the header if meta_rows != n + skipped_old: sys.exit("ERROR: %s has %d rows but %d subtracks were kept (+%d old-dir " "skipped); the .ra and the facet metadata must match 1:1" % (meta, meta_rows, n, skipped_old)) else: sys.stderr.write("WARNING: no facet metadata at %s, skipping the 1:1 check\n" % meta) # Indent subtracks under the composite, as the trackDb .ra files in the tree do # (chainNet, encode3): the container sits flush left and each level below it is # indented one step, with every line of the stanza moving together. def indent(stanza, width=4): lines = stanza.split("\n") if not any(l.startswith("parent ") for l in lines): return stanza pad = " " * width return "\n".join(pad + l if l.strip() else l for l in lines) with open(os.path.abspath(out), "w") as fh: fh.write("\n\n".join(indent(s) for s in out_stanzas) + "\n") print("wrote %s: %d subtracks (assembly=%s, composite=%s, group=%s; skipped %d old-dir)" % ( out, n, asm, hub_composite, GROUP, skipped_old)) if __name__ == "__main__": main()