8fc41cc92ad76bdd8fd35671329b47be0b8d29a5 mspeir Fri Aug 7 13:27:42 2026 -0700 singleCellSignalsPeaks: indent the generated .ra, guard the subtrack rename Indenting the hub stanzas broke the "track " match here, so subtracks kept their cellBrowser<Asm>_ names and no longer matched the facet metadata. The counts still passed, so nothing caught it. Now dedents on read, and refuses to write if a subtrack lands outside the track's namespace or a source name is not cellBrowser<Asm>_* (the rename slices at a fixed offset, so an odd name came out mangled but well-formed). Also indents subtracks one step under the composite, matching chainNet/encode3. That produced the whitespace-only .ra change committed separately; this reproduces it exactly. refs #37914 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> diff --git src/hg/makeDb/scripts/singleCellSignalsPeaks/makeSingleCellSignalsPeaksRa.py src/hg/makeDb/scripts/singleCellSignalsPeaks/makeSingleCellSignalsPeaksRa.py index 74a09b868e6..cfe8c684937 100644 --- src/hg/makeDb/scripts/singleCellSignalsPeaks/makeSingleCellSignalsPeaksRa.py +++ src/hg/makeDb/scripts/singleCellSignalsPeaks/makeSingleCellSignalsPeaksRa.py @@ -1,163 +1,201 @@ #!/usr/bin/env python3 """ Regenerate the native singleCellSignalsPeaks trackDb .ra for a genome assembly from the Cell Browser all-tracks-hub build (Redmine #37820 for hg38, #37914 for mm10). The native "singleCellSignalsPeaks" faceted composite is the Genome Browser version of the hub's main signal-&-peaks composite for that assembly (cellBrowserHg38 for hg38, cellBrowserMm10 for mm10). This script takes that composite's stanzas from the hub build and rewrites them into a native track: - the composite is renamed cellBrowser<Asm> -> singleCellSignalsPeaks - each subtrack bigDataUrl is repointed to the local /gbdb copy - subtrack colors / labels / types are carried through unchanged (so the harmonized cell-type labels and any per-track colors come along for free) The data files themselves are copied into /hive/data/genomes/<asm>/bed/singleCellSignalsPeaks/<served-relpath> (see copySingleCellSignalsPeaksFiles.py) and served via the /gbdb/<asm>/bbi/singleCellSignalsPeaks symlink; this script only (re)writes the trackDb .ra. Usage: makeSingleCellSignalsPeaksRa.py [--assembly hg38|mm10] [--stanzas STANZAS] [--out OUT] """ import re, os, sys, argparse from urllib.parse import urlparse # Where the hub build (build_manifest.py / build_stanzas.py) writes its stanzas and # metadata. That machinery builds the whole Cell Browser super hub, not just this track, # so it lives outside the kent tree; override with HUB_BUILD when it moves. HUB_BUILD = os.environ.get( "HUB_BUILD", "/hive/users/mspeir/claude/cell-browser/all-tracks-hub-build") TRACK = "singleCellSignalsPeaks" GROUP = "regulation" # ATAC-seq signal/peaks live with the ENCODE # regulatory tracks, not under singleCell ORG = {"hg38": "human", "mm10": "mouse"} # trackDb organism subdir per assembly def main(): ap = argparse.ArgumentParser() ap.add_argument("--assembly", default="hg38", choices=sorted(ORG)) ap.add_argument("--stanzas") ap.add_argument("--out") args = ap.parse_args() asm = args.assembly hub_composite = "cellBrowser" + asm.capitalize() # cellBrowserHg38 / cellBrowserMm10 gbdb = "/gbdb/%s/bbi/%s" % (asm, TRACK) stanzas = args.stanzas or os.path.join(HUB_BUILD, "stanzas/%s.trackDb.txt" % asm) out = args.out or os.path.join( os.path.dirname(os.path.abspath(__file__)), "../../trackDb/%s/%s/%s.ra" % (ORG[asm], asm, TRACK)) header = "\n".join([ "track " + TRACK, "compositeTrack faceted", "group " + GROUP, "visibility hide", "type bigBed 3", "shortLabel Single-cell ATAC-seq", "longLabel Single-cell ATAC-seq Peaks and Signals for UCSC Cell Browser datasets", "metaDataUrl %s/%s_metadata.tsv" % (gbdb, TRACK), "primaryKey Track", "subtrackUrls Dataset=https://cells.ucsc.edu/?ds=$$", "defaultSortField Cell_class", "maxCheckboxes 200", ]) # class ordering for subtrack priority: palette line order (neurons, glia, # vascular, immune, ...) so same-class tracks group together in the display, # with the source (hub/dataset) order preserved within a class. The subtrack's # broad class is recovered from its color (palette is 1:1 class<->color). color_rank = {} # prefer the palette archived alongside this script (the copy of record, written by # build_celltype_crosswalks.py); fall back to the hub build dir _palf = os.path.join(os.path.dirname(os.path.abspath(__file__)), "celltype-crosswalks", "celltype-palette.tsv") if not os.path.isfile(_palf): _palf = os.path.join(HUB_BUILD, "celltype-crosswalks", "celltype-palette.tsv") for _i, _l in enumerate(open(_palf)): _pp = _l.rstrip("\n").split("\t") if len(_pp) >= 2: color_rank[_pp[1]] = _i class_seq = {} # rank -> running counter within that class # a source path segment of "old" / "*.old" / "*_old" marks deprecated data # (e.g. cortex-atac/hub/interact.old/); the hub may keep it, but the native # track must not carry it. OLD_SEG = re.compile(r"(^|/)[^/]*(\.old|_old|\bold)($|/)", re.I) out_stanzas = [header] n = skipped_old = n_stanzas = n_parented = 0 + bad_src = [] # source track names not in the cellBrowser<Asm>_ namespace for s in re.split(r"\n\s*\n", open(stanzas).read().strip()): - lines = s.splitlines() + # Dedent: the hub stanzas are indented to show their hierarchy, and this script + # re-indents on write. Parsing them indented broke the "track " rename below + # without tripping any count check -- subtracks kept their cellBrowser<Asm> names + # and no longer matched their facet metadata rows. + lines = [l.lstrip() for l in s.splitlines()] n_stanzas += 1 # Match on the parent's first token rather than the whole line. The exact-string # compare this replaces would have silently skipped every stanza if the hub ever # emitted "parent <composite> off" or changed its spacing, leaving a header-only # .ra and a zero exit status. parent = next((l for l in lines if l.strip().startswith("parent ")), "") ptoks = parent.split() if len(ptoks) >= 2: n_parented += 1 if len(ptoks) < 2 or ptoks[1] != hub_composite: continue bdu = next((l for l in lines if l.strip().startswith("bigDataUrl ")), "") rel_check = urlparse(bdu.split(None, 1)[1].strip()).path if bdu else "" if OLD_SEG.search(rel_check): skipped_old += 1 continue n += 1 # priority groups tracks by broad class (via color), source order within color = next((l.split(None, 1)[1].strip() for l in lines if l.strip().startswith("color ")), "") rank = color_rank.get(color, len(color_rank)) # unknown/uncolored last seq = class_seq.get(rank, 0); class_seq[rank] = seq + 1 priority = rank * 100000 + seq newl = [] for l in lines: if l.startswith("track "): - suffix = l.split(None, 1)[1][len(hub_composite) + 1:] + # Validate the SOURCE name before slicing it. This used to chop a fixed + # number of characters off whatever it was given, so a track not actually + # named cellBrowser<Asm>_* came out silently mangled but well-formed + # (BOGUS_allen_basal_ganglia_atac__dorsal -> ..._ia_atac__dorsal), which no + # check on the output could catch. + src_name = l.split(None, 1)[1] + if not src_name.startswith(hub_composite + "_"): + bad_src.append(src_name) + continue + suffix = src_name[len(hub_composite) + 1:] newl.append("track %s_%s" % (TRACK, suffix)) elif l.strip().startswith("parent ") and l.split()[1] == hub_composite: # "off" so every subtrack is unchecked by default; the user turns # on individual tracks via the faceted selector newl.append("parent " + TRACK + " off") newl.append("priority " + str(priority)) elif l.strip().startswith("bigDataUrl "): rel = urlparse(l.split(None, 1)[1].strip()).path.lstrip("/") newl.append("bigDataUrl %s/%s" % (gbdb, rel)) else: newl.append(l) out_stanzas.append("\n".join(newl)) + # Every emitted subtrack must be "track <TRACK>_<non-empty suffix>". Checking only the + # prefix was not enough: the rename always prepends it, so a source name that did not + # start with the hub composite still passed, just with an empty suffix. + if bad_src: + sys.exit("ERROR: %d source track name(s) are not %s_* , e.g. %r. Renaming them " + "would silently mangle the name. Has the hub stanza layout changed?" + % (len(bad_src), hub_composite, bad_src[0])) + _ok = re.compile(r"^track %s_\S+$" % re.escape(TRACK)) + bad = [x.split("\n", 1)[0] for x in out_stanzas[1:] + if not _ok.match(x.split("\n", 1)[0])] + if bad: + sys.exit("ERROR: %d subtrack(s) were not renamed into the %s namespace, e.g. %r. " + "Has the hub stanza layout changed?" % (len(bad), TRACK, bad[0])) + # Sanity checks: fail loudly rather than write a truncated .ra. A hub-format change # that stops the parent line matching would otherwise produce a header-only file and # exit 0, and the next trackDb load would quietly drop every subtrack. if n == 0: sys.exit("ERROR: no subtracks matched composite '%s' in %s " "(%d stanzas, %d with a parent line). Has the hub stanza format " "changed?" % (hub_composite, stanzas, n_stanzas, n_parented)) # The facet metadata is the parallel artifact: build_stanzas writes one row per # subtrack of this composite, so the counts must agree once the old-dir skips are # added back. A mismatch means the .ra and the metadata disagree, which shows up in # the browser as subtracks with no facet row (or facet rows with no track). meta = os.path.join(HUB_BUILD, "meta", "%s.metadata.tsv" % asm) if os.path.isfile(meta): with open(meta) as fh: meta_rows = sum(1 for _ in fh) - 1 # minus the header if meta_rows != n + skipped_old: sys.exit("ERROR: %s has %d rows but %d subtracks were kept (+%d old-dir " "skipped); the .ra and the facet metadata must match 1:1" % (meta, meta_rows, n, skipped_old)) else: sys.stderr.write("WARNING: no facet metadata at %s, skipping the 1:1 check\n" % meta) + # Indent subtracks under the composite, as the trackDb .ra files in the tree do + # (chainNet, encode3): the container sits flush left and each level below it is + # indented one step, with every line of the stanza moving together. + def indent(stanza, width=4): + lines = stanza.split("\n") + if not any(l.startswith("parent ") for l in lines): + return stanza + pad = " " * width + return "\n".join(pad + l if l.strip() else l for l in lines) + with open(os.path.abspath(out), "w") as fh: - fh.write("\n\n".join(out_stanzas) + "\n") + fh.write("\n\n".join(indent(s) for s in out_stanzas) + "\n") print("wrote %s: %d subtracks (assembly=%s, composite=%s, group=%s; skipped %d old-dir)" % ( out, n, asm, hub_composite, GROUP, skipped_old)) if __name__ == "__main__": main()