8fc41cc92ad76bdd8fd35671329b47be0b8d29a5 mspeir Fri Aug 7 13:27:42 2026 -0700 singleCellSignalsPeaks: indent the generated .ra, guard the subtrack rename Indenting the hub stanzas broke the "track " match here, so subtracks kept their cellBrowser<Asm>_ names and no longer matched the facet metadata. The counts still passed, so nothing caught it. Now dedents on read, and refuses to write if a subtrack lands outside the track's namespace or a source name is not cellBrowser<Asm>_* (the rename slices at a fixed offset, so an odd name came out mangled but well-formed). Also indents subtracks one step under the composite, matching chainNet/encode3. That produced the whitespace-only .ra change committed separately; this reproduces it exactly. refs #37914 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> diff --git src/hg/makeDb/scripts/singleCellSignalsPeaks/makeSingleCellSignalsPeaksRa.py src/hg/makeDb/scripts/singleCellSignalsPeaks/makeSingleCellSignalsPeaksRa.py index 74a09b868e6..cfe8c684937 100644 --- src/hg/makeDb/scripts/singleCellSignalsPeaks/makeSingleCellSignalsPeaksRa.py +++ src/hg/makeDb/scripts/singleCellSignalsPeaks/makeSingleCellSignalsPeaksRa.py @@ -77,87 +77,125 @@ if not os.path.isfile(_palf): _palf = os.path.join(HUB_BUILD, "celltype-crosswalks", "celltype-palette.tsv") for _i, _l in enumerate(open(_palf)): _pp = _l.rstrip("\n").split("\t") if len(_pp) >= 2: color_rank[_pp[1]] = _i class_seq = {} # rank -> running counter within that class # a source path segment of "old" / "*.old" / "*_old" marks deprecated data # (e.g. cortex-atac/hub/interact.old/); the hub may keep it, but the native # track must not carry it. OLD_SEG = re.compile(r"(^|/)[^/]*(\.old|_old|\bold)($|/)", re.I) out_stanzas = [header] n = skipped_old = n_stanzas = n_parented = 0 + bad_src = [] # source track names not in the cellBrowser<Asm>_ namespace for s in re.split(r"\n\s*\n", open(stanzas).read().strip()): - lines = s.splitlines() + # Dedent: the hub stanzas are indented to show their hierarchy, and this script + # re-indents on write. Parsing them indented broke the "track " rename below + # without tripping any count check -- subtracks kept their cellBrowser<Asm> names + # and no longer matched their facet metadata rows. + lines = [l.lstrip() for l in s.splitlines()] n_stanzas += 1 # Match on the parent's first token rather than the whole line. The exact-string # compare this replaces would have silently skipped every stanza if the hub ever # emitted "parent <composite> off" or changed its spacing, leaving a header-only # .ra and a zero exit status. parent = next((l for l in lines if l.strip().startswith("parent ")), "") ptoks = parent.split() if len(ptoks) >= 2: n_parented += 1 if len(ptoks) < 2 or ptoks[1] != hub_composite: continue bdu = next((l for l in lines if l.strip().startswith("bigDataUrl ")), "") rel_check = urlparse(bdu.split(None, 1)[1].strip()).path if bdu else "" if OLD_SEG.search(rel_check): skipped_old += 1 continue n += 1 # priority groups tracks by broad class (via color), source order within color = next((l.split(None, 1)[1].strip() for l in lines if l.strip().startswith("color ")), "") rank = color_rank.get(color, len(color_rank)) # unknown/uncolored last seq = class_seq.get(rank, 0); class_seq[rank] = seq + 1 priority = rank * 100000 + seq newl = [] for l in lines: if l.startswith("track "): - suffix = l.split(None, 1)[1][len(hub_composite) + 1:] + # Validate the SOURCE name before slicing it. This used to chop a fixed + # number of characters off whatever it was given, so a track not actually + # named cellBrowser<Asm>_* came out silently mangled but well-formed + # (BOGUS_allen_basal_ganglia_atac__dorsal -> ..._ia_atac__dorsal), which no + # check on the output could catch. + src_name = l.split(None, 1)[1] + if not src_name.startswith(hub_composite + "_"): + bad_src.append(src_name) + continue + suffix = src_name[len(hub_composite) + 1:] newl.append("track %s_%s" % (TRACK, suffix)) elif l.strip().startswith("parent ") and l.split()[1] == hub_composite: # "off" so every subtrack is unchecked by default; the user turns # on individual tracks via the faceted selector newl.append("parent " + TRACK + " off") newl.append("priority " + str(priority)) elif l.strip().startswith("bigDataUrl "): rel = urlparse(l.split(None, 1)[1].strip()).path.lstrip("/") newl.append("bigDataUrl %s/%s" % (gbdb, rel)) else: newl.append(l) out_stanzas.append("\n".join(newl)) + # Every emitted subtrack must be "track <TRACK>_<non-empty suffix>". Checking only the + # prefix was not enough: the rename always prepends it, so a source name that did not + # start with the hub composite still passed, just with an empty suffix. + if bad_src: + sys.exit("ERROR: %d source track name(s) are not %s_* , e.g. %r. Renaming them " + "would silently mangle the name. Has the hub stanza layout changed?" + % (len(bad_src), hub_composite, bad_src[0])) + _ok = re.compile(r"^track %s_\S+$" % re.escape(TRACK)) + bad = [x.split("\n", 1)[0] for x in out_stanzas[1:] + if not _ok.match(x.split("\n", 1)[0])] + if bad: + sys.exit("ERROR: %d subtrack(s) were not renamed into the %s namespace, e.g. %r. " + "Has the hub stanza layout changed?" % (len(bad), TRACK, bad[0])) + # Sanity checks: fail loudly rather than write a truncated .ra. A hub-format change # that stops the parent line matching would otherwise produce a header-only file and # exit 0, and the next trackDb load would quietly drop every subtrack. if n == 0: sys.exit("ERROR: no subtracks matched composite '%s' in %s " "(%d stanzas, %d with a parent line). Has the hub stanza format " "changed?" % (hub_composite, stanzas, n_stanzas, n_parented)) # The facet metadata is the parallel artifact: build_stanzas writes one row per # subtrack of this composite, so the counts must agree once the old-dir skips are # added back. A mismatch means the .ra and the metadata disagree, which shows up in # the browser as subtracks with no facet row (or facet rows with no track). meta = os.path.join(HUB_BUILD, "meta", "%s.metadata.tsv" % asm) if os.path.isfile(meta): with open(meta) as fh: meta_rows = sum(1 for _ in fh) - 1 # minus the header if meta_rows != n + skipped_old: sys.exit("ERROR: %s has %d rows but %d subtracks were kept (+%d old-dir " "skipped); the .ra and the facet metadata must match 1:1" % (meta, meta_rows, n, skipped_old)) else: sys.stderr.write("WARNING: no facet metadata at %s, skipping the 1:1 check\n" % meta) + # Indent subtracks under the composite, as the trackDb .ra files in the tree do + # (chainNet, encode3): the container sits flush left and each level below it is + # indented one step, with every line of the stanza moving together. + def indent(stanza, width=4): + lines = stanza.split("\n") + if not any(l.startswith("parent ") for l in lines): + return stanza + pad = " " * width + return "\n".join(pad + l if l.strip() else l for l in lines) + with open(os.path.abspath(out), "w") as fh: - fh.write("\n\n".join(out_stanzas) + "\n") + fh.write("\n\n".join(indent(s) for s in out_stanzas) + "\n") print("wrote %s: %d subtracks (assembly=%s, composite=%s, group=%s; skipped %d old-dir)" % ( out, n, asm, hub_composite, GROUP, skipped_old)) if __name__ == "__main__": main()