679b48855a32f8ea1e1ea8ca3c095a0c2c7f6c7a max Tue Sep 8 10:08:41 2026 -0700 detailsScript: add a scatterPlot plot type, and use it for pcLAI Clicking a pcLAI window now shows where that window sits in the ancestry space it was placed in: a scatterplot of the 1000 Genomes reference haplotypes with the window's own PCA coordinate and its segment's coordinate marked on it. The numbers were already on the details page and told a reader almost nothing. New plot type scatterPlot (hg/js/hgc.scatterPlot.js), driven the same way as histogram. Background points come from a JSON or TSV file named by dataUrl and may carry a category, which colors them and builds a legend, and a label, which is shown on mouseover. The cloud is drawn on a canvas, since these files hold thousands of points and that many <circle> elements make the page crawl; axes and the highlighted points stay SVG on top. Point lookup for the mouseover goes through a cell index so a large file stays smooth. Two additions serve every plot type, not just this one: - exportFields, a config key listing further bigBed fields whose values are passed to the module as a fieldValues object. Without it a plot needing two coordinates would need them packed into one field, and pcLAI keeps them in pca and pcaSegment. Only fields that exist in the bigBed are exported, at most 32, and the JSON types are checked rather than asserted because jsonListVal and jsonStringVal errAbort and this JSON is written by a hub. - a config key ending in Url is treated as a file, by the convention trackSettingIsFile() already uses, and a relative one is resolved against the track's own bigDataUrl. The module does not fetch it directly; it asks hgTrackUi for it, the route facetedComposite uses for its metadata. That checks the canonicalized path against the hubs on the cart and reads it with udc, so a hub-relative path works even for a hub loaded from a local path (the GenArk /gbdb hubs), no CORS header is needed, and a file outside a connected hub cannot be read. Verified that /etc/passwd, file://, a dot-dot escape, an unattached hub and an unrelated host are all refused with 400. When the session has file caching off, hgc now exports udcTimeout the way hgTrackUi does and the module POSTs, so the browser cannot answer from cache. Fixes a crash reachable from any hub: "detailsScript.<plotType>.<field> null" segfaulted hgc, because jsonObjectVal returns NULL for a JSON null and the hash routines dereference it. This hit the shipped histogram type too. trackDbSettingsGen.py stopped reading a setting's description at the first "Example:" paragraph and never read <ul> at all, so it dropped everything after the first example and every list item. That silently truncated 226 of the 264 descriptions, including spectrum's minGrayLevel/scoreMin/scoreMax bullets, and would have dropped this whole scatterPlot section. It now skips the Example label instead of stopping, and folds list items in. No setting loses a word and none gains or loses an example. pcLAI wiring: the background file is the authors' published reference panel (github.com/AI-sandbox/hprc-pclai reference_pca_metadata.tsv), converted by hprc2annotMakePclaiRefPanel.py -- 3122 haplotypes, 21 populations, 94 KB, one file for the collection since it is the reference space rather than per-assembly data. The four values pcaSegment takes across all 460 assemblies turn out to be the four continental cluster centres, so the highlighted segment dot always lands on one of them. genark: addContrib now rewrites a "...Url" inside a detailsScript value the same way it rewrites bigDataUrl, and symlinks the collection's shared root-level data files next to the docs, so contrib/<name>/<file> resolves in the deeper GenArk layout. It writes the alpha tier only, leaving the assembly's default hub alone, and clears any unmarked copy of the collection's stanzas that the assembly build baked in, which would otherwise leave the hub declaring each track twice. refs #35415 diff --git src/utils/genark/genark src/utils/genark/genark index 3d5d1f349cf..3a03ab50590 100755 --- src/utils/genark/genark +++ src/utils/genark/genark @@ -154,143 +154,201 @@ """Create/replace an absolute symlink linkPath -> target.""" if dryRun: print(" ln -sf %s %s" % (target, linkPath)) return if os.path.islink(linkPath) or os.path.exists(linkPath): os.remove(linkPath) os.symlink(target, linkPath) def rewriteTrackDb(srcPath, name): """Return the trackDb text with local bigDataUrl/html paths made hub-root relative (contrib/<name>/...). Remote (http/https/ftp) bigDataUrls and already-prefixed paths are left untouched.""" out = [] prefix = "contrib/%s/" % name + + def rebase(val): + """Make one local path hub-root relative; leave a URL or a done path alone.""" + if re.match(r"[a-z]+://", val) or val.startswith(prefix): + return val + return prefix + os.path.basename(val) + for line in open(srcPath): stripped = line.lstrip() indent = line[:len(line) - len(stripped)] m = re.match(r"(bigDataUrl|linkDataUrl)\s+(\S+)\s*$", stripped) if m: - key, val = m.group(1), m.group(2) - if not re.match(r"[a-z]+://", val) and not val.startswith(prefix): - val = prefix + os.path.basename(val) - out.append("%s%s %s\n" % (indent, key, val)) + out.append("%s%s %s\n" % (indent, m.group(1), rebase(m.group(2)))) continue m = re.match(r"html\s+(\S+)\s*$", stripped) if m: - val = m.group(1) - if not val.startswith(prefix): - val = prefix + os.path.basename(val) - out.append("%shtml %s\n" % (indent, val)) + out.append("%shtml %s\n" % (indent, rebase(m.group(1)))) + continue + # detailsScript carries its file in a "...Url" key inside a JSON value, and + # the collection writes it relative to the accession dir. In this layout the + # data sits one level deeper (contrib/<name>/), so rebase it the same way. + if stripped.startswith("detailsScript."): + line = re.sub(r'("[A-Za-z0-9_]*[Uu]rl"\s*:\s*")([^"]+)(")', + lambda m: m.group(1) + rebase(m.group(2)) + m.group(3), line) + out.append(line) + continue + out.append(line) + return "".join(out) + + +def dropTopLevelTracks(text, trackNames): + """Return the hub text with the top-level stanzas of trackNames removed. + + The assembly build can bake a snapshot of a contrib collection straight into + its hub file, without our markers. Left in place those stanzas would collide + with the block we add, giving the hub two "track <name>" entries. + + Stanzas in a useOneFile hub are separated by a blank line, so a dropped one + runs from its "track" line to the next blank line or the next stanza. Only a + "track" line at column zero starts a stanza, so the indented subtracks of a + composite that is not ours are never considered. + """ + if not trackNames: + return text + out = [] + dropping = False + for line in text.splitlines(keepends=True): + m = re.match(r"track\s+(\S+)\s*$", line) + if m: + # a new stanza starts here, whoever it belongs to + dropping = m.group(1) in trackNames + if dropping: + continue + elif dropping: + if line.strip() == "": + dropping = False # blank line closes the stanza; drop it too continue out.append(line) return "".join(out) -def wireHubTxt(hubTxt, name, block, remove, dryRun): +def wireHubTxt(hubTxt, name, block, remove, dryRun, trackNames=None): """Insert/replace (or remove) the marked contrib block in the assembly's - useOneFile hub.txt. Edits the real file the hub.txt symlink points at.""" + useOneFile hub file. Edits the real file the path points at.""" begin = "# BEGIN genark contrib: %s" % name end = "# END genark contrib: %s" % name blockRe = re.compile( r"\n*" + re.escape(begin) + r".*?" + re.escape(end) + r"\n?", re.DOTALL) realHub = os.path.realpath(hubTxt) text = open(realHub).read() - newText = blockRe.sub("\n", text).rstrip("\n") + "\n" + newText = blockRe.sub("\n", text) + if not remove: + # clear any unmarked copy the assembly build wrote, so ours is the only one + newText = dropTopLevelTracks(newText, trackNames) + newText = newText.rstrip("\n") + "\n" if not remove: newText += "\n%s\n%s\n%s\n" % (begin, block.rstrip("\n"), end) if dryRun: - print(" %s %s" % ("unwire hub.txt:" if remove else "wire hub.txt:", realHub)) + print(" %s %s" % ("unwire" if remove else "wire", realHub)) else: with open(realHub, "w") as fh: fh.write(newText) def addContrib(args): """Install a contrib track collection into the GenArk assembly hubs: symlink its data files + docs into <buildDir>/contrib/<name>/, write a per-assembly <name>.trackDb.txt with hub-root-relative paths, and wire that block into each assembly's served hub.txt. Everything is written into the assembly's GenArk build directory (see buildDir); the served /gbdb/genark and asmHubs/<acc> symlink trees are left alone. --remove undoes all of it.""" name = args.name.rstrip("/") root = os.path.join(CONTRIB, name) if not os.path.isdir(root): sys.exit("error: no such contrib collection: %s" % root) docsDir = os.path.join(root, "docs") docs = [] if os.path.isdir(docsDir): docs = sorted(f for f in os.listdir(docsDir) if f.endswith(".html")) + # Files at the collection root that all assemblies share, such as a scatterplot + # background panel named by a detailsScript dataUrl. Linked flat next to the + # docs, so contrib/<name>/<file> resolves for every assembly. + shared = sorted(f for f in os.listdir(root) + if f.endswith((".json", ".tsv")) + and os.path.isfile(os.path.join(root, f))) accs = sorted(d for d in os.listdir(root) if ACC_RE.match(d) and os.path.isdir(os.path.join(root, d))) if not accs: sys.exit("error: no accession directories (GCA_*/GCF_*) under %s" % root) + trackNames = contribTrackNames(root, accs) done = 0 skipped = 0 for acc in accs: accDir = os.path.join(root, acc) asmDir = buildDir(acc) if asmDir is None: sys.stderr.write("skip %s: no GenArk build directory under " "%s/{genbankBuild,refseqBuild}\n" % (acc, ASMHUBS)) skipped += 1 continue dest = os.path.join(asmDir, "contrib", name) - # the served single-file hub in the build dir (asmHubs/<acc>/hub.txt and - # /gbdb/genark/<acc>/hub.txt are symlinks to this); asmId is the build - # directory basename, which carries the assembly-name suffix. + # the alpha-tier single-file hub in the build dir (asmHubs/<acc>/alpha.hub.txt + # and /gbdb/genark/<acc>/alpha.hub.txt are symlinks to this); asmId is the + # build directory basename, which carries the assembly-name suffix. asmId = os.path.basename(asmDir) - hubTxt = os.path.join(asmDir, "%s.singleFile.hub.txt" % asmId) + # Only the alpha tier for now: <asmId>.singleFile.hub.txt is what + # /gbdb/genark/<acc>/hub.txt points at and is served as the assembly's + # default hub, so a contrib collection under test stays out of it. + hubTxt = os.path.join(asmDir, "alpha.hub.txt") if args.remove: if os.path.exists(hubTxt): wireHubTxt(hubTxt, name, "", remove=True, dryRun=args.dry_run) if args.dry_run: print(" rm -rf %s" % dest) elif os.path.isdir(dest): shutil.rmtree(dest) done += 1 continue if args.dry_run: print("# %s -> %s" % (acc, dest)) else: os.makedirs(dest, exist_ok=True) # symlink data files (.bb / .bw) from the collection's accession dir for f in sorted(os.listdir(accDir)): if f.endswith((".bb", ".bw")): symlink(os.path.join(accDir, f), os.path.join(dest, f), args.dry_run) # symlink shared doc pages (flat, so contrib/<name>/<doc> resolves) for d in docs: symlink(os.path.join(docsDir, d), os.path.join(dest, d), args.dry_run) + # and the collection's shared root-level data files + for f in shared: + symlink(os.path.join(root, f), os.path.join(dest, f), args.dry_run) # per-assembly trackDb with hub-root-relative paths, and wire it into hub.txt srcTdb = os.path.join(accDir, "trackDb.txt") if os.path.isfile(srcTdb): tdb = rewriteTrackDb(srcTdb, name) destTdb = os.path.join(dest, "%s.trackDb.txt" % name) if args.dry_run: print(" write %s (%d bytes)" % (destTdb, len(tdb))) else: with open(destTdb, "w") as fh: fh.write(tdb) if os.path.exists(hubTxt): - wireHubTxt(hubTxt, name, tdb, remove=False, dryRun=args.dry_run) + wireHubTxt(hubTxt, name, tdb, remove=False, dryRun=args.dry_run, + trackNames=trackNames) else: sys.stderr.write("warn %s: no hub.txt to wire at %s\n" % (acc, hubTxt)) done += 1 verb = "removed from" if args.remove else "installed into" print("addContrib %s: %s %d assemblies, skipped %d (no assembly hub)" % (name, verb, done, skipped)) def contribTrackNames(root, accs): """Track names defined by the collection (from any one accession trackDb).""" for acc in accs: tdb = os.path.join(root, acc, "trackDb.txt") if os.path.isfile(tdb): return set(re.findall(r"^track\s+(\S+)", open(tdb).read(), re.MULTILINE))