5c35098ff0bc6319b3c771ee8d683d2145fd75ac max Tue Aug 11 08:02:17 2026 -0700 genark: write contrib data into the GenArk build directory, not the served symlink trees addContrib was creating contrib// symlink dirs and trackDb under assemblyDir(acc) = asmHubs/GCA/.../, which (like /gbdb/genark/) is only a tree of symlinks the GenArk build system regenerates, so the contrib files got clobbered. Replace assemblyDir() with buildDir(), which resolves the real build directory under asmHubs/{genbankBuild,refseqBuild}/... (GCA -> genbankBuild, GCF -> refseqBuild), globbing the accession to pick up the assembly-name suffix. contrib symlinks, the per-assembly trackDb, and the hub.txt wiring (now the build dir's .singleFile.hub.txt) all land in the build directory. refs #35415 diff --git src/utils/genark/genark src/utils/genark/genark index 90050663da4..ecf45acd47f 100755 --- src/utils/genark/genark +++ src/utils/genark/genark @@ -1,301 +1,331 @@ #!/usr/bin/env python3 """genark - utilities to manage UCSC GenArk assembly hubs. This is a general, subcommand-based tool. Subcommands will be added over time. Subcommands: addContrib Install a contributed track collection into the GenArk assembly hubs. is a subdirectory of /hive/data/genomes/asmHubs/contrib/ that is laid out as one directory per assembly accession (GCA_*/GCF_*) plus a shared docs/ directory, e.g.: contrib//GCA_000000000.0/trackDb.txt contrib//GCA_000000000.0/*.bb (and/or *.bw) contrib//docs/*.html - For each accession it: - - creates /contrib// with symlinks to + For each accession it writes into that assembly's GenArk + *build directory* (asmHubs/{genbankBuild,refseqBuild}/...) + -- never the served /gbdb/genark or asmHubs/ symlink + trees, which the build system regenerates. There it: + - creates /contrib// with symlinks to the collection's data files (.bb/.bw) and doc pages; - writes a per-assembly .trackDb.txt whose bigDataUrl/html paths are rewritten to be hub-root relative (contrib//...); - wires that trackDb block into the assembly's useOneFile hub.txt, between BEGIN/END markers (idempotent). --remove uninstalls: strips the hub.txt block and removes the contrib// symlink dir. NOTE: a full GenArk hub rebuild regenerates hub.txt, so re-run addContrib afterwards (or add to the build's asmHubTrackDb.sh for a durable inclusion). checkContrib [accessions...] Run hubCheck on assembly hubs that carry the collection (a random --sample N by default, or --all, or the listed accessions) and report problems, separating contrib-specific issues from the assemblies' own pre-existing hub warnings. --noTracks does a faster structure-only check. """ import argparse +import glob import os import random import re import shutil import subprocess import sys ASMHUBS = "/hive/data/genomes/asmHubs" CONTRIB = os.path.join(ASMHUBS, "contrib") ACC_RE = re.compile(r"^GC[AF]_[0-9]{9}\.[0-9]+$") -def assemblyDir(acc): - """GCA_041900255.1 -> /hive/data/genomes/asmHubs/GCA/041/900/255/GCA_041900255.1""" - prefix = acc[0:3] # GCA or GCF - digits = acc[4:] # 041900255.1 - return os.path.join(ASMHUBS, prefix, digits[0:3], digits[3:6], digits[6:9], acc) +def accPath(acc): + """3-3-3 hashed subpath for an accession, e.g. + GCA_041900255.1 -> GCA/041/900/255/GCA_041900255.1""" + d = acc[4:] # 041900255.1 + return os.path.join(acc[0:3], d[0:3], d[3:6], d[6:9], acc) + + +def buildDir(acc): + """Resolve the GenArk *build* directory for an accession, or None. + + GCA_018506965.2 -> + /hive/data/genomes/asmHubs/genbankBuild/GCA/018/506/965/GCA_018506965.2_HG005_mat_hprc_f2 + + This is the directory the GenArk build system owns and builds from. The + served copies -- /gbdb/genark// and asmHubs// -- are nothing but + symlinks pointing back here, and they are regenerated by the build system, + so contrib data must NEVER be written into them; it goes here in the build + directory. GCF_* assemblies build under refseqBuild, GCA_* under + genbankBuild, and the on-disk directory name carries an assembly-name suffix + (…_HG005_mat_hprc_f2) beyond the bare accession, so glob to find it.""" + subtree = "refseqBuild" if acc.startswith("GCF_") else "genbankBuild" + stem = os.path.join(ASMHUBS, subtree, accPath(acc)) + for m in sorted(glob.glob(stem + "_*")) + [stem]: + if os.path.isdir(m): + return m + return None HUBS_URL = "https://hgdownload.soe.ucsc.edu/hubs" def hubUrl(acc): """Public served hub.txt URL for a GenArk assembly accession.""" - d = acc[4:] - return "%s/%s/%s/%s/%s/%s/hub.txt" % ( - HUBS_URL, acc[0:3], d[0:3], d[3:6], d[6:9], acc) + return "%s/%s/hub.txt" % (HUBS_URL, accPath(acc)) def symlink(target, linkPath, dryRun): """Create/replace an absolute symlink linkPath -> target.""" if dryRun: print(" ln -sf %s %s" % (target, linkPath)) return if os.path.islink(linkPath) or os.path.exists(linkPath): os.remove(linkPath) os.symlink(target, linkPath) def rewriteTrackDb(srcPath, name): """Return the trackDb text with local bigDataUrl/html paths made hub-root relative (contrib//...). Remote (http/https/ftp) bigDataUrls and already-prefixed paths are left untouched.""" out = [] prefix = "contrib/%s/" % name for line in open(srcPath): stripped = line.lstrip() indent = line[:len(line) - len(stripped)] m = re.match(r"(bigDataUrl|linkDataUrl)\s+(\S+)\s*$", stripped) if m: key, val = m.group(1), m.group(2) if not re.match(r"[a-z]+://", val) and not val.startswith(prefix): val = prefix + os.path.basename(val) out.append("%s%s %s\n" % (indent, key, val)) continue m = re.match(r"html\s+(\S+)\s*$", stripped) if m: val = m.group(1) if not val.startswith(prefix): val = prefix + os.path.basename(val) out.append("%shtml %s\n" % (indent, val)) continue out.append(line) return "".join(out) def wireHubTxt(hubTxt, name, block, remove, dryRun): """Insert/replace (or remove) the marked contrib block in the assembly's useOneFile hub.txt. Edits the real file the hub.txt symlink points at.""" begin = "# BEGIN genark contrib: %s" % name end = "# END genark contrib: %s" % name blockRe = re.compile( r"\n*" + re.escape(begin) + r".*?" + re.escape(end) + r"\n?", re.DOTALL) realHub = os.path.realpath(hubTxt) text = open(realHub).read() newText = blockRe.sub("\n", text).rstrip("\n") + "\n" if not remove: newText += "\n%s\n%s\n%s\n" % (begin, block.rstrip("\n"), end) if dryRun: print(" %s %s" % ("unwire hub.txt:" if remove else "wire hub.txt:", realHub)) else: with open(realHub, "w") as fh: fh.write(newText) def addContrib(args): """Install a contrib track collection into the GenArk assembly hubs: - symlink its data files + docs into /contrib//, write a + symlink its data files + docs into /contrib//, write a per-assembly .trackDb.txt with hub-root-relative paths, and wire that - block into each assembly's hub.txt. --remove undoes all of it.""" + block into each assembly's served hub.txt. Everything is written into the + assembly's GenArk build directory (see buildDir); the served /gbdb/genark + and asmHubs/ symlink trees are left alone. --remove undoes all of it.""" name = args.name.rstrip("/") root = os.path.join(CONTRIB, name) if not os.path.isdir(root): sys.exit("error: no such contrib collection: %s" % root) docsDir = os.path.join(root, "docs") docs = [] if os.path.isdir(docsDir): docs = sorted(f for f in os.listdir(docsDir) if f.endswith(".html")) accs = sorted(d for d in os.listdir(root) if ACC_RE.match(d) and os.path.isdir(os.path.join(root, d))) if not accs: sys.exit("error: no accession directories (GCA_*/GCF_*) under %s" % root) done = 0 skipped = 0 for acc in accs: accDir = os.path.join(root, acc) - asmDir = assemblyDir(acc) - if not os.path.isdir(asmDir): - sys.stderr.write("skip %s: no GenArk assembly hub at %s\n" % (acc, asmDir)) + asmDir = buildDir(acc) + if asmDir is None: + sys.stderr.write("skip %s: no GenArk build directory under " + "%s/{genbankBuild,refseqBuild}\n" % (acc, ASMHUBS)) skipped += 1 continue dest = os.path.join(asmDir, "contrib", name) - hubTxt = os.path.join(asmDir, "hub.txt") + # the served single-file hub in the build dir (asmHubs//hub.txt and + # /gbdb/genark//hub.txt are symlinks to this); asmId is the build + # directory basename, which carries the assembly-name suffix. + asmId = os.path.basename(asmDir) + hubTxt = os.path.join(asmDir, "%s.singleFile.hub.txt" % asmId) if args.remove: if os.path.exists(hubTxt): wireHubTxt(hubTxt, name, "", remove=True, dryRun=args.dry_run) if args.dry_run: print(" rm -rf %s" % dest) elif os.path.isdir(dest): shutil.rmtree(dest) done += 1 continue if args.dry_run: print("# %s -> %s" % (acc, dest)) else: os.makedirs(dest, exist_ok=True) # symlink data files (.bb / .bw) from the collection's accession dir for f in sorted(os.listdir(accDir)): if f.endswith((".bb", ".bw")): symlink(os.path.join(accDir, f), os.path.join(dest, f), args.dry_run) # symlink shared doc pages (flat, so contrib// resolves) for d in docs: symlink(os.path.join(docsDir, d), os.path.join(dest, d), args.dry_run) # per-assembly trackDb with hub-root-relative paths, and wire it into hub.txt srcTdb = os.path.join(accDir, "trackDb.txt") if os.path.isfile(srcTdb): tdb = rewriteTrackDb(srcTdb, name) destTdb = os.path.join(dest, "%s.trackDb.txt" % name) if args.dry_run: print(" write %s (%d bytes)" % (destTdb, len(tdb))) else: with open(destTdb, "w") as fh: fh.write(tdb) if os.path.exists(hubTxt): wireHubTxt(hubTxt, name, tdb, remove=False, dryRun=args.dry_run) else: sys.stderr.write("warn %s: no hub.txt to wire at %s\n" % (acc, hubTxt)) done += 1 verb = "removed from" if args.remove else "installed into" print("addContrib %s: %s %d assemblies, skipped %d (no assembly hub)" % (name, verb, done, skipped)) def contribTrackNames(root, accs): """Track names defined by the collection (from any one accession trackDb).""" for acc in accs: tdb = os.path.join(root, acc, "trackDb.txt") if os.path.isfile(tdb): return set(re.findall(r"^track\s+(\S+)", open(tdb).read(), re.MULTILINE)) return set() def checkContrib(args): """Run hubCheck on a set of assembly hubs that carry the collection, and classify any reported problems as contrib-specific vs pre-existing hub warnings (so the collection can be signed off without wading through the assemblies' own tracks).""" name = args.name.rstrip("/") root = os.path.join(CONTRIB, name) if not os.path.isdir(root): sys.exit("error: no such contrib collection: %s" % root) accs = sorted(d for d in os.listdir(root) if ACC_RE.match(d) and os.path.isdir(os.path.join(root, d))) tracks = contribTrackNames(root, accs) if args.accession: sel = args.accession elif args.all: sel = accs else: n = min(args.sample, len(accs)) sel = sorted(random.sample(accs, n)) cmd = ["hubCheck"] if args.noTracks: cmd.append("-noTracks") clean = other = flagged = failed = 0 for acc in sel: try: res = subprocess.run(cmd + [hubUrl(acc)], capture_output=True, text=True, timeout=args.timeout) out = res.stdout + res.stderr except subprocess.TimeoutExpired: print("%s: TIMEOUT" % acc); failed += 1; continue problems = [l for l in out.splitlines() if l.strip() and not l.startswith("Found ")] contribProblems = [l for l in problems if any(t in l for t in tracks) or "rror" in l] if not problems: print("%s: clean" % acc); clean += 1 elif contribProblems: print("%s: CONTRIB PROBLEMS (%d)" % (acc, len(contribProblems))) for l in contribProblems: print(" ! %s" % l.strip()) flagged += 1 else: print("%s: ok (%d pre-existing hub warning(s), no contrib issue)" % (acc, len(problems))) other += 1 print("checkContrib %s: %d checked -- %d clean, %d ok-with-hub-warnings, " "%d CONTRIB PROBLEMS, %d failed" % (name, len(sel), clean, other, flagged, failed)) if flagged or failed: sys.exit(1) def main(): ap = argparse.ArgumentParser( prog="genark", description="Manage UCSC GenArk assembly hubs.") ap.add_argument("--dry-run", action="store_true", help="show what would be done without changing anything") sub = ap.add_subparsers(dest="cmd", required=True) p = sub.add_parser("addContrib", help="install a contrib track collection into the assembly hubs") p.add_argument("name", help="contrib subdirectory name under %s" % CONTRIB) p.add_argument("--remove", action="store_true", help="uninstall: remove the symlinks and the hub.txt block") p.set_defaults(func=addContrib) c = sub.add_parser("checkContrib", help="run hubCheck on assembly hubs carrying the collection") c.add_argument("name", help="contrib subdirectory name under %s" % CONTRIB) c.add_argument("accession", nargs="*", help="specific accessions to check (default: a random sample)") c.add_argument("--sample", type=int, default=5, help="number of random assemblies to check (default 5)") c.add_argument("--all", action="store_true", help="check every assembly") c.add_argument("--noTracks", action="store_true", help="structure only, do not fetch track data (faster)") c.add_argument("--timeout", type=int, default=300, help="per-assembly hubCheck timeout in seconds (default 300)") c.set_defaults(func=checkContrib) args = ap.parse_args() args.func(args) if __name__ == "__main__": main()