5c35098ff0bc6319b3c771ee8d683d2145fd75ac
max
Tue Aug 11 08:02:17 2026 -0700
genark: write contrib data into the GenArk build directory, not the served symlink trees
addContrib was creating contrib/<name>/ symlink dirs and trackDb under
assemblyDir(acc) = asmHubs/GCA/.../<acc>, which (like /gbdb/genark/<acc>) is
only a tree of symlinks the GenArk build system regenerates, so the contrib
files got clobbered. Replace assemblyDir() with buildDir(), which resolves the
real build directory under asmHubs/{genbankBuild,refseqBuild}/... (GCA ->
genbankBuild, GCF -> refseqBuild), globbing the accession to pick up the
assembly-name suffix. contrib symlinks, the per-assembly trackDb, and the
hub.txt wiring (now the build dir's <asmId>.singleFile.hub.txt) all land in the
build directory. refs #35415
diff --git src/utils/genark/genark src/utils/genark/genark
index 90050663da4..ecf45acd47f 100755
--- src/utils/genark/genark
+++ src/utils/genark/genark
@@ -2,83 +2,106 @@
"""genark - utilities to manage UCSC GenArk assembly hubs.
This is a general, subcommand-based tool. Subcommands will be added over time.
Subcommands:
addContrib <name> Install a contributed track collection into the GenArk
assembly hubs. <name> is a subdirectory of
/hive/data/genomes/asmHubs/contrib/ that is laid out as
one directory per assembly accession (GCA_*/GCF_*) plus a
shared docs/ directory, e.g.:
contrib/<name>/GCA_000000000.0/trackDb.txt
contrib/<name>/GCA_000000000.0/*.bb (and/or *.bw)
contrib/<name>/docs/*.html
- For each accession it:
- - creates <assemblyHub>/contrib/<name>/ with symlinks to
+ For each accession it writes into that assembly's GenArk
+ *build directory* (asmHubs/{genbankBuild,refseqBuild}/...)
+ -- never the served /gbdb/genark or asmHubs/<acc> symlink
+ trees, which the build system regenerates. There it:
+ - creates <buildDir>/contrib/<name>/ with symlinks to
the collection's data files (.bb/.bw) and doc pages;
- writes a per-assembly <name>.trackDb.txt whose
bigDataUrl/html paths are rewritten to be hub-root
relative (contrib/<name>/...);
- wires that trackDb block into the assembly's useOneFile
hub.txt, between BEGIN/END markers (idempotent).
--remove uninstalls: strips the hub.txt block and removes
the contrib/<name>/ symlink dir.
NOTE: a full GenArk hub rebuild regenerates hub.txt, so
re-run addContrib afterwards (or add <name> to the build's
asmHubTrackDb.sh for a durable inclusion).
checkContrib <name> [accessions...]
Run hubCheck on assembly hubs that carry the collection
(a random --sample N by default, or --all, or the listed
accessions) and report problems, separating contrib-specific
issues from the assemblies' own pre-existing hub warnings.
--noTracks does a faster structure-only check.
"""
import argparse
+import glob
import os
import random
import re
import shutil
import subprocess
import sys
ASMHUBS = "/hive/data/genomes/asmHubs"
CONTRIB = os.path.join(ASMHUBS, "contrib")
ACC_RE = re.compile(r"^GC[AF]_[0-9]{9}\.[0-9]+$")
-def assemblyDir(acc):
- """GCA_041900255.1 -> /hive/data/genomes/asmHubs/GCA/041/900/255/GCA_041900255.1"""
- prefix = acc[0:3] # GCA or GCF
- digits = acc[4:] # 041900255.1
- return os.path.join(ASMHUBS, prefix, digits[0:3], digits[3:6], digits[6:9], acc)
+def accPath(acc):
+ """3-3-3 hashed subpath for an accession, e.g.
+ GCA_041900255.1 -> GCA/041/900/255/GCA_041900255.1"""
+ d = acc[4:] # 041900255.1
+ return os.path.join(acc[0:3], d[0:3], d[3:6], d[6:9], acc)
+
+
+def buildDir(acc):
+ """Resolve the GenArk *build* directory for an accession, or None.
+
+ GCA_018506965.2 ->
+ /hive/data/genomes/asmHubs/genbankBuild/GCA/018/506/965/GCA_018506965.2_HG005_mat_hprc_f2
+
+ This is the directory the GenArk build system owns and builds from. The
+ served copies -- /gbdb/genark/<acc>/ and asmHubs/<acc>/ -- are nothing but
+ symlinks pointing back here, and they are regenerated by the build system,
+ so contrib data must NEVER be written into them; it goes here in the build
+ directory. GCF_* assemblies build under refseqBuild, GCA_* under
+ genbankBuild, and the on-disk directory name carries an assembly-name suffix
+ (…_HG005_mat_hprc_f2) beyond the bare accession, so glob to find it."""
+ subtree = "refseqBuild" if acc.startswith("GCF_") else "genbankBuild"
+ stem = os.path.join(ASMHUBS, subtree, accPath(acc))
+ for m in sorted(glob.glob(stem + "_*")) + [stem]:
+ if os.path.isdir(m):
+ return m
+ return None
HUBS_URL = "https://hgdownload.soe.ucsc.edu/hubs"
def hubUrl(acc):
"""Public served hub.txt URL for a GenArk assembly accession."""
- d = acc[4:]
- return "%s/%s/%s/%s/%s/%s/hub.txt" % (
- HUBS_URL, acc[0:3], d[0:3], d[3:6], d[6:9], acc)
+ return "%s/%s/hub.txt" % (HUBS_URL, accPath(acc))
def symlink(target, linkPath, dryRun):
"""Create/replace an absolute symlink linkPath -> target."""
if dryRun:
print(" ln -sf %s %s" % (target, linkPath))
return
if os.path.islink(linkPath) or os.path.exists(linkPath):
os.remove(linkPath)
os.symlink(target, linkPath)
def rewriteTrackDb(srcPath, name):
"""Return the trackDb text with local bigDataUrl/html paths made hub-root
relative (contrib/<name>/...). Remote (http/https/ftp) bigDataUrls and
@@ -115,58 +138,65 @@
r"\n*" + re.escape(begin) + r".*?" + re.escape(end) + r"\n?", re.DOTALL)
realHub = os.path.realpath(hubTxt)
text = open(realHub).read()
newText = blockRe.sub("\n", text).rstrip("\n") + "\n"
if not remove:
newText += "\n%s\n%s\n%s\n" % (begin, block.rstrip("\n"), end)
if dryRun:
print(" %s %s" % ("unwire hub.txt:" if remove else "wire hub.txt:", realHub))
else:
with open(realHub, "w") as fh:
fh.write(newText)
def addContrib(args):
"""Install a contrib track collection into the GenArk assembly hubs:
- symlink its data files + docs into <assemblyHub>/contrib/<name>/, write a
+ symlink its data files + docs into <buildDir>/contrib/<name>/, write a
per-assembly <name>.trackDb.txt with hub-root-relative paths, and wire that
- block into each assembly's hub.txt. --remove undoes all of it."""
+ block into each assembly's served hub.txt. Everything is written into the
+ assembly's GenArk build directory (see buildDir); the served /gbdb/genark
+ and asmHubs/<acc> symlink trees are left alone. --remove undoes all of it."""
name = args.name.rstrip("/")
root = os.path.join(CONTRIB, name)
if not os.path.isdir(root):
sys.exit("error: no such contrib collection: %s" % root)
docsDir = os.path.join(root, "docs")
docs = []
if os.path.isdir(docsDir):
docs = sorted(f for f in os.listdir(docsDir) if f.endswith(".html"))
accs = sorted(d for d in os.listdir(root) if ACC_RE.match(d)
and os.path.isdir(os.path.join(root, d)))
if not accs:
sys.exit("error: no accession directories (GCA_*/GCF_*) under %s" % root)
done = 0
skipped = 0
for acc in accs:
accDir = os.path.join(root, acc)
- asmDir = assemblyDir(acc)
- if not os.path.isdir(asmDir):
- sys.stderr.write("skip %s: no GenArk assembly hub at %s\n" % (acc, asmDir))
+ asmDir = buildDir(acc)
+ if asmDir is None:
+ sys.stderr.write("skip %s: no GenArk build directory under "
+ "%s/{genbankBuild,refseqBuild}\n" % (acc, ASMHUBS))
skipped += 1
continue
dest = os.path.join(asmDir, "contrib", name)
- hubTxt = os.path.join(asmDir, "hub.txt")
+ # the served single-file hub in the build dir (asmHubs/<acc>/hub.txt and
+ # /gbdb/genark/<acc>/hub.txt are symlinks to this); asmId is the build
+ # directory basename, which carries the assembly-name suffix.
+ asmId = os.path.basename(asmDir)
+ hubTxt = os.path.join(asmDir, "%s.singleFile.hub.txt" % asmId)
if args.remove:
if os.path.exists(hubTxt):
wireHubTxt(hubTxt, name, "", remove=True, dryRun=args.dry_run)
if args.dry_run:
print(" rm -rf %s" % dest)
elif os.path.isdir(dest):
shutil.rmtree(dest)
done += 1
continue
if args.dry_run:
print("# %s -> %s" % (acc, dest))
else:
os.makedirs(dest, exist_ok=True)