a8694a3b22d43f0536c02101e9f3b56b5339b4dc max Wed Sep 9 05:47:17 2026 -0700 hubtools: add "import igv" and "splitHap", and the bTaeGut7 zebra finch hub import igv builds a hub from an IGV session XML. Every Track element becomes a track, in session order, with the IGV display attributes translated to trackDb settings. Files the browser can read over the network are linked where they are; bed, gff, gtf, wig and bedGraph are downloaded and converted, which needs chrom.sizes and gets them from --chromSizes, from the UCSC assembly, or from a bigWig of the session itself, the only source there is for a custom assembly. The BED cleaner exists because real files are not to spec: reversed start/end, scores over 1000, "#rrggbb" colours, names past 255 characters, and columns that are not the BED field they sit in, such as trf writing the repeat motif where thickStart belongs. splitHap turns a hub built on a diploid assembly into one hub with a genome per haplotype, reading both assemblies' chrom.sizes and chromAlias from GenArk and sending each record to whichever assembly has its sequence. It writes splitHap.report.txt with the records per track per haplotype, the sequences neither assembly has, and the records reaching past a sequence end, and checks every track as it goes: records read must equal records matched plus records with no sequence, and every match must produce an output record or a drop. A track that does not add up stops the run rather than being written up as a finding. Two conversion fixes that came out of the zebra finch data. GFF3 requires unique IDs, but an annotation of a phased assembly often gives both haplotypes the same ID; gff3ToGenePred then merges the two copies into one transcript spanning two chromosomes and discards it, which was losing 31 of 182 retrocopies. IDs that occur on more than one sequence are now made unique per sequence first. And a feature name is now taken from the first non-numeric attribute, so a RepeatMasker GFF gives Motif:Tgut716A rather than the running number in ID=. genark addContrib gains --tier alpha|beta|public. It edits only betaGenArk.txt and publicGenArk.txt; beta.hub.txt and public.hub.txt are generated from those lists and shipped by quickPush.pl, so writing them by hand would push content outside the normal flow and lose it at the next clade build. The default alpha tier leaves the lists untouched, so re-running an install cannot demote a collection that is already promoted. doc/contrib/bTaeGut7 and trackDb/contrib/bTaeGut7 are the zebra finch telomere-to-telomere hub built with the above, from the IGV session the authors ship with the annotations on GenomeArk (Formenti et al, Cell 2026, PMID 42561917). 21 tracks in 6 collections plus 3 standalone, 27 description pages, and a makeDoc recording where every record went. diff --git src/utils/genark/genark src/utils/genark/genark index 3a03ab50590..2f962cbb9d0 100755 --- src/utils/genark/genark +++ src/utils/genark/genark @@ -55,30 +55,49 @@ import concurrent.futures import datetime import glob import os import random import re import shutil import subprocess import sys import tempfile import time ASMHUBS = "/hive/data/genomes/asmHubs" CONTRIB = os.path.join(ASMHUBS, "contrib") +# Release control for contrib collections. mkGenomes.pl reads these two lists when it +# composes the per-assembly hub.txt tiers: a collection named in betaGenArk.txt goes into +# beta.hub.txt, one named in publicGenArk.txt goes into public.hub.txt and the hub.txt +# that hgwdev and hgdownload serve. alpha.hub.txt ignores both and picks up whatever is +# on disk under /contrib/, which is why installing a collection makes it visible +# in alpha with no list edit at all. +TRACKDB_SRC = os.path.expanduser("~/kent/src/hg/makeDb/trackDb") +TIER_LISTS = { + "beta": os.path.join(TRACKDB_SRC, "betaGenArk.txt"), + "public": os.path.join(TRACKDB_SRC, "publicGenArk.txt"), +} +# public is cumulative: every collection in publicGenArk.txt is also in betaGenArk.txt, +# so promoting to public keeps the beta entry rather than moving it. +TIER_MEMBERSHIP = { + "alpha": [], + "beta": ["beta"], + "public": ["beta", "public"], +} + ACC_RE = re.compile(r"^GC[AF]_[0-9]{9}\.[0-9]+$") # --- syncFtp: local NCBI genomes mirror -------------------------------------- GENOMES = "/hive/data/outside/ncbi/genomes" FTP_HOST = "https://ftp.ncbi.nlm.nih.gov" # assembly dirs live under genomes/all//... on the server; the local # mirror drops the "all" component, so genomes/all/GCA/000/001/405/GCA_... maps # to /GCA/000/001/405/GCA_... ALL_MARKER = "/genomes/all/" # per-accession-type authority file: the current "latest" assemblies SUMMARY_PATH = { "GCA": "/genomes/genbank/assembly_summary_genbank.txt", "GCF": "/genomes/refseq/assembly_summary_refseq.txt", @@ -238,30 +257,80 @@ text = open(realHub).read() newText = blockRe.sub("\n", text) if not remove: # clear any unmarked copy the assembly build wrote, so ours is the only one newText = dropTopLevelTracks(newText, trackNames) newText = newText.rstrip("\n") + "\n" if not remove: newText += "\n%s\n%s\n%s\n" % (begin, block.rstrip("\n"), end) if dryRun: print(" %s %s" % ("unwire" if remove else "wire", realHub)) else: with open(realHub, "w") as fh: fh.write(newText) +def listMembers(path): + """Collection names in one of the GenArk release lists, comments and blanks out.""" + if not os.path.isfile(path): + return [] + return [l.strip() for l in open(path) + if l.strip() and not l.lstrip().startswith("#")] + + +def setTierMembership(name, tier, remove, dryRun): + """Add or remove a collection in the release lists that match the tier, and say what + changed. Idempotent: a name already listed is left alone. + + Only the lists are touched. beta.hub.txt and public.hub.txt are generated by + mkGenomes.pl from these lists and shipped by the otto quickPush.pl, so writing into + them by hand here would put content into pushed files outside the normal flow, and + the next clade build would drop it again.""" + # alpha is not a list at all, it is what you get from being on disk, so installing at + # the default tier must leave the lists exactly as they are. Only --remove takes a + # collection out of them; a tier only ever adds. Otherwise re-running addContrib on an + # already promoted collection would quietly demote it. + if tier == "alpha" and not remove: + return [] + wanted = TIER_MEMBERSHIP[tier] + changed = [] + for key in ("beta", "public"): + path = TIER_LISTS[key] + members = listMembers(path) + present = name in members + want = present or ((not remove) and (key in wanted)) + if remove: + want = False + if want == present: + continue + if dryRun: + print(" %s %s in %s" % ("add" if want else "remove", name, + os.path.basename(path))) + changed.append(key) + continue + text = open(path).read() if os.path.isfile(path) else "" + if want: + text = text.rstrip("\n") + "\n" + name + "\n" + else: + keep = [l for l in text.split("\n") if l.strip() != name] + text = "\n".join(keep).rstrip("\n") + "\n" + with open(path, "w") as fh: + fh.write(text) + changed.append(key) + return changed + + def addContrib(args): """Install a contrib track collection into the GenArk assembly hubs: symlink its data files + docs into /contrib//, write a per-assembly .trackDb.txt with hub-root-relative paths, and wire that block into each assembly's served hub.txt. Everything is written into the assembly's GenArk build directory (see buildDir); the served /gbdb/genark and asmHubs/ symlink trees are left alone. --remove undoes all of it.""" name = args.name.rstrip("/") root = os.path.join(CONTRIB, name) if not os.path.isdir(root): sys.exit("error: no such contrib collection: %s" % root) docsDir = os.path.join(root, "docs") docs = [] if os.path.isdir(docsDir): docs = sorted(f for f in os.listdir(docsDir) if f.endswith(".html")) @@ -333,30 +402,49 @@ print(" write %s (%d bytes)" % (destTdb, len(tdb))) else: with open(destTdb, "w") as fh: fh.write(tdb) if os.path.exists(hubTxt): wireHubTxt(hubTxt, name, tdb, remove=False, dryRun=args.dry_run, trackNames=trackNames) else: sys.stderr.write("warn %s: no hub.txt to wire at %s\n" % (acc, hubTxt)) done += 1 verb = "removed from" if args.remove else "installed into" print("addContrib %s: %s %d assemblies, skipped %d (no assembly hub)" % (name, verb, done, skipped)) + tier = getattr(args, "tier", "alpha") + changed = setTierMembership(name, tier, args.remove, args.dry_run) + if args.remove: + if changed: + print(" also removed from: %s" + % ", ".join(os.path.basename(TIER_LISTS[k]) for k in changed)) + elif tier == "alpha": + print(" tier alpha: visible now at " + "https://genome-test.gi.ucsc.edu/h/, and nowhere else. " + "The pushes exclude every hub.txt tier file, so this cannot leak.") + else: + if changed: + print(" tier %s: added to %s" % (tier, + ", ".join(os.path.basename(TIER_LISTS[k]) for k in changed))) + else: + print(" tier %s: already listed, nothing to change" % tier) + print(" commit the list change, then the tier hub files are rewritten by the " + "next mkGenomes for the clade and shipped by the otto quickPush.pl.") + def contribTrackNames(root, accs): """Track names defined by the collection (from any one accession trackDb).""" for acc in accs: tdb = os.path.join(root, acc, "trackDb.txt") if os.path.isfile(tdb): return set(re.findall(r"^track\s+(\S+)", open(tdb).read(), re.MULTILINE)) return set() def checkContrib(args): """Run hubCheck on a set of assembly hubs that carry the collection, and classify any reported problems as contrib-specific vs pre-existing hub warnings (so the collection can be signed off without wading through the assemblies' own tracks).""" @@ -667,31 +755,38 @@ if totFail: sys.exit(1) def main(): ap = argparse.ArgumentParser( prog="genark", description="Manage UCSC GenArk assembly hubs.") ap.add_argument("--dry-run", action="store_true", help="show what would be done without changing anything") sub = ap.add_subparsers(dest="cmd", required=True) p = sub.add_parser("addContrib", help="install a contrib track collection into the assembly hubs") p.add_argument("name", help="contrib subdirectory name under %s" % CONTRIB) p.add_argument("--remove", action="store_true", - help="uninstall: remove the symlinks and the hub.txt block") + help="uninstall: remove the symlinks, the hub.txt block, and the " + "collection's entry in both release lists") + p.add_argument("--tier", choices=["alpha", "beta", "public"], default="alpha", + help="how far to release the collection. alpha (default) installs it " + "and wires alpha.hub.txt, which is hgwdev only. beta adds it to " + "betaGenArk.txt, public adds it to publicGenArk.txt as well; " + "those two lists are what mkGenomes.pl reads when it builds the " + "beta and public hub.txt tiers.") p.set_defaults(func=addContrib) c = sub.add_parser("checkContrib", help="run hubCheck on assembly hubs carrying the collection") c.add_argument("name", help="contrib subdirectory name under %s" % CONTRIB) c.add_argument("accession", nargs="*", help="specific accessions to check (default: a random sample)") c.add_argument("--sample", type=int, default=5, help="number of random assemblies to check (default 5)") c.add_argument("--all", action="store_true", help="check every assembly") c.add_argument("--noTracks", action="store_true", help="structure only, do not fetch track data (faster)") c.add_argument("--timeout", type=int, default=300, help="per-assembly hubCheck timeout in seconds (default 300)") c.set_defaults(func=checkContrib)