edf45d7b82651acd7bb9cf00cc492e9b19936f1f max Fri Sep 11 09:58:18 2026 -0700 uniprot otto: write a GenArk contrib collection instead of /gbdb symlinks A GenArk assembly is not served out of /gbdb/<db>/, so the symlinks and the trackDb .ra stanza that classic assemblies get do not apply to it. Its files belong in a contrib collection. For a GenArk assembly the pipeline now writes contrib/uniprot/<accession>/ with the bigBeds and a trackDb.txt, and skips three things that only make sense for a classic assembly: the /gbdb/<db>/uniprot symlinks, the version.txt symlink, and the goldenPath archive copy, whose download path does not exist for a hub assembly. The trackDb comes from the same template the archive hub uses, with three changes: track names must not carry the release or they would change every month and break saved sessions; the data files sit beside trackDb.txt rather than in a per-release subdirectory; and dataVersion has to be the literal release string because a contrib trackDb cannot read a /gbdb path. It deliberately stops there. Nothing is symlinked into the GenArk build directories and no hub.txt is touched: installing the collection is a separate, deliberate step with "genark addContrib uniprot", not something a monthly data update should do on its own. Checked by writing a collection for one assembly and running the genark tool's --dry-run over it: all 16 bigDataUrls resolve to files that are present and non-empty, the html path resolves to the shared docs page, and the tool reports it would wire the block into alpha.hub.txt only. The shared docs page is the trackDb description with the archive-hub paragraph and the /gbdb download links removed, since neither exists for a GenArk assembly. refs #38300 diff --git src/hg/utils/otto/uniprot/doUniprot src/hg/utils/otto/uniprot/doUniprot index 7f855ed2169..66d6f243c5b 100755 --- src/hg/utils/otto/uniprot/doUniprot +++ src/hg/utils/otto/uniprot/doUniprot @@ -2321,30 +2321,37 @@ if len(fnames)==0: logging.error("NOT FOUND: %s" % fileMask) return None assert(len(fnames)==1) fname = fnames[0] return fname def makeLinks(bigBedDir, onlyDbs, taxIdDbs): " check the /gbdb symlinks " for taxId, dbs in taxIdDbs.items(): for db in dbs: if onlyDbs is not None and db not in onlyDbs: continue + if genArkHubDir(db) is not None: + # a GenArk assembly is served from its hub, not from /gbdb/<db>/, and + # /gbdb/genark belongs to the GenArk build. Its files go into the contrib + # collection instead, see installGenArkContrib. + logging.debug("%s is a GenArk assembly, no /gbdb symlinks" % db) + continue + dbBigBedDir = join(bigBedDir, db) # find the bigBed files bbTargetDir = makeSubDir(join("/gbdb", db), "uniprot") bbFnames = glob.glob(join(dbBigBedDir, "*.bb")) logging.debug("Found %d bigBed files in %s" % (len(bbFnames), dbBigBedDir)) # and create links to them for bbFname in bbFnames: # the following 'if' should not be necessary, but it has happened to me that during testing I had # the liftOver files or .new.bb files in bigBed/, so make sure that they never get linked, because our # qa symlink checker will immediately complain if not "_" in basename(bbFname) and ".new" not in basename(bbFname): bbLink = join(bbTargetDir, basename(bbFname)) makeSymlink(bbFname, bbLink) @@ -2512,58 +2519,131 @@ # Write only when the release string really changed. This file feeds the # trackDb dataVersion setting and its date is what people check to see whether # the pipeline is alive, so rewriting an identical file would make a stalled # track look freshly updated. refs #38300 oldVersion = None if isfile(versionFname): oldVersion = open(versionFname, encoding="utf8").read() if oldVersion == fullVersion: logging.debug("%s already holds the current release string, not rewriting" % versionFname) else: with open(versionFname, "w", encoding="utf8") as versionOfh: versionOfh.write(fullVersion) logging.info("Wrote release string to %s" % versionFname) + hubDir = genArkHubDir(db) + if hubDir is not None: + # a GenArk assembly has no /gbdb/<db>/uniprot to link into; its files and + # its trackDb go into the contrib collection, which carries the version + # string inline because a contrib trackDb cannot read a /gbdb path + installGenArkContrib(db, hubDir, dbDir, fullVersion, shortVersion) + continue + linkName = join("/gbdb", db, "uniprot", "version.txt") makeSymlink(versionFname, linkName) logging.debug("Release string in %s, symlink from %s" % (versionFname, linkName)) return versionString, shortVersion +genArkContribRoot = "/hive/data/genomes/asmHubs/contrib" +genArkContribName = "uniprot" + +def makeContribTrackDb(versionString, shortVersion): + """ trackDb stanzas for the GenArk contrib collection, from the same template the + archive hub uses. Three differences from the archive: the track names must not carry + the release, or they would change every month and break saved sessions; the data files + sit beside trackDb.txt rather than in a per-release subdirectory; and dataVersion has + to be the literal string, because a contrib trackDb cannot point at a /gbdb path. + """ + templFname = join(dirname(__file__), "trackDb.template.txt") + tdb = open(templFname).read() + + tdb = re.sub(r'^(\s*(?:track|parent)\s+\S+?)\$VER', r'\1', tdb, flags=re.MULTILINE) + tdb = tdb.replace("bigDataUrl $VER/", "bigDataUrl ") + tdb = tdb.replace("$VER", shortVersion) + + lines = [l for l in tdb.splitlines() if not l.startswith("#")] + # the composite stanza is first, hang the shared description page and the version off it + out = [] + for line in lines: + out.append(line) + if line.strip() == "track " + genArkContribName: + out.append("html ../docs/%s" % genArkContribName) + out.append("dataVersion %s" % versionString) + return "\n".join(out) + "\n" + +def installGenArkContrib(db, hubDir, dbBigBedDir, versionString, shortVersion): + """ write this assembly's bigBeds and a trackDb into the GenArk contrib collection. + A GenArk assembly is not served out of /gbdb/<db>/, so the /gbdb symlinks and the + trackDb .ra stanza that classic assemblies get do not apply. + + This only fills in the collection under contrib/uniprot/. It deliberately does NOT + symlink anything into the GenArk build directories and does not touch any hub.txt: + installing the collection is a separate, deliberate step, done with + "genark addContrib uniprot" when someone decides to, not something a data update + should do behind your back. + """ + acc = basename(hubDir.rstrip("/")) + collDir = join(genArkContribRoot, genArkContribName, acc) + if not isdir(collDir): + os.makedirs(collDir) + + count = 0 + for bbName in sorted(glob.glob(join(dbBigBedDir, "*.bb"))): + # skip the historical per-geneTable files, only the current track files belong here + if not basename(bbName).startswith("unip"): + continue + shutil.copyfile(bbName, join(collDir, basename(bbName))) + count += 1 + + tdbFname = join(collDir, "trackDb.txt") + with open(tdbFname, "w") as ofh: + ofh.write(makeContribTrackDb(versionString, shortVersion)) + + logging.info("%s: wrote %d bigBeds and a trackDb to %s" % (db, count, collDir)) + return collDir + def makeTrackDb(archDir, shortVersion): " create a trackDb.txt file for the archive hub in archDir " templFname = join(dirname(__file__), "trackDb.template.txt") tdb = open(templFname).read() tdb = tdb.replace("$VER", shortVersion) tdbLines = tdb.splitlines() tdbLines = [l for l in tdbLines if not l.startswith("#")] tdbFname = join(archDir, "trackDb.txt") with open(tdbFname, "w") as ofh: ofh.write("\n".join(tdbLines)) logging.info("Created %s" % tdbFname) def copyToArchive(bigBedDir, archRoot, shortVersion, onlyDbs): " make copies of the track files under the archiveDir and adapt the 'current' symlink " for db in os.listdir(bigBedDir): if onlyDbs and db not in onlyDbs: continue + if genArkHubDir(db) is not None: + # goldenPath/archive/<db>/ is a classic-assembly download path; a GenArk + # assembly is published under its accession in the hubs tree instead, so + # archiving one here would just create a directory nothing can reach + logging.debug("%s is a GenArk assembly, not archiving under goldenPath" % db) + continue + archDir = join(archRoot, db, "uniprot", shortVersion) if not isdir(archDir): os.makedirs(archDir) inDir = join(bigBedDir, db) count = 0 for inFname in glob.glob(join(inDir, "*")): shutil.copyfile(inFname, join(archDir, basename(inFname))) count += 1 logging.info("Archive: Copied %d files from %s to %s" % (count, inDir, archDir)) makeTrackDb(archDir, shortVersion) currLink = join(archRoot, db, "uniprot", "current")