bee41315e53da6fb5671f96302233c655906dfd5 max Wed Sep 9 06:56:49 2026 -0700 UniProt otto: rebuild the venv rather than relying on a system lxml Correction to the previous commit: hgwdev has no system-wide lxml at all. The import I tested was resolving to my own ~/.local/lib/python3.9/site-packages, which cron never sees, because it runs the pipeline as otto. So the environment is a virtualenv again, but a reproducible one. makeVenv.sh deletes venv/ and rebuilds it from /usr/bin/python3, installs lxml, opens up the permissions for otto, and then checks that lxml imports with an empty environment so we know the venv stands on its own instead of borrowing from whoever ran it. Built with --copies, so venv/bin/python is a real copy rather than a symlink that would silently follow a system python upgrade while its compiled modules stayed behind. doUpdate.sh activates venv/ again and says to run makeVenv.sh if it is missing or if the parser will not start. Verified: /usr/bin/python3 without per-user packages cannot import lxml, the venv can, and after activation the parser runs and converts real 2026_02 records. Also shortened the README to how the pipeline is started and how it works, and trimmed the history out of the code comments, leaving the ticket as the pointer. refs #38300 diff --git src/hg/utils/otto/uniprot/doUniprot src/hg/utils/otto/uniprot/doUniprot index caf121b8e7f..aeeb99b6888 100755 --- src/hg/utils/otto/uniprot/doUniprot +++ src/hg/utils/otto/uniprot/doUniprot @@ -1956,44 +1956,43 @@ relString = " ".join(relString.split()[:4]) return relString def writeReleaseString(uprotDir, tabDir): " read release info file from uprotDir and create a shorter version of it in tabDir " relString = readRelStringUniprot(uprotDir) relFname = join(tabDir, "version.txt") relFh = open(relFname, "w") relFh.write(relString) relFh.close() logging.debug("Wrote release version string '%s' to %s" % (relString, relFname)) def checkParserDeps(): """ make sure that uniprotToTab can start up at all, i.e. that its XML library imports. - Called before the download, because the download takes over half an hour and it is - pointless to run it when the parser that comes next cannot start. This check exists - because exactly that happened for 19 months: the lxml module could not be imported, - every run died right here and the tracks stayed on release 2024_06 (redmine #38300). + Called before the download, which takes over half an hour, so a missing lxml is + reported in the first second of a run instead of after it. refs #38300 """ parserFname = join(dirname(__file__), "uniprotToTab") if not isfile(parserFname): errAbort("%s does not exist. Run 'make install' in the kent tree's " "src/hg/utils/otto/uniprot directory." % parserFname) # --help imports lxml and then exits, so this is a cheap test of the real interpreter if os.system("%s --help > /dev/null 2>&1" % parserFname) != 0: os.system("%s --help" % parserFname) # show the error message in the log errAbort("%s cannot be run, see the error above. Most likely the lxml python module " - "is not importable. Check with: python3 -c 'import lxml.etree'" % parserFname) + "does not import. Activate the venv with 'source venv/bin/activate', or " + "rebuild it with ./makeVenv.sh" % parserFname) def updateUniprot(args, onlyDbs, taxIdDbs, options): " This is the main function that runs a single uniprot update for a list of DBs " uprotDir = options.uniprotDir tabDir = options.tabDir mapDir = options.mapDir bigBedDir = options.bigBedDir faDir = options.faDir doTrembl = not options.skipTrembl if not options.skipParse: checkParserDeps() if not options.skipDownload and not options.skipParse and not options.onlyDbs: # download NCBI -> refseq tables @@ -2311,36 +2310,34 @@ for taxId, dbs in taxIdDbs.items(): for db in dbs: if onlyDbs is not None and db not in onlyDbs: continue dbDir = join(bigBedDir, db) if not isdir(dbDir): continue liftInfo = json.load(open(join(dbDir, "liftInfo.json"))) versionFname = join(dbDir, "version.txt") fullVersion = versionString + (", mapped to genome through gene transcripts from %s %s on %s (taxId %s, %s)" % \ (liftInfo["geneTable"], liftInfo["version"], liftInfo["createdDate"], liftInfo["taxId"], liftInfo["fullMd5"])) - # Write only when the string really changed. This file is what the trackDb - # dataVersion setting shows in the track description, and its date on disk is - # what everyone looks at to decide whether the pipeline is still alive. - # Rewriting an identical file every run makes a track that has not moved for a - # year look like it was updated yesterday, which is how the 19-month stall in - # redmine #38300 stayed hidden. An unchanged track keeps its old date. + # Write only when the release string really changed. This file feeds the + # trackDb dataVersion setting and its date is what people check to see whether + # the pipeline is alive, so rewriting an identical file would make a stalled + # track look freshly updated. refs #38300 oldVersion = None if isfile(versionFname): oldVersion = open(versionFname, encoding="utf8").read() if oldVersion == fullVersion: logging.debug("%s already holds the current release string, not rewriting" % versionFname) else: with open(versionFname, "w", encoding="utf8") as versionOfh: versionOfh.write(fullVersion) logging.info("Wrote release string to %s" % versionFname) linkName = join("/gbdb", db, "uniprot", "version.txt") makeSymlink(versionFname, linkName) logging.debug("Release string in %s, symlink from %s" % (versionFname, linkName))