af613a331e6839c6513c3e366abcb67af0fe8386 max Wed Sep 9 06:47:14 2026 -0700 UniProt otto: get the monthly update running again and make a stalled run visible The monthly UniProt job had produced nothing since January 2025. The tracks served release 2024_06 while the download sitting on disk was at 2026_02, on every assembly the job builds. Cause: uniprotToTab appended a personal conda site-packages directory to sys.path, and doUpdate.sh sourced a virtualenv, both built for python 3.6. A venv's python is only a symlink to the system one, so when hgwdev moved to python 3.9 the compiled lxml in there stopped loading and every run died at the parse step. Removed both. The system python3 has lxml from python3-lxml and the two are upgraded together, so there is nothing left here to go stale. Verified by parsing real 2026_02 records under python 3.9 with lxml 5.4. Why nobody noticed for nineteen months: - doUpdate.sh read $? after an intervening echo, so it captured the echo's exit code and mailed "Big Uniprot update OK" every month while the job was dying. It now reads the real exit code, says FAILED, prints the tail of the log and exits nonzero. A month with no new UniProt release stays silent, which is the normal otto behaviour, so silence again means "nothing to do". - The logs were overwritten on every run, so a failure left no trace on disk. doUpdate.sh now appends one line per run to runLog.txt, which is never truncated, and keeps a failing log as lastFail.log. - version.txt in each bigBed directory was rewritten on every run even when the release string was identical. That is the file the trackDb dataVersion setting shows, and its date is what people check to decide whether a pipeline is still alive, so a stalled track could look freshly updated. It is now written only when the release actually changes. Also, so this cannot come back: - doUniprot checks that uniprotToTab can start before the download, instead of finding out 35 minutes later. - pylint on hgwdev is itself pinned to pythons that no longer exist, so "make install" aborted on its first line and could not be used. Replaced with a syntax check that needs nothing but python3; pylint stays best-effort. - uniprotToTab, pslProtCnv, trackDb.template.txt and README.txt ran from /hive/data/outside/otto/uniprot without being in the makefile's copy list. The tree copy of uniprotToTab was still python 2 from 2021. All are now listed and in sync, and "make diff" reports drift. - Brought the two live-only fixes into the tree: mkdir -p in makeUniProtPsl.sh and the pslMap -inType/-mapType flags. refs #38300 diff --git src/hg/utils/otto/uniprot/doUniprot src/hg/utils/otto/uniprot/doUniprot index 21b5dbb1445..caf121b8e7f 100755 --- src/hg/utils/otto/uniprot/doUniprot +++ src/hg/utils/otto/uniprot/doUniprot @@ -1407,37 +1407,39 @@ ] if doTrembl: fnames.append( (join(tabDir,"trembl.%d.tab" % taxId)) ) ret = {} for fname in fnames: for row in iterTsvRows(open(fname)): for acc in getAllIsoAccs(row): ret[acc] = row return ret def blatProteinsKeepBest(fullFaFname, db, mapFname, stats): " blat the proteins directly, only good for small genomes where we don't have any gene models " myDir = dirname(__file__) + # consider using miniprot here one day? cmd = "blat -q=prot -t=dnax /gbdb/{db}/{db}.2bit {inf} stdout -noHead | sort -k10 | pslReps stdin stdout /dev/null " \ "-minAli=0.95 -nohead | {myDir}/pslProtCnv > {out}".format(db=db, inf=fullFaFname, out=mapFname, myDir=myDir) stats["minAli"]=0.95 #nuclFname = mapFname+".nucl" # originally used "| {myDir}/pslProtCnv " but tried to get rid of the markd-script dependency #pslToProtPsl(nuclFname, mapFname) # then later figured that pslToProtPsl() is not the same as pslProtCnv and went back the MarkD way + # then in 2024 finally used pslProtToRnaCoords #os.remove(nuclFname) run(cmd) def getTransIds(db, geneTable, transcriptFa): """ return all possible transcript IDs given a gene table. The reason that we need is that Uniprot often refers to transcripts that don't exist and we are using a select file to map only to those. In these cases, we want to make sure that our select file contains only transcript that we actually have, so we can log how many transcript can possibly mapped (and which ones). This is important for debugging. """ # no idea how to get the version suffix out of our mysql tables - thanks Obama! if geneTable=="refGene": logging.debug("Gene table is refGene, reading IDs from %s" % transcriptFa) allTrans = set(parseFasta(open(transcriptFa)).keys()) return allTrans @@ -1952,39 +1954,59 @@ # 'UniProt Knowledgebase Release 2017_09 consists of:' relString = open(join(uprotDir, "reldate.txt")).read().splitlines()[0] relString = " ".join(relString.split()[:4]) return relString def writeReleaseString(uprotDir, tabDir): " read release info file from uprotDir and create a shorter version of it in tabDir " relString = readRelStringUniprot(uprotDir) relFname = join(tabDir, "version.txt") relFh = open(relFname, "w") relFh.write(relString) relFh.close() logging.debug("Wrote release version string '%s' to %s" % (relString, relFname)) +def checkParserDeps(): + """ make sure that uniprotToTab can start up at all, i.e. that its XML library imports. + Called before the download, because the download takes over half an hour and it is + pointless to run it when the parser that comes next cannot start. This check exists + because exactly that happened for 19 months: the lxml module could not be imported, + every run died right here and the tracks stayed on release 2024_06 (redmine #38300). + """ + parserFname = join(dirname(__file__), "uniprotToTab") + if not isfile(parserFname): + errAbort("%s does not exist. Run 'make install' in the kent tree's " + "src/hg/utils/otto/uniprot directory." % parserFname) + # --help imports lxml and then exits, so this is a cheap test of the real interpreter + if os.system("%s --help > /dev/null 2>&1" % parserFname) != 0: + os.system("%s --help" % parserFname) # show the error message in the log + errAbort("%s cannot be run, see the error above. Most likely the lxml python module " + "is not importable. Check with: python3 -c 'import lxml.etree'" % parserFname) + def updateUniprot(args, onlyDbs, taxIdDbs, options): " This is the main function that runs a single uniprot update for a list of DBs " uprotDir = options.uniprotDir tabDir = options.tabDir mapDir = options.mapDir bigBedDir = options.bigBedDir faDir = options.faDir doTrembl = not options.skipTrembl + if not options.skipParse: + checkParserDeps() + if not options.skipDownload and not options.skipParse and not options.onlyDbs: # download NCBI -> refseq tables downloadAndSplitNcbi(taxIdDbs) downloadUniprot(uprotDir) downVersion = readRelStringUniprot(uprotDir) tabVersion = open(join(tabDir, "version.txt")).read() doParse = True if downVersion==tabVersion: logging.info("Not converting to tab again, found same version is in tab directory") doParse = False if options.skipParse: logging.info("Not converting to tab, was switched off by option") @@ -2284,41 +2306,56 @@ versionString = open(relFname, encoding="utf8").read().strip() shortVersion = versionString.split()[-1] if (len(shortVersion)!=7): logging.warning("UniProt Version string is not seven characters long") logging.debug("Release is: %s" % shortVersion) for taxId, dbs in taxIdDbs.items(): for db in dbs: if onlyDbs is not None and db not in onlyDbs: continue dbDir = join(bigBedDir, db) if not isdir(dbDir): continue liftInfo = json.load(open(join(dbDir, "liftInfo.json"))) - versionOfh = open(join(dbDir, "version.txt"), "w") + versionFname = join(dbDir, "version.txt") fullVersion = versionString + (", mapped to genome through gene transcripts from %s %s on %s (taxId %s, %s)" % \ (liftInfo["geneTable"], liftInfo["version"], liftInfo["createdDate"], liftInfo["taxId"], liftInfo["fullMd5"])) + + # Write only when the string really changed. This file is what the trackDb + # dataVersion setting shows in the track description, and its date on disk is + # what everyone looks at to decide whether the pipeline is still alive. + # Rewriting an identical file every run makes a track that has not moved for a + # year look like it was updated yesterday, which is how the 19-month stall in + # redmine #38300 stayed hidden. An unchanged track keeps its old date. + oldVersion = None + if isfile(versionFname): + oldVersion = open(versionFname, encoding="utf8").read() + + if oldVersion == fullVersion: + logging.debug("%s already holds the current release string, not rewriting" % versionFname) + else: + with open(versionFname, "w", encoding="utf8") as versionOfh: versionOfh.write(fullVersion) - versionOfh.close() + logging.info("Wrote release string to %s" % versionFname) linkName = join("/gbdb", db, "uniprot", "version.txt") - makeSymlink(versionOfh.name, linkName) + makeSymlink(versionFname, linkName) - logging.debug("Wrote release string to %s, symlink from %s" % (versionOfh.name, linkName)) + logging.debug("Release string in %s, symlink from %s" % (versionFname, linkName)) return versionString, shortVersion def makeTrackDb(archDir, shortVersion): " create a trackDb.txt file for the archive hub in archDir " templFname = join(dirname(__file__), "trackDb.template.txt") tdb = open(templFname).read() tdb = tdb.replace("$VER", shortVersion) tdbLines = tdb.splitlines() tdbLines = [l for l in tdbLines if not l.startswith("#")] tdbFname = join(archDir, "trackDb.txt") with open(tdbFname, "w") as ofh: