51975d67889e63fbc5fbad0cf1a909ebcca5f25e max Sat Sep 26 14:15:10 2026 -0700 uniprot otto: let a run finish when a few taxa fail, and say where to read about them A run over hundreds of organisms always has a few that cannot be built: one UniProt barely annotates, one NCBI has no gene table for, a genome needing more memory than was reserved. Until now any single one of them aborted the run before the flip, so 24 failures out of 664 taxa kept the other 640 from being published, twice. --allowFailures=N carries on and publishes when no more than N taxa failed, and doUpdate.sh passes 10 unless the caller says otherwise, so the monthly cron is no longer hostage to a handful of awkward organisms. Above the threshold it still aborts and publishes nothing, which is the right answer when something systemic has broken. Nothing is quieter as a result. Every failure is reported with its traceback as before, and now each one also gets its own file under failedTaxa/<taxId>.log holding the taxon, its assemblies, the time and the traceback. lastRun.log is overwritten by the next run and interleaves every taxon, so a failure someone wants to look at a day later was hard to find in it; these files are not overwritten except by another failure of the same taxon. The end-of-run report names each failed taxon with its assemblies and the path to its log, and prints the --dbs argument to retry exactly those. The failure mail from doUpdate.sh lists the log paths too. Checked all three paths: below the threshold the run continues, above it aborts, with no failures nothing changes. Checked that the cron form gets --allowFailures=10, that an explicit --allowFailures wins, and that it does not disturb other arguments. refs #38300 diff --git src/hg/utils/otto/uniprot/doUniprot src/hg/utils/otto/uniprot/doUniprot index 9e178188f3b..0ac83f3e0e0 100755 --- src/hg/utils/otto/uniprot/doUniprot +++ src/hg/utils/otto/uniprot/doUniprot @@ -354,30 +354,37 @@ help="do not create the /gbdb/ symlinks") parser.add_option("", "--archiveDir", dest="archiveDir", action="store", \ default="/usr/local/apache/htdocs-hgdownload/goldenPath/archive/", help="Location of archive directory, default %default") parser.add_option("", "--taxonThreads", dest="taxonThreads", action="store", type="int", default=20, help="how many taxa to process at the same time, default %default. Raising this " "keeps the cluster busy: one taxon at a time leaves it idle during the long " "single-threaded steps between batches. Around 20 is a reasonable working value. Assemblies " "of the same taxon always run one after the other, they share a fasta file.") parser.add_option("", "--genArkList", dest="genArkList", action="store", help="build the plan from this GenArk assembly list instead of from dbDb, e.g. " "/hive/data/genomes/asmHubs/UCSC_GI.assemblyHubList.txt or a fresh copy of " "https://hgdownload.soe.ucsc.edu/hubs/UCSC_GI.assemblyHubList.txt . Use with " "--minProteins, which decides how much of GenArk is worth running.") + parser.add_option("", "--allowFailures", dest="allowFailures", action="store", type="int", + default=0, + help="carry on and publish if no more than this many taxa fail, default %default. " + "A run across hundreds of assemblies will always have a few organisms that cannot " + "be built, and without this a handful of them stops the other hundreds from being " + "published. Every failure is still reported and gets its own log under %s/." % + taxonFailDir) parser.add_option("", "--skipList", dest="skipList", action="store", help="file of assemblies to leave alone, one name or accession per line, with " "# comments. Entries may carry the full asmId, so a GenArk orderList works as " "is: --skipList=kent/src/hg/makeDb/doc/hprcAsmHub/hprc.orderList.tsv drops the " "464 HPRC haplotype assemblies, which take human from 572 assemblies to 108.") parser.add_option("", "--minProteins", dest="minProteins", action="store", type="int", default=1, help="skip a taxon with fewer than this many UniProt proteins, default %default, " "i.e. skip only the empty ones. UniProt annotates most species barely at all: of " "the 3974 taxa that have a GenArk assembly and a SwissProt entry, the median has " "four proteins and only 558 have more than a hundred. Raise this when running " "across many assemblies, to skip the ones that cannot produce a useful track.") parser.add_option("", "--mapQa", dest="mapQa", action="store_true", \ help="output some QA stats for the maps") parser.add_option("", "--db", dest="db", action="store_true", \ @@ -2640,30 +2647,65 @@ convMapToBigPsl(fullFaFname, mapFname, dbBigBedDir, accToDb, accToMeta, accToMapSource, accToTrans, chromSizesFname, doTrembl) #shutil.copy(mapDescFname, mapDir) if options.onlyMap: continue annotFnames = ["tab/swissprot.%d.annots.tab" % taxId] if doTrembl: annotFnames.append( "tab/trembl.%d.annots.tab" % taxId ) uniprotLift(fullFaFname, annotFnames, chromSizesFname, mapFname, dbBigBedDir, accToDb, accToMeta, options) shutil.copyfile(mapDescFname, join(dbBigBedDir, "liftInfo.json")) +taxonFailDir = "failedTaxa" + +def writeTaxonFailLog(taxId, dbs, ex, trace): + """ write one file per failed taxon, holding what failed and how, and return its name. + + lastRun.log is overwritten by the next run and holds every taxon interleaved, so a + failure that someone wants to look at a day later is hard to find in it. One file per + taxon, named after the taxon, survives and is easy to point at. + """ + if not isdir(taxonFailDir): + os.makedirs(taxonFailDir, exist_ok=True) + fname = join(taxonFailDir, "%s.log" % taxId) + with open(fname, "w") as ofh: + ofh.write("taxon: %s\n" % taxId) + ofh.write("assemblies: %s\n" % ", ".join(dbs)) + ofh.write("when: %s\n" % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + ofh.write("failure: %s: %s\n\n" % (type(ex).__name__, ex)) + ofh.write(trace) + return fname + +def reportFailedTaxa(failed, todo): + " print a block naming every failed taxon, its assemblies and where to read about it " + dbsByTax = dict(todo) + logging.error("") + logging.error("%d taxa failed. Each one has a log of its own:" % len(failed)) + for taxId in sorted(failed): + dbs = ", ".join(dbsByTax.get(taxId, [])) + logging.error(" taxon %-10s %-56s %s/%s.log" % (taxId, dbs[:56], taxonFailDir, taxId)) + logging.error("") + logging.error("Those logs are under %s in %s and are not overwritten by the next run." % \ + (taxonFailDir, os.getcwd())) + logging.error("To retry just these, add: --dbs=%s" % \ + ",".join(sorted(set(d for t in failed for d in dbsByTax.get(t, []))))) + logging.error("") + def runTaxa(taxIdDbs, onlyDbs, options, tabDir, faDir, mapDir, bigBedDir, doTrembl): """ work through the taxa, optionally several at a time. One taxon at a time leaves the cluster mostly idle: each assembly submits its BLAST batch, waits for it to drain, and then spends the best part of an hour in the single-threaded pslReps that follows before the next assembly submits anything. Running several taxa at once keeps jobs queued while others are in that serial phase. Taxa, not assemblies: the assemblies of one taxon share fasta/<taxId>.fa, which is rebuilt at the start of each taxon, so two threads on the same taxon would race on it. Everything below that point is per-assembly - protToGenome/<db>, bigBed/<db>, and the cluster batch directory - so separate taxa do not touch the same files. """ todo = [(taxId, dbs) for taxId, dbs in taxIdDbs.items() if onlyDbs is None or len(set(dbs).intersection(onlyDbs))>0] @@ -2680,36 +2722,52 @@ for taxId, dbs in todo: fut = pool.submit(oneTaxon, taxId, dbs, onlyDbs, options, tabDir, faDir, mapDir, bigBedDir, doTrembl) futures[fut] = taxId for fut in concurrent.futures.as_completed(futures): taxId = futures[fut] try: fut.result() except BaseException as ex: # BaseException, not Exception: run() reports a failed command with # sys.exit(), which raises SystemExit. In a worker thread that would # otherwise kill just that thread and leave the run looking successful, # which is exactly the kind of silent failure this pipeline has a history of. # Log the traceback too: errAbort raises a bare AssertionError, so without # one the message is just "AssertionError:" and says nothing about where. - logging.error("taxon %s failed: %s: %s\n%s" % (taxId, type(ex).__name__, ex, - "".join(traceback.format_exception(type(ex), ex, ex.__traceback__)))) + trace = "".join(traceback.format_exception(type(ex), ex, ex.__traceback__)) + logFname = writeTaxonFailLog(taxId, dict(todo).get(taxId, []), ex, trace) + logging.error("taxon %s failed: %s: %s (details in %s)" % \ + (taxId, type(ex).__name__, ex, logFname)) + logging.error(trace) failed.append(taxId) - if failed: - errAbort("%d of %d taxa failed: %s" % (len(failed), len(todo), ", ".join(str(t) for t in sorted(failed)))) + if not failed: + return + + reportFailedTaxa(failed, todo) + + allowed = options.allowFailures + if len(failed) <= allowed: + logging.warning("%d taxa failed, which is within the %d allowed, so the run carries " + "on and the assemblies that did build will be published. Re-run the failed " + "taxa with --dbs once they are fixed." % (len(failed), allowed)) + return + + errAbort("%d of %d taxa failed, more than the %d allowed by --allowFailures. " + "Nothing will be published. See %s for what went wrong." % \ + (len(failed), len(todo), allowed, taxonFailDir)) def makeSymlink(target, linkName): assert(".new" not in linkName) # make sure that this bug never happens again if target is None: logging.error("internal error, but not stopping: Symlink to a None target?") return targetPath = abspath(target) if not isfile(targetPath): logging.error("Cannot symlink: %s does not exist" % str(target)) return if isfile(linkName):