1769eab7a2ee4a732e0de1152e6f50c1be8dc585
max
  Thu Sep 17 11:00:45 2026 -0700
uniprot otto: only the newest version of an assembly, and skip whole assemblies

#Preview2 week - bugs introduced now will need a build patch to fix
GenArk lists every version it has ever carried, so GCA_018466835.1 sits beside
GCA_018466835.2. Building the old one produces a track nobody will look at. Keep
only the highest version of each accession.

The order matters, and getting it wrong is silent. Doing this after the skip list
leaves the old version behind wherever the new one was skipped, which is exactly the
HPRC case: 94 of the 108 human assemblies that survived the HPRC orderList were .1
versions whose .2 had just been removed, so the filtering had quietly swapped the
current assemblies for their predecessors. The versions are now collapsed against
the whole list, before anything else is filtered.

For the same reason an entry on the skip list now matches the assembly rather than
one of its versions, so naming GCA_018466835.2 also drops GCA_018466835.1.

Human goes from 572 GenArk assemblies to 14, and those 14 are the genuinely distinct
ones: NA12878, mHomSap3, HG03492, HG01243, NA24385, RPE-1, H9 and an iPSC line. It
is no longer the longest pole in the run; worm and mouse are.

refs #38300

diff --git src/hg/utils/otto/uniprot/doUniprot src/hg/utils/otto/uniprot/doUniprot
index a31f64630bf..c9007777c2c 100755
--- src/hg/utils/otto/uniprot/doUniprot
+++ src/hg/utils/otto/uniprot/doUniprot
@@ -200,53 +200,71 @@
     """ build the plan from the GenArk assembly list instead of from dbDb.
 
     dbDb holds the few thousand assemblies the browser serves directly; GenArk holds about
     fifty thousand. Most of those are not worth running: UniProt annotates most species
     barely at all, and an assembly whose organism has a handful of proteins cannot produce
     a useful track. --minProteins is the filter, and it wants to be high to start with.
     A bacterial proteome is only a few thousand proteins, so a threshold of a few thousand
     pulls in the whole of the bacteria clade; 20000 keeps it to the large eukaryotic
     proteomes. refs #38300
     """
     accToTax = readGenArkList(options.genArkList)
     counts = taxonProteinCounts(options.uniprotDir)
     minProt = options.minProteins
     skipAsm = readSkipList(options.skipList)
 
+    # Keep only the newest version of each assembly, and do it against the whole list
+    # before anything else is filtered out. GenArk lists every version it has carried, so
+    # GCA_018466835.1 sits beside GCA_018466835.2. Doing this after the skip list would
+    # leave the old version behind whenever the new one was skipped, which is exactly the
+    # HPRC case: 94 of the 108 human assemblies that survived the HPRC list were .1
+    # versions of accessions whose .2 had just been removed.
+    newest = {}
+    for acc in accToTax:
+        base, _, ver = acc.rpartition(".")
+        ver = int(ver) if ver.isdigit() else 0
+        if base not in newest or ver > newest[base][0]:
+            newest[base] = (ver, acc)
+    oldCount = len(accToTax) - len(newest)
+    accToTax = {acc: accToTax[acc] for _, acc in newest.values()}
+
+    # an entry on the skip list names an assembly, not one of its versions
+    skipBases = set(a.rpartition(".")[0] for a in skipAsm if "." in a)
+
     taxIdToDbs = defaultdict(list)
     skippedTaxa, skippedAsm, noHub, blacklisted = set(), 0, 0, 0
     for acc, taxId in sorted(accToTax.items()):
         if onlyDbs is not None and acc not in onlyDbs:
             continue
-        if acc in skipAsm:
+        if acc in skipAsm or acc.rpartition(".")[0] in skipBases:
             blacklisted += 1
             continue
         if counts.get(taxId, 0) < minProt:
             skippedTaxa.add(taxId)
             skippedAsm += 1
             continue
         if genArkAccDir(acc) is None or not isdir(genArkAccDir(acc)):
             # listed but not served from /gbdb/genark here, nothing to build against
             noHub += 1
             continue
         taxIdToDbs[taxId].append(acc)
 
     logging.info("GenArk plan: %d assemblies over %d taxa with at least %d UniProt proteins" % \
             (sum(len(v) for v in taxIdToDbs.values()), len(taxIdToDbs), minProt))
     logging.info("GenArk plan: skipped %d assemblies over %d taxa below the protein threshold, "
-            "%d on the skip list, and %d not present under %s" % \
-            (skippedAsm, len(skippedTaxa), blacklisted, noHub, genArkRoot))
+            "%d on the skip list, %d superseded by a newer version, and %d not present "
+            "under %s" % (skippedAsm, len(skippedTaxa), blacklisted, oldCount, noHub, genArkRoot))
     return taxIdToDbs
 
 def getTaxIdDbs(onlyDbs, options):
     """ return a dict with taxonId -> list of most recent dbs (e.g. for human, it's hg19 and hg38) 
     if onlyDbs is a set, keep only these dbs.
     If onlyDbs is None, remove all nonAutoDbs.
     """
     if options.genArkList:
         return getGenArkTaxIdDbs(onlyDbs, options)
 
     skipAsm = readSkipList(options.skipList)
     query = "select taxId, name, nibPath from dbDb where active=1 order by orderKey;"
     rows = runQuery("hgcentral", query, usePublic=True)
     taxIdToDbs = defaultdict(list)
     # Take the first assembly of each kind for every taxon: the first GenArk one and the