2b14cc7d4cd492874f2e2a89e05d5cd720dbe304 hiram Thu Oct 1 15:54:21 2026 -0700 do not use any alias names if such a name, case insensitive, is already a name in hgcentral.dbDb.name column refs #38444 diff --git src/hg/utils/otto/genArk/asmAlias/asmAliasUpdate.py src/hg/utils/otto/genArk/asmAlias/asmAliasUpdate.py index 088eb029402..9850d0e8741 100755 --- src/hg/utils/otto/genArk/asmAlias/asmAliasUpdate.py +++ src/hg/utils/otto/genArk/asmAlias/asmAliasUpdate.py @@ -3,50 +3,58 @@ asmAliasUpdate.py Prepare a complete replacement tab separated two column (alias, browser) dump of the hgcentraltest 'asmAlias' table: every existing row is carried forward untouched, EXCEPT: - rows on a GenBank (GCA) browser that has since been superseded by an equivalent live RefSeq (GCF) browser -- those are dropped and replaced with equivalent rows pointing at the new RefSeq browser - a row whose alias's genark data now resolves to a newer dot version of the *same* GCA/GCF accession root (see isSameAssemblyNewerVersion) -- these are repointed ('UPDATE:') to the newer version. This is never done when the existing browser is a plain UCSC database name (e.g. 'hg38', 'dm6') rather than a GCA_/GCF_ accession -- those are always left alone, and any other kind of mismatch is only reported ('ERROR:'), never silently changed + - a row whose alias text collides (case insensitive) with an existing + hgcentraltest.dbDb 'name' is dropped ('DBDB COLLISION:') -- using it + would make that real dbDb browser's own full-text search term also + match some unrelated genArk assembly (see hg/hubApi/assemblyList.py, + which builds its browsable index from both dbDb and asmAlias data) New rows discovered from the genark data are appended; a candidate alias that is nothing but digits and dots (e.g. '1.0', '26') -- a bare version -number, never a useful alias -- is never generated in the first place. +number, never a useful alias -- is never generated in the first place, +and the same goes for a candidate that collides with an existing +hgcentraltest.dbDb 'name' (case insensitive). As a final pass over the complete row set (existing + new), any two alias spellings that differ only by case (e.g. 'FF3'/'Ff3') are resolved, since asmAlias lookups are case-insensitive and such pairs can never coexist as distinct primary keys: if they point to the same browser, one spelling is kept and the duplicate silently dropped ('NOTE:'); if they point to different browsers, there is no way to know which is right, so the whole group is dropped and reported ('ERROR:'). The intent is that this file becomes the entire new table content (e.g. via a truncate-and-reload), not a delta to apply by hand. The script only reports (to stderr) other problems it finds with the existing data -- orphaned browsers, alias collisions, apparent name changes -- for a human to decide what to do about; it does not fix those itself. Data sources, all read live and kept in memory -- no intermediate files: - hgcentraltest.asmAlias (via hgsql) + - hgcentraltest.dbDb (via hgsql) - genark.assemblySummaryGenbank (via hgsql) - genark.assemblySummaryGenbankHistorical (via hgsql) - genark.assemblySummaryRefseq (via hgsql) - genark.assemblySummaryRefseqHistorical (via hgsql) - https://hgdownload.gi.ucsc.edu/hubs/UCSC_GI.assemblyHubList.txt column 1 is the set of accessions that actually have a working GenArk hub (browser) instance today. asmAlias.browser must reference something that actually exists, so only accessions found in this list can become a 'browser' value here. Every live-hub accession is first assigned a "target" browser value (computeBrowserTargets): normally an accession targets itself, but a GenBank (GCA) accession whose paired RefSeq (GCF) accession also has a live hub is considered superseded -- RefSeq is always preferred, so its target is redirected to that GCF instead. All of the accession's own @@ -182,52 +190,75 @@ continue genarkData[accession] = (asmName, gbrsPairedAsm, ftpPath, table) return genarkData def fetchExistingAsmAlias(): """Return (aliasToBrowser dict, browserToAliases dict) from hgcentraltest.""" aliasToBrowser = {} browserToAliases = {} for alias, browser in hgsqlRows(centralDb, "SELECT alias, browser FROM asmAlias"): aliasToBrowser[alias] = browser browserToAliases.setdefault(browser, set()).add(alias) return aliasToBrowser, browserToAliases +def fetchDbDbNames(): + """Return the lower-cased set of every hgcentraltest.dbDb 'name' -- an + alias that collides with one of these (case insensitive) must never be + used for anything else: it would make that real dbDb browser's own + full-text search term also match some unrelated genArk assembly (see + hg/hubApi/assemblyList.py, which builds its browsable index from both + dbDb and asmAlias data).""" + names = set() + for row in hgsqlRows(centralDb, "SELECT name FROM dbDb"): + if row and row[0]: + names.add(row[0].lower()) + return names + + +def findDbDbNameCollisions(aliasToBrowser, dbDbNames): + """Return the set of existing asmAlias aliases (by alias text) that + collide (case insensitive) with an hgcentraltest.dbDb 'name' -- these + must be dropped from the carried-forward set, not just prevented from + being generated going forward.""" + return {alias for alias in aliasToBrowser if alias.lower() in dbDbNames} + + def ftpPathBasename(ftpPath): if isEmpty(ftpPath): return "" return ftpPath.rstrip("/").split("/")[-1] def ftpDerivedName(accession, ftpPath): """The ftpPath basename with the leading 'accession_' prefix removed, e.g. ftpPath .../GCA_012345678.1_someAsmName -> 'someAsmName'.""" base = ftpPathBasename(ftpPath) if not base: return "" prefix = accession + "_" if base.startswith(prefix): return base[len(prefix):] if base == accession: return "" # unexpected shape, but still usable as-is return base -def buildCandidates(accession, target, asmName, gbrsPairedAsm, ftpPath, hubAccessions): +def buildCandidates(accession, target, asmName, gbrsPairedAsm, ftpPath, hubAccessions, + dbDbNames): """Alias candidates derived from 'accession's own data that should point at 'target' -- normally target == accession, but a GenBank accession superseded by a live paired RefSeq hub redirects target to that RefSeq accession (see computeBrowserTargets).""" candidates = set() bestName = ftpDerivedName(accession, ftpPath) if bestName and bestName != target: candidates.add(bestName) # asmName is only usable as an alias when it has no spaces; it's worth # adding in addition to bestName when it differs (underscore/hyphen # style differences between the two are common and both are useful) if (not isEmpty(asmName) and asmName != target and " " not in asmName and asmName != bestName): @@ -240,32 +271,35 @@ candidates.add(fullIdentifier) if target != accession: # this accession has been superseded by its paired browser (see # computeBrowserTargets); its own bare accession becomes an alias # to the new target, e.g. alias=GCA_xxx browser=GCF_xxx candidates.add(accession) if not isEmpty(gbrsPairedAsm) and gbrsPairedAsm != target: # only claim the paired accession as an alias if it does NOT have # its own separate hub at all -- otherwise it's handled either by # its own row directly, or via the "superseded" branch above if gbrsPairedAsm not in hubAccessions: candidates.add(gbrsPairedAsm) - # a bare version number (e.g. '1.0', '26') is never a useful alias - return {c for c in candidates if not isNumericOnlyAlias(c)} + # a bare version number (e.g. '1.0', '26') is never a useful alias, + # and a candidate colliding with an existing dbDb.name must never be + # used as an alias for something else (case insensitive) + return {c for c in candidates if not isNumericOnlyAlias(c) + and c.lower() not in dbDbNames} def computeBrowserTargets(genarkData, hubAccessions): """For every live-hub accession, decide which browser value its own alias candidates should point at: itself, unless it is a GenBank (GCA) accession whose paired RefSeq (GCF) accession also has a live hub -- in that case it is superseded and everything about it redirects to the paired GCF (GCF is always preferred over GCA).""" targets = {} for accession, (asmName, gbrsPairedAsm, ftpPath, table) in genarkData.items(): if accession not in hubAccessions: continue if (accession.startswith("GCA_") and not isEmpty(gbrsPairedAsm) and gbrsPairedAsm in hubAccessions): targets[accession] = gbrsPairedAsm @@ -366,60 +400,60 @@ def isSameNameDifferentFormat(accession, bestName, priorAliases): """True if bestName is just a reformatted version of an alias already on file for this browser -- e.g. the old full ftpPath-basename-style alias 'GCA_000182895.1_CC3' vs. the new prefix-stripped 'CC3', or an underscore/hyphen variant -- rather than an actual name change.""" normBest = normalizeNameForCompare(bestName) prefix = accession + "_" for prior in priorAliases: candidate = prior[len(prefix):] if prior.startswith(prefix) else prior if normalizeNameForCompare(candidate) == normBest: return True return False def buildNewRows(genarkData, hubAccessions, browserTargets, aliasToBrowser, - browserToAliases): + browserToAliases, dbDbNames): # first pass: gather every alias candidate text -> set of target # browsers that want it, and report apparent name changes along the way aliasCandidates = {} # alias -> set of target browsers reportedNameChecks = set() # (target, bestName) already logged once for accession, (asmName, gbrsPairedAsm, ftpPath, table) in genarkData.items(): target = browserTargets.get(accession) if target is None: continue # no live browser instance, can't be used as 'browser' bestName = ftpDerivedName(accession, ftpPath) priorAliases = browserToAliases.get(target, set()) if (priorAliases and bestName and bestName not in priorAliases and (target, bestName) not in reportedNameChecks): reportedNameChecks.add((target, bestName)) if isSameNameDifferentFormat(accession, bestName, priorAliases): sys.stderr.write( "NEW ALIAS: browser=%s -- '%s' is just a reformatted " "version of existing alias(es) %s, adding it as an " "additional alias, not a name change\n" % (target, bestName, sorted(priorAliases))) else: sys.stderr.write( "NOTE: possible name change for browser=%s -- current " "name='%s' not among existing aliases %s\n" % (target, bestName, sorted(priorAliases))) for alias in buildCandidates(accession, target, asmName, gbrsPairedAsm, - ftpPath, hubAccessions): + ftpPath, hubAccessions, dbDbNames): aliasCandidates.setdefault(alias, set()).add(target) # second pass: resolve each alias to a single winning browser, then # reconcile against what's already in the table newRows = [] for alias, accessions in aliasCandidates.items(): winner, discarded = pickWinner(accessions) if winner is None: sys.stderr.write( "ERROR: alias=%s is claimed by unrelated accessions %s, " "not resolvable by version/RefSeq preference, skipping\n" % (alias, sorted(accessions))) continue if discarded: sys.stderr.write( @@ -509,73 +543,88 @@ "allowed)") args = parser.parse_args() sys.stderr.write("fetching GenArk hub list ...\n") hubAccessions = fetchHubList(hubListUrl) sys.stderr.write(" %d hub accessions found\n" % len(hubAccessions)) sys.stderr.write("fetching genark assembly summary tables ...\n") genarkData = fetchGenarkData() sys.stderr.write(" %d accessions found\n" % len(genarkData)) sys.stderr.write("fetching existing hgcentraltest.asmAlias ...\n") aliasToBrowser, browserToAliases = fetchExistingAsmAlias() sys.stderr.write(" %d existing alias rows\n" % len(aliasToBrowser)) + sys.stderr.write("fetching hgcentraltest.dbDb names ...\n") + dbDbNames = fetchDbDbNames() + sys.stderr.write(" %d dbDb names found\n" % len(dbDbNames)) + reportExistingRowProblems(aliasToBrowser, hubAccessions, genarkData) # decide, once, which browser value every live accession's own alias # candidates should point at (itself, or its live paired GCF if it's # a superseded GCA) -- this is the single source of truth the rest of # the script builds on browserTargets = computeBrowserTargets(genarkData, hubAccessions) obsoleteGca = findObsoleteGcaBrowsers(browserTargets, browserToAliases) for browser in sorted(obsoleteGca): sys.stderr.write( "OBSOLETE: browser=%s (GenBank) replaced by browser=%s " "(RefSeq) -- used by alias(es) %s, dropping those rows and " "replacing them\n" % (browser, obsoleteGca[browser], sorted(browserToAliases[browser]))) - # existing rows on a retired GCA browser are dropped from the carried- - # forward set, and must not block their replacement rows from being + dbDbCollisions = findDbDbNameCollisions(aliasToBrowser, dbDbNames) + for alias in sorted(dbDbCollisions): + sys.stderr.write( + "DBDB COLLISION: existing asmAlias row alias=%s browser=%s -- " + "alias matches an existing dbDb.name, dropping this row\n" + % (alias, aliasToBrowser[alias])) + + # existing rows on a retired GCA browser, or whose alias collides with + # a real dbDb.name, are dropped from the carried-forward set; dropping + # the GCA rows here must not block their replacement rows from being # added pointing at the new GCF browser instead - survivors = {a: b for a, b in aliasToBrowser.items() if b not in obsoleteGca} + survivors = {a: b for a, b in aliasToBrowser.items() + if b not in obsoleteGca and a not in dbDbCollisions} newRows = buildNewRows(genarkData, hubAccessions, browserTargets, - survivors, browserToAliases) + survivors, browserToAliases, dbDbNames) # merge as a dict, not a concatenation: newRows normally only adds # aliases absent from survivors, but a same-assembly version-bump # update (see isSameAssemblyNewerVersion) intentionally reuses an # alias already in survivors to repoint it -- the dict assignment # is what makes that repointing take effect instead of duplicating # the alias key finalDict = dict(survivors) updated = sum(1 for alias, browser in newRows if alias in finalDict and finalDict[alias] != browser) for alias, browser in newRows: finalDict[alias] = browser beforeCaseCheck = len(finalDict) finalDict = resolveCaseInsensitiveConflicts(finalDict) caseDropped = beforeCaseCheck - len(finalDict) finalRows = sorted(finalDict.items()) outHandle, closeOut = mustOpen(args.outFile) for alias, browser in finalRows: outHandle.write("%s\t%s\n" % (alias, browser)) if closeOut: outHandle.close() sys.stderr.write("%d obsolete GCA browser(s) dropped\n" % len(obsoleteGca)) + sys.stderr.write("%d existing row(s) dropped for colliding with a dbDb name\n" + % len(dbDbCollisions)) sys.stderr.write("%d existing row(s) repointed to a newer version\n" % updated) sys.stderr.write("%d new asmAlias rows added\n" % (len(newRows) - updated)) sys.stderr.write("%d row(s) dropped for case-insensitive conflicts\n" % caseDropped) sys.stderr.write("%d total rows in replacement table (was %d)\n" % (len(finalRows), len(aliasToBrowser))) if __name__ == "__main__": main()