3c814b674f49f9a30d4b8d227e0fe7061a18766a braney Tue Sep 1 09:21:28 2026 -0700 registryPages: correlate the trackDb settings docs with the cart, refs #37908 #37838 Two files in the tree say which track types a setting applies to, and they were written from different evidence. trackDbLibrary.shtml carries a hand-written types list per setting, which is what #37908 has been correcting. cartTrackVarCatalog files each cart variable under the config function that reads it and records the trackDb types that function serves, which came from reading hui.c and the per-type Ui functions. Where a trackDb setting and a cart variable are the same knob, the two are answering the same question, so they can be compared. 60 of the 261 documented settings have a runtime override. 46 of those pairs are comparable, 12 agree exactly, and 28 have a type the config code serves that the docs do not list. The output is a candidate list, not a verdict, and the page says so. Two known false-positive shapes are called out on it: a variable read by two config functions collects the types of both, and a pair joined by tdbDefault rather than by name is weaker evidence, so those are reported separately. Also factors the palette and the shared reset out of venn.css and index.css into tokens.css, since a third page now needs them. diff --git src/hg/utils/registryPages/registryPages.py src/hg/utils/registryPages/registryPages.py index 024479459e3..ee67bab65f5 100755 --- src/hg/utils/registryPages/registryPages.py +++ src/hg/utils/registryPages/registryPages.py @@ -29,59 +29,65 @@ that needs a person to read a call site. See KNOWN_SHARED in registryData.py. --audit runs sessionCartAudit, which needs the database and takes about fifteen seconds. Without it the pages leave out the one paragraph that talks about real saved sessions. """ import argparse import datetime import html import os import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import registryData as rd # noqa: E402 +import trackDbData as td # noqa: E402 NUM_WORD = {0: "no", 1: "one", 2: "two", 3: "three", 4: "four", 5: "five", 6: "six", 7: "seven", 8: "eight", 9: "nine", 10: "ten", 11: "eleven", 12: "twelve", 13: "thirteen", 14: "fourteen", 15: "fifteen", 16: "sixteen", 17: "seventeen", 18: "eighteen", 19: "nineteen", 20: "twenty"} def word(n): """Small numbers read better spelled out in a sentence.""" return NUM_WORD.get(n, "{:,}".format(n)) def esc(s): return html.escape(s, quote=False) def shortPath(path): """Write a path under the user's home as ~/... so provenance is readable.""" home = os.path.expanduser("~") return "~" + path[len(home):] if path.startswith(home + os.sep) else path def asset(name): """Read one of the stylesheet or script files that sits next to this one.""" with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), name)) as f: return f.read().rstrip("\n") +def style(name): + """The shared tokens plus one page's own rules, as a single stylesheet.""" + return asset("tokens.css") + "\n\n" + asset(name) + + # ============================================================ the Venn page == # Four congruent ellipses in the classic four-set arrangement: two rotated one # way, two the other, so all fifteen regions exist. Order matters, and it is # the order in rd.REG_ORDER: the two outer ellipses are the first and last. ELLIPSES = { "track": (350, 418, 360, 225, -140), "url": (450, 318, 360, 225, -140), "file": (544, 318, 360, 225, -40), "conf": (644, 418, 360, 225, -40), } VIEWBOX = (1000, 730) # Where each region's label goes. Found by rasterizing the four ellipses and @@ -258,31 +264,31 @@ return ("

No name is spelled the same way in two registries while meaning two " "different variables.

") lines = [] for name, where in coll: keys = [k for k in rd.REG_ORDER if k in where] lines.append("%s is %s" % (esc(name), " and ".join( "%s in the %s" % (", ".join("%s" % esc(v) for v in where[k]), esc(REG_PHRASE[k])) for k in keys))) return ("

%s %s in two registries and mean two different variables. They are " "counted as separate names above.

\n

%s.

" % (word(len(coll)).capitalize(), "spelling appears" if len(coll) == 1 else "spellings appear", ". ".join(lines))) -def vennPage(regs, counts, shared, coll, baseline, audit, indexName, today): +def vennPage(regs, counts, shared, coll, baseline, audit, indexName, corrName, today): """The whole Venn page.""" byKey = {r["key"]: r for r in regs} total = sum(counts.values()) rowTotal = sum(r["rows"] for r in regs) alone = sum(n for r, n in counts.items() if len(r) == 1) nShared = total - alone track = byKey["track"] prefixes = next(g for g in track["groups"] if g["title"].endswith("the track name")) exceptions = next((g for g in track["groups"] if g["title"].startswith("Exceptions")), {"rows": []}) plain = track["rows"] - len(prefixes["rows"]) - len(exceptions["rows"]) confShared = sorted(n for n, keys in shared.items() if "conf" in keys) if confShared == ["textSize"]: @@ -386,61 +392,63 @@

%(urlBaseline)d URL names and %(trackBaseline)d cart variable names are recorded in the baseline files as out of scope. Those files were accepted wholesale on the day they were written, so a name being in one is not evidence that anybody reviewed it.

Global cart variables, the ones scoped to no track, have no registry at all. textSize is one of them, which is why it enters the picture through the URL registry rather than a cart one.

%(auditPara)s """ % { - "css": asset("venn.css"), + "css": style("venn.css"), "today": today, "tree": esc(shortPath(rd.kentSrc())), "svg": "\n".join(" " + line for line in vennSvg(regs, counts).splitlines()), "rowTotal": "{:,}".format(rowTotal), "total": "{:,}".format(total), "alone": "{:,}".format(alone), "sharedWord": word(nShared), "plain": plain, "prefixes": len(prefixes["rows"]), "exceptions": len(exceptions["rows"]), "slivers": sliverList(regs, shared), "table": regionTable(regs, counts), "nRegions": word(len(REGION_LABEL)), "emptyWord": word(sum(1 for r in REGION_LABEL if not counts.get(r, 0))).capitalize(), "fullWord": word(sum(1 for r in REGION_LABEL if counts.get(r, 0) and len(r) > 1)), "collisions": collisionNote(coll), "confNames": len(byKey["conf"]["names"]), "confShared": word(len(confShared)), "confExcept": confExcept, "urlBaseline": baseline["url"], "trackBaseline": baseline["track"], "auditPara": auditPara, "indexName": esc(indexName), + "corrName": esc(corrName), "footRegs": " · ".join("%s #%s" % (r["tool"], r["ticket"]) for r in regs), } # =========================================================== the index page == def sortKey(name): """Sort a name by the part that varies, setting the shared scope aside. Every track-scoped name starts with the same seven characters, so sorting on the raw string files two hundred and seventy-odd names under punctuation and leaves the alphabet half empty. Dropping the scope puts .heightPer under H, where somebody looking for heightPer will go. """ key = name @@ -527,31 +535,31 @@ for i in buckets[letter]: name = names[i] dots = "".join('' % k for k in rd.REG_ORDER if k in merged[name]) chips.append('%s%s' % (i, html.escape(name.lower(), quote=True), html.escape(name), dots)) parts.append('

%s

' '%d
%s
' % (anchor, letter, len(buckets[letter]), "".join(chips))) markup = '\n%s' % ("".join(jump), "\n".join(parts)) scoped = sum(1 for n in names if n.startswith("") and n != "") return markup, data, len(buckets.get("#", [])), scoped -def indexPage(regs, shared, vennName, today): +def indexPage(regs, shared, vennName, corrName, today): """The whole name index, both views.""" import json tips = [] grouped, nav = groupedView(regs, shared, tips) azMarkup, azData, nSymbols, nScoped = alphabeticalView(regs) rowTotal = sum(r["rows"] for r in regs) nGroups = sum(len(r["groups"]) for r in regs) nNames = len(azData) return """Registry Name Index The %(nNames)s distinct names from all four catalogs, merged and sorted. Sorting sets aside the <track> scope that every track-scoped name shares, so <track>.heightPer is filed under H rather than under the punctuation with the other %(nScoped)d names that carry that scope. The # bucket at the end holds the %(nSymbols)s names that start with punctuation.

%(az)s """ % { - "css": asset("index.css"), + "css": style("index.css"), "js": asset("index.js"), "today": today, "tree": esc(shortPath(rd.kentSrc())), "nNames": "{:,}".format(nNames), "rowTotal": "{:,}".format(rowTotal), "nGroups": nGroups, "nScoped": nScoped, "nSymbols": word(nSymbols), "legend": "\n".join(' %s' % (r["key"], esc(REG_PHRASE[r["key"]])) for r in regs), "nav": nav, "grouped": grouped, "az": azMarkup, "tips": json.dumps(tips, separators=(",", ":")), "azdata": json.dumps(azData, separators=(",", ":")), "vennName": esc(vennName), + "corrName": esc(corrName), "footRegs": " · ".join("%s #%s" % (r["tool"], r["ticket"]) for r in regs), } +# ======================================================= the correlation page == + +def workList(pairs, how, weak=False): + """The candidate edits, one block per type the docs do not list.""" + groups = td.byMissingType(pairs, how) + if not groups: + return '

Nothing. The two files agree on every comparable pair.

' + out = ['
' % (" weak" if weak else "")] + for tdbType, ps in groups.items(): + bySetting = {} + for p in ps: + bySetting.setdefault(p["setting"], p) + out.append('
') + out.append('
%s' + '%d %s
' + % (esc(tdbType), len(bySetting), + "setting" if len(bySetting) == 1 else "settings")) + out.append('
') + for name in sorted(bySetting, key=str.lower): + p = bySetting[name] + out.append('
%s' + 'now: %s%s
' + % (esc(name), esc(", ".join(p["docTypes"])), + "" if p["how"] == "same name" + else " · joined through %s" % esc(p["var"]))) + out.append('
') + out.append('
') + out.append('
') + return "\n".join(out) + + +def pairTable(pairs): + """Every setting that has a runtime override, with both type lists.""" + rows = [] + for p in pairs: + if not p["comparable"]: + verdict = 'not comparable' + elif p["missing"]: + verdict = 'docs may be short' + elif p["extra"]: + verdict = 'cart has no UI for some' + else: + verdict = 'agree' + cartTypes = " ".join( + ('%s' % esc(t)) if t in p["missing"] else esc(t) + for t in p["cartTypes"]) or "—" + rows.append( + ' %s%s' + '%s%s%s' + '%s' + % (esc(p["setting"]), esc(p["var"]), esc(p["how"]), + esc(" ".join(p["docTypes"])) or "—", cartTypes, verdict)) + return (' \n' + ' \n' + ' \n' + ' ' + '' + '\n' + ' \n \n%s\n \n
%d settings with a runtime override
trackDb settingcart variablejoined bydocs saycart servesverdict
' + % (len(pairs), "\n".join(rows))) + + +def chipList(names): + return '
%s
' % "".join( + '%s' % html.escape(n) for n in names) + + +def correlatePage(data, vennName, indexName, today): + """The whole trackDb override map.""" + pairs = data["pairs"] + strong = [p for p in pairs if p["how"] == "same name"] + weak = [p for p in pairs if p["how"] == "tdbDefault"] + comparable = [p for p in pairs if p["comparable"]] + agree = [p for p in comparable if not p["missing"] and not p["extra"]] + gaps = td.byMissingType(pairs, "same name") + nGapSettings = len({p["setting"] for ps in gaps.values() for p in ps}) + + return """trackDb Override Map + + + + + +
+ +
+

UCSC Genome Browser · trackDb and the cart

+

Which trackDb settings a user can change

+

A trackDb setting names a default. For some settings the browser also offers the + reader a control, and the reader's choice lands in a cart variable. %(nPairs)d + of the %(nSettings)d documented settings work that way.

+

Both files also say which track types a setting applies to, and they were + written from different evidence: the documentation by hand, the cart catalog by reading the + config code. They disagree about %(nGapSettings)d settings.

+
+
+

trackDbSettings.json %(version)s

+

%(nSettings)d

+

Settings, generated from trackDbLibrary.shtml by make settings. + The types list on each one is what the hub wizard publishes and what + #37908 has been correcting.

+
+
+

cartTrackVarCatalog #37838

+

%(nCart)d

+

Track cart variables, each filed under the config function that reads it, with the + trackDb types that function serves. Built by reading hui.c and the per-type Ui functions, + not by reading the documentation.

+
+
+
+ +
+

Types the config code serves and the documentation does not list

+

Each block is one type and the settings whose types list omits it. + A block is usually one edit repeated across a family, which is how it is worth fixing. These are + candidates, not verdicts: every row still has to be read against the code. The pairs below are + joined by name, so the setting and the cart variable are one knob under two spellings.

+%(work)s +
+ +
+

Weaker candidates, joined through a default

+

Here the setting and the cart variable are spelled differently, and the join is + the cart catalog's own note that the variable takes its default from that setting. A variable + can be offered for a type whose default comes from somewhere else, so a type on this list is a + question rather than a candidate.

+%(weakWork)s +
+ +
+

Every setting with a runtime override

+

All %(nPairs)d pairs. A type in the cart column that the docs do not list is + marked. Not comparable means one side has nothing to say: a variable filed by + track name or in a wildcard family carries no type list, and a setting that applies to every + track cannot disagree about which types it covers.

+
+%(table)s +
+
+ +
+

The two sides that did not join

+

Most of both files has no counterpart in the other, and that is the expected + shape. A setting with no cart variable is one the browser reads and never offers to change. A + cart variable with no setting is a control with no trackDb default behind it.

+
+
+

%(nNoOverride)d settings a reader cannot change

+

They configure the track once, from trackDb, and the browser offers no control.

+
+
+

%(nNoSetting)d cart variables with no documented setting

+

Controls whose value has no trackDb default, plus every variable the cart catalog files + by track name or in a wildcard family.

+
+
+

+
+ The %(nNoOverride)d settings with no runtime override +%(noOverride)s +
+

+
+ The %(nNoSetting)d cart variables with no trackDb setting +%(noSetting)s +
+
+ +
+

How much to trust this

+
+
+

The cart catalog only knows types with a control

+

It files a variable under the config function that draws it, so it can only speak about + types that have one. A type in the documentation and not in the cart column is usually the + documentation being right about a type with no UI.

+
+
+

A variable can sit in two groups

+

One variable read by two config functions collects the types of both. + aggregate is filed under multiWig and under wig from the same source line, so + the wig types on its row are the catalog being generous rather than the documentation being + short.

+
+
+

Both files can be wrong together

+

%(nAgree)d comparable pairs agree exactly. That is two independent readings landing in + the same place, which is worth something, but neither was checked against a track that + actually renders.

+
+
+
+ + + +
+""" % { + "css": style("correlate.css"), + "today": today, + "tree": esc(shortPath(rd.kentSrc())), + "version": esc(data["version"]), + "nSettings": len(data["settings"]), + "nCart": len(data["cart"]), + "nPairs": len(pairs), + "nAgree": len(agree), + "nGapSettings": nGapSettings, + "work": workList(pairs, "same name"), + "weakWork": workList(pairs, "tdbDefault", weak=True), + "table": pairTable(pairs), + "nNoOverride": len(data["noOverride"]), + "nNoSetting": len(data["noSetting"]), + "noOverride": chipList(data["noOverride"]), + "noSetting": chipList(data["noSetting"]), + "vennName": esc(vennName), + "indexName": esc(indexName), + } + + # ====================================================================== cli == def main(): parser = argparse.ArgumentParser( description=__doc__.split("\n\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--outDir", help="write both pages into this directory") parser.add_argument("--venn", help="write the Venn page here") parser.add_argument("--index", help="write the name index here") + parser.add_argument("--correlate", help="write the trackDb override map here") parser.add_argument("--check", action="store_true", help="write nothing, just audit the shared names; for a cron") parser.add_argument("--audit", action="store_true", help="also run sessionCartAudit, which needs the database") parser.add_argument("--date", help="date to stamp on the pages (default today)") parser.add_argument("--vennLink", help="href each page uses to point at the Venn page " "(default its file name, which is right when both sit in one " "directory; give a full URL when they do not)") parser.add_argument("--indexLink", help="href each page uses to point at the name index") + parser.add_argument("--correlateLink", + help="href the other pages use to point at the trackDb override map") args = parser.parse_args() - if not (args.outDir or args.venn or args.index or args.check): - parser.error("nothing to do: give --outDir, --venn, --index or --check") + if not (args.outDir or args.venn or args.index or args.correlate or args.check): + parser.error("nothing to do: give --outDir, --venn, --index, --correlate or --check") regs = rd.loadRegistries() ok = rd.checkShared(regs) - if args.check and not (args.outDir or args.venn or args.index): + if args.check and not (args.outDir or args.venn or args.index or args.correlate): sys.exit(0 if ok else 1) counts = rd.regionCounts(regs) shared = rd.computeShared(regs) coll = rd.computeCollisions(regs) baseline = rd.baselineCounts() audit = rd.sessionAudit() if args.audit else None today = args.date or datetime.date.today().isoformat() vennPath = args.venn indexPath = args.index + corrPath = args.correlate if args.outDir: vennPath = vennPath or os.path.join(args.outDir, "registryVenn.html") indexPath = indexPath or os.path.join(args.outDir, "registryIndex.html") + corrPath = corrPath or os.path.join(args.outDir, "trackDbOverrides.html") vennName = args.vennLink or (os.path.basename(vennPath) if vennPath else "registryVenn.html") indexName = args.indexLink or (os.path.basename(indexPath) if indexPath else "registryIndex.html") + corrName = args.correlateLink or (os.path.basename(corrPath) if corrPath + else "trackDbOverrides.html") if vennPath: with open(vennPath, "w") as f: - f.write(vennPage(regs, counts, shared, coll, baseline, audit, indexName, today)) + f.write(vennPage(regs, counts, shared, coll, baseline, audit, indexName, corrName, + today)) print("wrote %s" % vennPath) if indexPath: with open(indexPath, "w") as f: - f.write(indexPage(regs, shared, vennName, today)) + f.write(indexPage(regs, shared, vennName, corrName, today)) print("wrote %s" % indexPath) + if corrPath: + with open(corrPath, "w") as f: + f.write(correlatePage(td.load(), vennName, indexName, today)) + print("wrote %s" % corrPath) sys.exit(0 if ok else 1) if __name__ == "__main__": main()