3c814b674f49f9a30d4b8d227e0fe7061a18766a braney Tue Sep 1 09:21:28 2026 -0700 registryPages: correlate the trackDb settings docs with the cart, refs #37908 #37838 Two files in the tree say which track types a setting applies to, and they were written from different evidence. trackDbLibrary.shtml carries a hand-written types list per setting, which is what #37908 has been correcting. cartTrackVarCatalog files each cart variable under the config function that reads it and records the trackDb types that function serves, which came from reading hui.c and the per-type Ui functions. Where a trackDb setting and a cart variable are the same knob, the two are answering the same question, so they can be compared. 60 of the 261 documented settings have a runtime override. 46 of those pairs are comparable, 12 agree exactly, and 28 have a type the config code serves that the docs do not list. The output is a candidate list, not a verdict, and the page says so. Two known false-positive shapes are called out on it: a variable read by two config functions collects the types of both, and a pair joined by tdbDefault rather than by name is weaker evidence, so those are reported separately. Also factors the palette and the shared reset out of venn.css and index.css into tokens.css, since a third page now needs them. diff --git src/hg/utils/registryPages/registryPages.py src/hg/utils/registryPages/registryPages.py index 024479459e3..ee67bab65f5 100755 --- src/hg/utils/registryPages/registryPages.py +++ src/hg/utils/registryPages/registryPages.py @@ -29,59 +29,65 @@ that needs a person to read a call site. See KNOWN_SHARED in registryData.py. --audit runs sessionCartAudit, which needs the database and takes about fifteen seconds. Without it the pages leave out the one paragraph that talks about real saved sessions. """ import argparse import datetime import html import os import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import registryData as rd # noqa: E402 +import trackDbData as td # noqa: E402 NUM_WORD = {0: "no", 1: "one", 2: "two", 3: "three", 4: "four", 5: "five", 6: "six", 7: "seven", 8: "eight", 9: "nine", 10: "ten", 11: "eleven", 12: "twelve", 13: "thirteen", 14: "fourteen", 15: "fifteen", 16: "sixteen", 17: "seventeen", 18: "eighteen", 19: "nineteen", 20: "twenty"} def word(n): """Small numbers read better spelled out in a sentence.""" return NUM_WORD.get(n, "{:,}".format(n)) def esc(s): return html.escape(s, quote=False) def shortPath(path): """Write a path under the user's home as ~/... so provenance is readable.""" home = os.path.expanduser("~") return "~" + path[len(home):] if path.startswith(home + os.sep) else path def asset(name): """Read one of the stylesheet or script files that sits next to this one.""" with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), name)) as f: return f.read().rstrip("\n") +def style(name): + """The shared tokens plus one page's own rules, as a single stylesheet.""" + return asset("tokens.css") + "\n\n" + asset(name) + + # ============================================================ the Venn page == # Four congruent ellipses in the classic four-set arrangement: two rotated one # way, two the other, so all fifteen regions exist. Order matters, and it is # the order in rd.REG_ORDER: the two outer ellipses are the first and last. ELLIPSES = { "track": (350, 418, 360, 225, -140), "url": (450, 318, 360, 225, -140), "file": (544, 318, 360, 225, -40), "conf": (644, 418, 360, 225, -40), } VIEWBOX = (1000, 730) # Where each region's label goes. Found by rasterizing the four ellipses and @@ -258,31 +264,31 @@ return ("<p>No name is spelled the same way in two registries while meaning two " "different variables.</p>") lines = [] for name, where in coll: keys = [k for k in rd.REG_ORDER if k in where] lines.append("<code>%s</code> is %s" % (esc(name), " and ".join( "%s in the %s" % (", ".join("<code>%s</code>" % esc(v) for v in where[k]), esc(REG_PHRASE[k])) for k in keys))) return ("<p>%s %s in two registries and mean two different variables. They are " "counted as separate names above.</p>\n <p>%s.</p>" % (word(len(coll)).capitalize(), "spelling appears" if len(coll) == 1 else "spellings appear", ". ".join(lines))) -def vennPage(regs, counts, shared, coll, baseline, audit, indexName, today): +def vennPage(regs, counts, shared, coll, baseline, audit, indexName, corrName, today): """The whole Venn page.""" byKey = {r["key"]: r for r in regs} total = sum(counts.values()) rowTotal = sum(r["rows"] for r in regs) alone = sum(n for r, n in counts.items() if len(r) == 1) nShared = total - alone track = byKey["track"] prefixes = next(g for g in track["groups"] if g["title"].endswith("the track name")) exceptions = next((g for g in track["groups"] if g["title"].startswith("Exceptions")), {"rows": []}) plain = track["rows"] - len(prefixes["rows"]) - len(exceptions["rows"]) confShared = sorted(n for n, keys in shared.items() if "conf" in keys) if confShared == ["textSize"]: @@ -386,61 +392,63 @@ <p>%(urlBaseline)d URL names and %(trackBaseline)d cart variable names are recorded in the baseline files as out of scope. Those files were accepted wholesale on the day they were written, so a name being in one is not evidence that anybody reviewed it.</p> <p>Global cart variables, the ones scoped to no track, have no registry at all. <code>textSize</code> is one of them, which is why it enters the picture through the URL registry rather than a cart one.</p>%(auditPara)s </div> </div> </section> <footer> Registries: %(footRegs)s<br> Counts taken %(today)s from each catalog's --json output. A track variable is matched on the cart variable it names, not on the suffix the catalog stores.<br> - Name by name, with descriptions: <a href="%(indexName)s">Registry Name Index</a>.<br> - Both pages are drawn by hg/utils/registryPages/registryPages.py. + Name by name, with descriptions: <a href="%(indexName)s">Registry Name Index</a> · + which of these a trackDb setting can preset: <a href="%(corrName)s">trackDb Override Map</a><br> + Drawn by hg/utils/registryPages/registryPages.py. </footer> </div> """ % { - "css": asset("venn.css"), + "css": style("venn.css"), "today": today, "tree": esc(shortPath(rd.kentSrc())), "svg": "\n".join(" " + line for line in vennSvg(regs, counts).splitlines()), "rowTotal": "{:,}".format(rowTotal), "total": "{:,}".format(total), "alone": "{:,}".format(alone), "sharedWord": word(nShared), "plain": plain, "prefixes": len(prefixes["rows"]), "exceptions": len(exceptions["rows"]), "slivers": sliverList(regs, shared), "table": regionTable(regs, counts), "nRegions": word(len(REGION_LABEL)), "emptyWord": word(sum(1 for r in REGION_LABEL if not counts.get(r, 0))).capitalize(), "fullWord": word(sum(1 for r in REGION_LABEL if counts.get(r, 0) and len(r) > 1)), "collisions": collisionNote(coll), "confNames": len(byKey["conf"]["names"]), "confShared": word(len(confShared)), "confExcept": confExcept, "urlBaseline": baseline["url"], "trackBaseline": baseline["track"], "auditPara": auditPara, "indexName": esc(indexName), + "corrName": esc(corrName), "footRegs": " · ".join("%s #%s" % (r["tool"], r["ticket"]) for r in regs), } # =========================================================== the index page == def sortKey(name): """Sort a name by the part that varies, setting the shared <track> scope aside. Every track-scoped name starts with the same seven characters, so sorting on the raw string files two hundred and seventy-odd names under punctuation and leaves the alphabet half empty. Dropping the scope puts <track>.heightPer under H, where somebody looking for heightPer will go. """ key = name @@ -527,31 +535,31 @@ for i in buckets[letter]: name = names[i] dots = "".join('<i class="sh %s" aria-hidden="true"></i>' % k for k in rd.REG_ORDER if k in merged[name]) chips.append('<span class="nm" tabindex="0" data-j="%d" data-n="%s">%s%s</span>' % (i, html.escape(name.lower(), quote=True), html.escape(name), dots)) parts.append('<div class="grp azgrp"><div class="ghead"><h3 id="%s">%s</h3>' '<span class="gn">%d</span></div><div class="chips">%s</div></div>' % (anchor, letter, len(buckets[letter]), "".join(chips))) markup = '<nav class="jump">%s</nav>\n%s' % ("".join(jump), "\n".join(parts)) scoped = sum(1 for n in names if n.startswith("<track>") and n != "<track>") return markup, data, len(buckets.get("#", [])), scoped -def indexPage(regs, shared, vennName, today): +def indexPage(regs, shared, vennName, corrName, today): """The whole name index, both views.""" import json tips = [] grouped, nav = groupedView(regs, shared, tips) azMarkup, azData, nSymbols, nScoped = alphabeticalView(regs) rowTotal = sum(r["rows"] for r in regs) nGroups = sum(len(r["groups"]) for r in regs) nNames = len(azData) return """<title>Registry Name Index</title> <link rel="preconnect" href="https://fonts.googleapis.com"> <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Archivo:wght@500;600;700\ &family=IBM+Plex+Mono:wght@400;500;600&family=IBM+Plex+Serif:ital,wght@0,400;0,600;1,400\ @@ -605,112 +613,355 @@ <p class="rblurb">The %(nNames)s distinct names from all four catalogs, merged and sorted. Sorting sets aside the <code class="il"><track></code> scope that every track-scoped name shares, so <code class="il"><track>.heightPer</code> is filed under H rather than under the punctuation with the other %(nScoped)d names that carry that scope. The <code class="il">#</code> bucket at the end holds the %(nSymbols)s names that start with punctuation.</p> </div> %(az)s </section> <footer> Catalogs: %(footRegs)s<br> Built %(today)s from each catalog's --json output against %(tree)s. Every description and source citation is the catalog's own text, unedited. Where a row carries no prose, the tooltip shows what the catalog does record: kind, default, and the file the tree reads it in.<br> - How the four registries overlap: <a href="%(vennName)s">Four Config Registries</a>.<br> - Both pages are drawn by hg/utils/registryPages/registryPages.py. + How the four registries overlap: <a href="%(vennName)s">Four Config Registries</a> · + which of these a trackDb setting can preset: <a href="%(corrName)s">trackDb Override Map</a><br> + Drawn by hg/utils/registryPages/registryPages.py. </footer> </div> <div id="tip" role="tooltip" aria-hidden="true"></div> <script id="tipdata" type="application/json">%(tips)s</script> <script id="azdata" type="application/json">%(azdata)s</script> <script> %(js)s </script> """ % { - "css": asset("index.css"), + "css": style("index.css"), "js": asset("index.js"), "today": today, "tree": esc(shortPath(rd.kentSrc())), "nNames": "{:,}".format(nNames), "rowTotal": "{:,}".format(rowTotal), "nGroups": nGroups, "nScoped": nScoped, "nSymbols": word(nSymbols), "legend": "\n".join(' <span><i class="sw %s"></i>%s</span>' % (r["key"], esc(REG_PHRASE[r["key"]])) for r in regs), "nav": nav, "grouped": grouped, "az": azMarkup, "tips": json.dumps(tips, separators=(",", ":")), "azdata": json.dumps(azData, separators=(",", ":")), "vennName": esc(vennName), + "corrName": esc(corrName), "footRegs": " · ".join("%s #%s" % (r["tool"], r["ticket"]) for r in regs), } +# ======================================================= the correlation page == + +def workList(pairs, how, weak=False): + """The candidate edits, one block per type the docs do not list.""" + groups = td.byMissingType(pairs, how) + if not groups: + return '<p class="sub">Nothing. The two files agree on every comparable pair.</p>' + out = ['<div class="work%s">' % (" weak" if weak else "")] + for tdbType, ps in groups.items(): + bySetting = {} + for p in ps: + bySetting.setdefault(p["setting"], p) + out.append(' <div class="wrow">') + out.append(' <div class="wtype"><code>%s</code>' + '<span class="wcount">%d %s</span></div>' + % (esc(tdbType), len(bySetting), + "setting" if len(bySetting) == 1 else "settings")) + out.append(' <div class="wsettings">') + for name in sorted(bySetting, key=str.lower): + p = bySetting[name] + out.append(' <div class="wsetting"><code>%s</code>' + '<span class="wnow">now: %s%s</span></div>' + % (esc(name), esc(", ".join(p["docTypes"])), + "" if p["how"] == "same name" + else " · joined through %s" % esc(p["var"]))) + out.append(' </div>') + out.append(' </div>') + out.append('</div>') + return "\n".join(out) + + +def pairTable(pairs): + """Every setting that has a runtime override, with both type lists.""" + rows = [] + for p in pairs: + if not p["comparable"]: + verdict = '<span class="pill none">not comparable</span>' + elif p["missing"]: + verdict = '<span class="pill gap">docs may be short</span>' + elif p["extra"]: + verdict = '<span class="pill none">cart has no UI for some</span>' + else: + verdict = '<span class="pill same">agree</span>' + cartTypes = " ".join( + ('<span class="add">%s</span>' % esc(t)) if t in p["missing"] else esc(t) + for t in p["cartTypes"]) or "—" + rows.append( + ' <tr><td class="mono">%s</td><td class="mono">%s</td>' + '<td class="how">%s</td><td class="types">%s</td><td class="types">%s</td>' + '<td>%s</td></tr>' + % (esc(p["setting"]), esc(p["var"]), esc(p["how"]), + esc(" ".join(p["docTypes"])) or "—", cartTypes, verdict)) + return (' <table>\n' + ' <caption>%d settings with a runtime override</caption>\n' + ' <thead>\n' + ' <tr><th scope="col">trackDb setting</th><th scope="col">cart variable</th>' + '<th scope="col">joined by</th><th scope="col">docs say</th>' + '<th scope="col">cart serves</th><th scope="col">verdict</th></tr>\n' + ' </thead>\n <tbody>\n%s\n </tbody>\n </table>' + % (len(pairs), "\n".join(rows))) + + +def chipList(names): + return '<div class="chips">%s</div>' % "".join( + '<span class="nm">%s</span>' % html.escape(n) for n in names) + + +def correlatePage(data, vennName, indexName, today): + """The whole trackDb override map.""" + pairs = data["pairs"] + strong = [p for p in pairs if p["how"] == "same name"] + weak = [p for p in pairs if p["how"] == "tdbDefault"] + comparable = [p for p in pairs if p["comparable"]] + agree = [p for p in comparable if not p["missing"] and not p["extra"]] + gaps = td.byMissingType(pairs, "same name") + nGapSettings = len({p["setting"] for ps in gaps.values() for p in ps}) + + return """<title>trackDb Override Map</title> +<link rel="preconnect" href="https://fonts.googleapis.com"> +<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> +<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Archivo:wght@500;600;700\ +&family=IBM+Plex+Mono:wght@400;500&family=IBM+Plex+Serif:ital,wght@0,400;0,600;1,400&display=swap"> +<style> +%(css)s +</style> + +<div class="wrap"> + +<header> + <p class="eyebrow">UCSC Genome Browser · trackDb and the cart</p> + <h1>Which trackDb settings a user can change</h1> + <p class="lede">A trackDb setting names a default. For some settings the browser also offers the + reader a control, and the reader's choice lands in a cart variable. <strong>%(nPairs)d</strong> + of the <strong>%(nSettings)d</strong> documented settings work that way.</p> + <p class="lede">Both files also say which track types a setting applies to, and they were + written from different evidence: the documentation by hand, the cart catalog by reading the + config code. They disagree about <strong>%(nGapSettings)d</strong> settings.</p> + <div class="sources"> + <div class="sourcecard tdb"> + <h3>trackDbSettings.json %(version)s</h3> + <p class="big">%(nSettings)d</p> + <p>Settings, generated from trackDbLibrary.shtml by <code class="t">make settings</code>. + The <code class="t">types</code> list on each one is what the hub wizard publishes and what + #37908 has been correcting.</p> + </div> + <div class="sourcecard cart"> + <h3>cartTrackVarCatalog #37838</h3> + <p class="big">%(nCart)d</p> + <p>Track cart variables, each filed under the config function that reads it, with the + trackDb types that function serves. Built by reading hui.c and the per-type Ui functions, + not by reading the documentation.</p> + </div> + </div> +</header> + +<section> + <h2>Types the config code serves and the documentation does not list</h2> + <p class="sub">Each block is one type and the settings whose <code>types</code> list omits it. + A block is usually one edit repeated across a family, which is how it is worth fixing. These are + candidates, not verdicts: every row still has to be read against the code. The pairs below are + joined by name, so the setting and the cart variable are one knob under two spellings.</p> +%(work)s +</section> + +<section> + <h2>Weaker candidates, joined through a default</h2> + <p class="sub">Here the setting and the cart variable are spelled differently, and the join is + the cart catalog's own note that the variable takes its default from that setting. A variable + can be offered for a type whose default comes from somewhere else, so a type on this list is a + question rather than a candidate.</p> +%(weakWork)s +</section> + +<section> + <h2>Every setting with a runtime override</h2> + <p class="sub">All %(nPairs)d pairs. A type in the cart column that the docs do not list is + marked. <strong>Not comparable</strong> means one side has nothing to say: a variable filed by + track name or in a wildcard family carries no type list, and a setting that applies to every + track cannot disagree about which types it covers.</p> + <div class="tablewrap"> +%(table)s + </div> +</section> + +<section> + <h2>The two sides that did not join</h2> + <p class="sub">Most of both files has no counterpart in the other, and that is the expected + shape. A setting with no cart variable is one the browser reads and never offers to change. A + cart variable with no setting is a control with no trackDb default behind it.</p> + <div class="notes"> + <div class="notecard"> + <h3>%(nNoOverride)d settings a reader cannot change</h3> + <p>They configure the track once, from trackDb, and the browser offers no control.</p> + </div> + <div class="notecard"> + <h3>%(nNoSetting)d cart variables with no documented setting</h3> + <p>Controls whose value has no trackDb default, plus every variable the cart catalog files + by track name or in a wildcard family.</p> + </div> + </div> + <p></p> + <details> + <summary>The %(nNoOverride)d settings with no runtime override</summary> +%(noOverride)s + </details> + <p></p> + <details> + <summary>The %(nNoSetting)d cart variables with no trackDb setting</summary> +%(noSetting)s + </details> +</section> + +<section> + <h2>How much to trust this</h2> + <div class="notes"> + <div class="notecard"> + <h3>The cart catalog only knows types with a control</h3> + <p>It files a variable under the config function that draws it, so it can only speak about + types that have one. A type in the documentation and not in the cart column is usually the + documentation being right about a type with no UI.</p> + </div> + <div class="notecard"> + <h3>A variable can sit in two groups</h3> + <p>One variable read by two config functions collects the types of both. + <code>aggregate</code> is filed under multiWig and under wig from the same source line, so + the wig types on its row are the catalog being generous rather than the documentation being + short.</p> + </div> + <div class="notecard"> + <h3>Both files can be wrong together</h3> + <p>%(nAgree)d comparable pairs agree exactly. That is two independent readings landing in + the same place, which is worth something, but neither was checked against a track that + actually renders.</p> + </div> + </div> +</section> + +<footer> + Sources: trackDbSettings.json %(version)s, generated from trackDbLibrary.shtml · + cartTrackVarCatalog #37838<br> + Built %(today)s against %(tree)s. Refs #37908 #37838.<br> + The four registries and how they overlap: <a href="%(vennName)s">Four Config Registries</a> + · every name with its description: <a href="%(indexName)s">Registry Name Index</a><br> + Drawn by hg/utils/registryPages/registryPages.py. +</footer> + +</div> +""" % { + "css": style("correlate.css"), + "today": today, + "tree": esc(shortPath(rd.kentSrc())), + "version": esc(data["version"]), + "nSettings": len(data["settings"]), + "nCart": len(data["cart"]), + "nPairs": len(pairs), + "nAgree": len(agree), + "nGapSettings": nGapSettings, + "work": workList(pairs, "same name"), + "weakWork": workList(pairs, "tdbDefault", weak=True), + "table": pairTable(pairs), + "nNoOverride": len(data["noOverride"]), + "nNoSetting": len(data["noSetting"]), + "noOverride": chipList(data["noOverride"]), + "noSetting": chipList(data["noSetting"]), + "vennName": esc(vennName), + "indexName": esc(indexName), + } + + # ====================================================================== cli == def main(): parser = argparse.ArgumentParser( description=__doc__.split("\n\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--outDir", help="write both pages into this directory") parser.add_argument("--venn", help="write the Venn page here") parser.add_argument("--index", help="write the name index here") + parser.add_argument("--correlate", help="write the trackDb override map here") parser.add_argument("--check", action="store_true", help="write nothing, just audit the shared names; for a cron") parser.add_argument("--audit", action="store_true", help="also run sessionCartAudit, which needs the database") parser.add_argument("--date", help="date to stamp on the pages (default today)") parser.add_argument("--vennLink", help="href each page uses to point at the Venn page " "(default its file name, which is right when both sit in one " "directory; give a full URL when they do not)") parser.add_argument("--indexLink", help="href each page uses to point at the name index") + parser.add_argument("--correlateLink", + help="href the other pages use to point at the trackDb override map") args = parser.parse_args() - if not (args.outDir or args.venn or args.index or args.check): - parser.error("nothing to do: give --outDir, --venn, --index or --check") + if not (args.outDir or args.venn or args.index or args.correlate or args.check): + parser.error("nothing to do: give --outDir, --venn, --index, --correlate or --check") regs = rd.loadRegistries() ok = rd.checkShared(regs) - if args.check and not (args.outDir or args.venn or args.index): + if args.check and not (args.outDir or args.venn or args.index or args.correlate): sys.exit(0 if ok else 1) counts = rd.regionCounts(regs) shared = rd.computeShared(regs) coll = rd.computeCollisions(regs) baseline = rd.baselineCounts() audit = rd.sessionAudit() if args.audit else None today = args.date or datetime.date.today().isoformat() vennPath = args.venn indexPath = args.index + corrPath = args.correlate if args.outDir: vennPath = vennPath or os.path.join(args.outDir, "registryVenn.html") indexPath = indexPath or os.path.join(args.outDir, "registryIndex.html") + corrPath = corrPath or os.path.join(args.outDir, "trackDbOverrides.html") vennName = args.vennLink or (os.path.basename(vennPath) if vennPath else "registryVenn.html") indexName = args.indexLink or (os.path.basename(indexPath) if indexPath else "registryIndex.html") + corrName = args.correlateLink or (os.path.basename(corrPath) if corrPath + else "trackDbOverrides.html") if vennPath: with open(vennPath, "w") as f: - f.write(vennPage(regs, counts, shared, coll, baseline, audit, indexName, today)) + f.write(vennPage(regs, counts, shared, coll, baseline, audit, indexName, corrName, + today)) print("wrote %s" % vennPath) if indexPath: with open(indexPath, "w") as f: - f.write(indexPage(regs, shared, vennName, today)) + f.write(indexPage(regs, shared, vennName, corrName, today)) print("wrote %s" % indexPath) + if corrPath: + with open(corrPath, "w") as f: + f.write(correlatePage(td.load(), vennName, indexName, today)) + print("wrote %s" % corrPath) sys.exit(0 if ok else 1) if __name__ == "__main__": main()