cab003c85820ef181c537ad402aad0044e4e3878 braney Tue Sep 1 09:06:56 2026 -0700 registryPages: draw two pages from all four configuration catalogs, refs #37838 #37923 #37925 #37623 Each of the four catalogs answers for its own surface and none of them knows the others exist. This reads all four and draws the two pages that need them at once: a Venn of the registries, and an index of every name they describe with its description on hover. The reason it is a program and not two files: every count on both pages is computed from the catalogs, so a page cannot drift from the tree. A hand-built version of the Venn had the track registry 12 names short and one region count wrong, and neither error was visible on the page. The one judgement is which names two registries describe as the same variable. That is matched on the cart variable a row names, not on the suffix the track catalog stores, because comparing bare names both misses the four track-scoped names and invents six overlaps that are not there. KNOWN_SHARED records the eleven confirmed by reading the call sites, and --check fails when the computed set no longer matches it, the same contract the sibling catalogs' --reconcile has. --check writes nothing and is the mode for a cron. --audit adds the saved session counts from sessionCartAudit and needs the database. diff --git src/hg/utils/registryPages/registryPages.py src/hg/utils/registryPages/registryPages.py new file mode 100755 index 00000000000..024479459e3 --- /dev/null +++ src/hg/utils/registryPages/registryPages.py @@ -0,0 +1,716 @@ +#!/usr/bin/env python3 +"""registryPages.py - draw two web pages from the four configuration catalogs. + +Refs #37838, #37923, #37925, #37623. The four catalogs each answer for one +part of the browser's configuration surface, and each one prints its own page. +This draws the two pages that need all four at once: + + registryVenn.html a four-set Venn of the registries, so the shape of the + whole surface is visible at once: which registry owns + what, and the handful of names more than one describes. + + registryIndex.html every name in all four catalogs, readable either in each + catalog's own groups or as one alphabetical list, with + the description on hover. + +Everything on both pages is computed from the catalogs. No count is typed in +here, so the pages cannot drift from the tree the way a hand-written summary +would. registryData.py does the reading and the matching; this file does the +drawing. + +Usage: + registryPages.py --outDir ~/public_html # both pages + registryPages.py --venn v.html --index i.html # or name them + registryPages.py --check # no output, just the audit + registryPages.py --outDir DIR --audit # add the saved-session count + +--check is the mode for a nightly cron. It writes nothing and fails when a name +starts or stops being shared between two registries, which is the one thing here +that needs a person to read a call site. See KNOWN_SHARED in registryData.py. + +--audit runs sessionCartAudit, which needs the database and takes about fifteen +seconds. Without it the pages leave out the one paragraph that talks about real +saved sessions. +""" + +import argparse +import datetime +import html +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import registryData as rd # noqa: E402 + + +NUM_WORD = {0: "no", 1: "one", 2: "two", 3: "three", 4: "four", 5: "five", 6: "six", + 7: "seven", 8: "eight", 9: "nine", 10: "ten", 11: "eleven", 12: "twelve", + 13: "thirteen", 14: "fourteen", 15: "fifteen", 16: "sixteen", + 17: "seventeen", 18: "eighteen", 19: "nineteen", 20: "twenty"} + + +def word(n): + """Small numbers read better spelled out in a sentence.""" + return NUM_WORD.get(n, "{:,}".format(n)) + + +def esc(s): + return html.escape(s, quote=False) + + +def shortPath(path): + """Write a path under the user's home as ~/... so provenance is readable.""" + home = os.path.expanduser("~") + return "~" + path[len(home):] if path.startswith(home + os.sep) else path + + +def asset(name): + """Read one of the stylesheet or script files that sits next to this one.""" + with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), name)) as f: + return f.read().rstrip("\n") + + +# ============================================================ the Venn page == + +# Four congruent ellipses in the classic four-set arrangement: two rotated one +# way, two the other, so all fifteen regions exist. Order matters, and it is +# the order in rd.REG_ORDER: the two outer ellipses are the first and last. +ELLIPSES = { + "track": (350, 418, 360, 225, -140), + "url": (450, 318, 360, 225, -140), + "file": (544, 318, 360, 225, -40), + "conf": (644, 418, 360, 225, -40), +} + +VIEWBOX = (1000, 730) + +# Where each region's label goes. Found by rasterizing the four ellipses and +# taking the deepest point of each region, then checked so that every corner of +# every text block lands inside the region it belongs to. Do not nudge these by +# eye: change the ellipses and these have to be found again. +# big the exclusive region of one registry: rule, count, name, ticket +# pair two registries: the count with a colored dot per registry, or a zero +# small three or four registries: a zero, since none of them has ever held a name +REGION_LABEL = { + ("track",): ("big", 150, 417), + ("url",): ("big", 338, 80), + ("file",): ("big", 655, 80), + ("conf",): ("big", 843, 417), + + ("track", "url"): ("pair", 257, 166), + ("url", "file"): ("pair", 495, 166), + ("track", "file"): ("pair", 290, 511), + ("url", "conf"): ("pair", 703, 511), + ("file", "conf"): ("pair", 737, 166), + ("track", "conf"): ("pair", 497, 614), + + ("track", "url", "file"): ("small", 400, 252), + ("url", "file", "conf"): ("small", 593, 252), + ("track", "file", "conf"): ("small", 401, 588), + ("track", "url", "conf"): ("small", 593, 588), + + ("track", "url", "file", "conf"): ("small", 497, 514), +} + +REG_SHORT = {"track": "CART / TRACK", "url": "URL PARAMS", + "file": "CART / FILE", "conf": "HG.CONF"} + +REG_PHRASE = {"track": "track cart variables", "url": "URL parameters", + "file": "file cart variables", "conf": "hg.conf settings"} + +# The same four names at the head of a sentence or a table cell. A plain +# .capitalize() would turn "URL parameters" into "Url parameters". +REG_ONLY = {"track": "Track cart variables only", "url": "URL parameters only", + "file": "File cart variables only", "conf": "hg.conf settings only"} + +REG_BRIEF = {"track": "Track cart", "url": "URL", "file": "File cart", "conf": "hg.conf"} + + +def vennSvg(regs, counts): + """The four-set diagram, with the count of every region drawn in it.""" + byKey = {r["key"]: r for r in regs} + out = ['' % esc(vennAria(regs, counts))] + + out.append(' ') + for key in rd.REG_ORDER: + cx, cy, rx, ry, rot = ELLIPSES[key] + reg = byKey[key] + out.append(' ' % (key, key)) + out.append(' %s (#%s): %d rows, %d distinct names' + % (esc(reg["title"]), reg["ticket"], reg["rows"], len(reg["names"]))) + out.append(' ') + + out.append(' ') + for region in sorted(REGION_LABEL, key=lambda r: (len(r), r)): + kind, x, y = REGION_LABEL[region] + n = counts.get(region, 0) + if kind == "big": + key = region[0] + out.append(' ' + % (x - 30, y - 39, key)) + out.append(' %d' + % (x, y, n)) + out.append(' %s' + % (x, y + 20, REG_SHORT[key])) + out.append(' #%s' + % (x, y + 38, byKey[key]["ticket"])) + elif kind == "pair" and n: + out.append(' %d' % (x, y, n)) + for i, key in enumerate(region): + out.append(' ' + % (x - 7 + 14 * i, y + 14, key)) + else: + out.append(' %d' % (x, y + 6, n)) + out.append(' ') + out.append('') + return "\n".join(out) + + +def vennAria(regs, counts): + """One sentence saying what the picture shows, for a reader who cannot see it.""" + only = ", ".join(str(counts.get((r["key"],), 0)) for r in regs) + pairs = sorted((n, r) for r, n in counts.items() if len(r) == 2 and n) + pairText = ", ".join(str(n) for n, _ in pairs) + names = [REG_PHRASE[r["key"]] for r in regs] + listed = ", ".join(names[:-1]) + " and " + names[-1] + return ("A four-ellipse Venn diagram of the browser's configuration registries: %s. " + "The exclusive regions hold %s names. %s pairwise regions hold %s. The other " + "regions, including every three-way and four-way region, are empty." + % (listed, only, word(len(pairs)).capitalize(), pairText)) + + +def sliverList(regs, shared): + """The shared names, one block per pair of registries, with the catalogs' own notes.""" + byKey = {r["key"]: r for r in regs} + note = {} + for reg in regs: + for group in reg["groups"]: + for r in group["rows"]: + if r["desc"]: + note.setdefault((reg["key"], r["name"]), r["desc"]) + + pairs = {} + for name, keys in shared.items(): + pairs.setdefault(keys, []).append(name) + + out = ['
'] + for keys in sorted(pairs, key=lambda k: (-len(pairs[k]), k)): + names = sorted(pairs[keys], key=lambda n: n.lower()) + out.append('
') + out.append('
') + out.append('
%s' + % "".join('' % k for k in keys)) + out.append(' %s
' + % esc(" + ".join(REG_PHRASE[k] for k in keys))) + out.append('
%d%s
' + % (len(names), "name" if len(names) == 1 else "names")) + out.append('
') + out.append(' ') + out.append('
') + out.append('
') + return "\n".join(out) + + +def regionTable(regs, counts): + """The same figure as a table, so identity is never carried by color alone.""" + total = sum(counts.values()) + rows = [] + order = sorted(REGION_LABEL, key=lambda r: (len(r), -counts.get(r, 0), r)) + for region in order: + n = counts.get(region, 0) + dots = "".join('' % (k if k in region else "off") + for k in rd.REG_ORDER) + if len(region) == 1: + label = REG_ONLY[region[0]] + elif len(region) == len(rd.REG_ORDER): + label = "All four" + else: + label = " + ".join(REG_BRIEF[k] for k in region) + rows.append(' %s%s' + '%d' + % ('' if n else ' class="z"', dots, esc(label), n)) + return (' \n' + ' \n' + ' \n' + ' ' + '\n' + ' \n' + ' \n%s\n \n' + '
Region counts, %s distinct names
RegistriesRegionNames
' % ("{:,}".format(total), "\n".join(rows))) + + +def collisionNote(coll): + """The names that match across registries without being the same variable.""" + if not coll: + return ("

No name is spelled the same way in two registries while meaning two " + "different variables.

") + lines = [] + for name, where in coll: + keys = [k for k in rd.REG_ORDER if k in where] + lines.append("%s is %s" % (esc(name), " and ".join( + "%s in the %s" % (", ".join("%s" % esc(v) for v in where[k]), + esc(REG_PHRASE[k])) for k in keys))) + return ("

%s %s in two registries and mean two different variables. They are " + "counted as separate names above.

\n

%s.

" + % (word(len(coll)).capitalize(), + "spelling appears" if len(coll) == 1 else "spellings appear", + ". ".join(lines))) + + +def vennPage(regs, counts, shared, coll, baseline, audit, indexName, today): + """The whole Venn page.""" + byKey = {r["key"]: r for r in regs} + total = sum(counts.values()) + rowTotal = sum(r["rows"] for r in regs) + alone = sum(n for r, n in counts.items() if len(r) == 1) + nShared = total - alone + + track = byKey["track"] + prefixes = next(g for g in track["groups"] if g["title"].endswith("the track name")) + exceptions = next((g for g in track["groups"] + if g["title"].startswith("Exceptions")), {"rows": []}) + plain = track["rows"] - len(prefixes["rows"]) - len(exceptions["rows"]) + + confShared = sorted(n for n, keys in shared.items() if "conf" in keys) + if confShared == ["textSize"]: + confExcept = ("textSize is the exception, and it is deliberate. The setting " + "names the site default and the cart holds the visitor's choice.") + elif confShared: + confExcept = ("The %s a visitor can also set: %s." + % ("one" if len(confShared) == 1 else "ones", + ", ".join("%s" % esc(n) for n in confShared))) + else: + confExcept = "Nothing a mirror admin sets in hg.conf can be set by a visitor." + + auditPara = "" + if audit: + auditPara = ("\n

The session audit reads {sessions:,} saved sessions and finds " + "{names:,} distinct variable names in them. {unknown:,} match nothing in " + "any catalog.

".format(**audit)) + + return """Four Config Registries + + + + + +
+ +
+

UCSC Genome Browser · configuration surface

+

Four registries, %(sharedWord)s shared names

+

Four catalogs in hg/utils/ describe what can be configured in the + browser: the settings in hg.conf, the parameters on a CGI URL, + the track-scoped cart variables, and the cart variables that hold a + file name. Together they hold %(rowTotal)s rows covering %(total)s distinct names. Only + %(sharedWord)s names appear in more than one registry, and no name appears in three.

+

A name is in a registry when that catalog gives it a + row. For the track catalog that means its %(plain)d variables, its %(prefixes)d track-name + prefixes, and the %(exceptions)d names spelled out in its exceptions list. Every name is listed, + with its description, in the Registry Name Index.

+

+ generated %(today)s + source: each catalog's --json + tree: %(tree)s +

+
+ +
+
+%(svg)s +
+
Every name in the four registries, placed by which registries hold it. A pair of + dots marks which two registries share a region. The four ellipses are nearly disjoint: + %(alone)s of the %(total)s names sit alone in one registry, %(sharedWord)s sit in a pair, and + every three-way and four-way region is empty.
+
+ +
+

The %(sharedWord)s shared names

+

Each of these is one variable that two registries both describe, not two + variables that happen to share a spelling. The text is each catalog's own.

+ +%(slivers)s +
+ +
+

All %(nRegions)s regions

+

The same figure as a table. Filled dots mark the registries that hold the names + in that region.

+
+%(table)s +
+
+ +
+

Reading the empty regions

+

%(emptyWord)s of the %(nRegions)s regions are empty, and the %(fullWord)s that + are not hold %(sharedWord)s names between them. Three things explain that, and one of them is + a gap.

+
+ +
+

Some names collide but are not shared

+ %(collisions)s +

A bare name is not an identity. Matching on the name alone would have missed the + track-scoped names, which one catalog spells <track>_sel and the other + spells _sel, and would have claimed those collisions instead.

+
+ +
+

hg.conf barely touches the rest

+

%(confNames)d settings, %(confShared)s of which a visitor can also set. That is the + shape you want: what a mirror admin configures and what a visitor configures are two + different sets.

+

%(confExcept)s

+
+ +
+

What sits outside all four

+

%(urlBaseline)d URL names and %(trackBaseline)d cart variable names are recorded in the + baseline files as out of scope. Those files were accepted wholesale on the day they were + written, so a name being in one is not evidence that anybody reviewed it.

+

Global cart variables, the ones scoped to no track, have no registry at all. + textSize is one of them, which is why it enters the picture through the URL + registry rather than a cart one.

%(auditPara)s +
+ +
+
+ +
+ Registries: %(footRegs)s
+ Counts taken %(today)s from each catalog's --json output. A track variable is matched on the + cart variable it names, not on the suffix the catalog stores.
+ Name by name, with descriptions: Registry Name Index.
+ Both pages are drawn by hg/utils/registryPages/registryPages.py. +
+ +
+""" % { + "css": asset("venn.css"), + "today": today, + "tree": esc(shortPath(rd.kentSrc())), + "svg": "\n".join(" " + line for line in vennSvg(regs, counts).splitlines()), + "rowTotal": "{:,}".format(rowTotal), + "total": "{:,}".format(total), + "alone": "{:,}".format(alone), + "sharedWord": word(nShared), + "plain": plain, + "prefixes": len(prefixes["rows"]), + "exceptions": len(exceptions["rows"]), + "slivers": sliverList(regs, shared), + "table": regionTable(regs, counts), + "nRegions": word(len(REGION_LABEL)), + "emptyWord": word(sum(1 for r in REGION_LABEL if not counts.get(r, 0))).capitalize(), + "fullWord": word(sum(1 for r in REGION_LABEL + if counts.get(r, 0) and len(r) > 1)), + "collisions": collisionNote(coll), + "confNames": len(byKey["conf"]["names"]), + "confShared": word(len(confShared)), + "confExcept": confExcept, + "urlBaseline": baseline["url"], + "trackBaseline": baseline["track"], + "auditPara": auditPara, + "indexName": esc(indexName), + "footRegs": " · ".join("%s #%s" % (r["tool"], r["ticket"]) for r in regs), + } + + +# =========================================================== the index page == + +def sortKey(name): + """Sort a name by the part that varies, setting the shared scope aside. + + Every track-scoped name starts with the same seven characters, so sorting on + the raw string files two hundred and seventy-odd names under punctuation and + leaves the alphabet half empty. Dropping the scope puts .heightPer + under H, where somebody looking for heightPer will go. + """ + key = name + for prefix in (".", "_", ""): + if key.startswith(prefix): + key = key[len(prefix):] + break + key = key.lstrip("_.<") + return (key.lower() or name.lower(), name.lower()) + + +def bucketOf(name): + """The letter a name is filed under, or # when it starts with punctuation.""" + first = sortKey(name)[0][:1] + return first.upper() if first.isalpha() else "#" + + +def groupedView(regs, shared, tips): + """One section per registry, in the catalog's own groups.""" + out, nav = [], [] + for reg in regs: + key = reg["key"] + nav.append('%s%d' + % (key, key, key, esc(reg["title"]), reg["rows"])) + body = [] + for group in reg["groups"]: + chips = [] + for r in group["rows"]: + i = len(tips) + tips.append([r["name"], r["desc"], r["meta"]]) + also = [k for k in shared.get(r["name"], ()) if k != key] + dots = "".join('' % k for k in also) + chips.append('%s%s' + % (" also-in" if also else "", i, + html.escape(r["name"].lower(), quote=True), + html.escape(r["name"]), dots)) + what = ('

%s

' % esc(group["what"])) if group["what"] else "" + body.append('

%s

' + '%d
%s
%s
' + % (esc(group["title"]), len(group["rows"]), what, "".join(chips))) + out.append('
\n' + '
\n' + '

%s · Redmine #%s

\n' + '

%s

\n' + '

%s

\n' + '

%d rows · %d ' + 'distinct names · %d groups

\n' + '
\n %s\n
' + % (key, key, key, esc(reg["tool"]), reg["ticket"], esc(reg["title"]), + esc(reg["blurb"]), reg["rows"], len(reg["names"]), len(reg["groups"]), + "\n ".join(body))) + return "\n".join(out), "".join(nav) + + +def alphabeticalView(regs): + """All four registries merged into one A to Z list. + + A name two registries hold appears once, and its entry carries a block per + registry. A name with several rows in one registry, because several track + types set their own default for it, keeps every row. + """ + merged = {} + for reg in regs: + for group in reg["groups"]: + for r in group["rows"]: + merged.setdefault(r["name"], {}).setdefault(reg["key"], []).append( + [r["desc"], r["meta"]]) + + names = sorted(merged, key=sortKey) + data = [[n, [[k, merged[n][k]] for k in rd.REG_ORDER if k in merged[n]]] for n in names] + + buckets = {} + for i, name in enumerate(names): + buckets.setdefault(bucketOf(name), []).append(i) + letters = sorted(b for b in buckets if b != "#") + if "#" in buckets: + letters.append("#") + + jump, parts = [], [] + for letter in letters: + anchor = "az-" + ("sym" if letter == "#" else letter) + jump.append('%s' % (anchor, letter)) + chips = [] + for i in buckets[letter]: + name = names[i] + dots = "".join('' % k + for k in rd.REG_ORDER if k in merged[name]) + chips.append('%s%s' + % (i, html.escape(name.lower(), quote=True), + html.escape(name), dots)) + parts.append('

%s

' + '%d
%s
' + % (anchor, letter, len(buckets[letter]), "".join(chips))) + markup = '\n%s' % ("".join(jump), "\n".join(parts)) + scoped = sum(1 for n in names if n.startswith("") and n != "") + return markup, data, len(buckets.get("#", [])), scoped + + +def indexPage(regs, shared, vennName, today): + """The whole name index, both views.""" + import json + tips = [] + grouped, nav = groupedView(regs, shared, tips) + azMarkup, azData, nSymbols, nScoped = alphabeticalView(regs) + + rowTotal = sum(r["rows"] for r in regs) + nGroups = sum(len(r["groups"]) for r in regs) + nNames = len(azData) + + return """Registry Name Index + + + + + +
+ +
+

UCSC Genome Browser · configuration surface

+

Registry Name Index

+

Every name in the four configuration catalogs. Hover a name, or tab to it, to + read its description and where the tree reads it. %(nNames)s distinct names, + held in %(rowTotal)s catalog rows.

+

Read it two ways. By group keeps each + catalog's own arrangement, one section per registry. A to Z merges all four + into one alphabetical list, where a name that two registries hold appears once and its tooltip + carries both descriptions.

+
+%(legend)s + in A to Z, a dot for every registry + that holds the name; by group, a dot when a second registry holds the same variable +
+
+ +
+
+
+ + +
+ + +
+ +
+ +

No name matches that.

+ +
+%(grouped)s +
+ +
+
+

all four registries · one list

+

A to Z

+

The %(nNames)s distinct names from all four catalogs, merged and sorted. + Sorting sets aside the <track> scope that every track-scoped + name shares, so <track>.heightPer is filed under H rather than + under the punctuation with the other %(nScoped)d names that carry that scope. The + # bucket at the end holds the %(nSymbols)s names that start with + punctuation.

+
+%(az)s +
+ +
+ Catalogs: %(footRegs)s
+ Built %(today)s from each catalog's --json output against %(tree)s. Every description and + source citation is the catalog's own text, unedited. Where a row carries no prose, the tooltip + shows what the catalog does record: kind, default, and the file the tree reads it in.
+ How the four registries overlap: Four Config Registries.
+ Both pages are drawn by hg/utils/registryPages/registryPages.py. +
+ +
+ + + + + + +""" % { + "css": asset("index.css"), + "js": asset("index.js"), + "today": today, + "tree": esc(shortPath(rd.kentSrc())), + "nNames": "{:,}".format(nNames), + "rowTotal": "{:,}".format(rowTotal), + "nGroups": nGroups, + "nScoped": nScoped, + "nSymbols": word(nSymbols), + "legend": "\n".join(' %s' + % (r["key"], esc(REG_PHRASE[r["key"]])) for r in regs), + "nav": nav, + "grouped": grouped, + "az": azMarkup, + "tips": json.dumps(tips, separators=(",", ":")), + "azdata": json.dumps(azData, separators=(",", ":")), + "vennName": esc(vennName), + "footRegs": " · ".join("%s #%s" % (r["tool"], r["ticket"]) for r in regs), + } + + +# ====================================================================== cli == + +def main(): + parser = argparse.ArgumentParser( + description=__doc__.split("\n\n")[0], + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--outDir", help="write both pages into this directory") + parser.add_argument("--venn", help="write the Venn page here") + parser.add_argument("--index", help="write the name index here") + parser.add_argument("--check", action="store_true", + help="write nothing, just audit the shared names; for a cron") + parser.add_argument("--audit", action="store_true", + help="also run sessionCartAudit, which needs the database") + parser.add_argument("--date", help="date to stamp on the pages (default today)") + parser.add_argument("--vennLink", + help="href each page uses to point at the Venn page " + "(default its file name, which is right when both sit in one " + "directory; give a full URL when they do not)") + parser.add_argument("--indexLink", help="href each page uses to point at the name index") + args = parser.parse_args() + + if not (args.outDir or args.venn or args.index or args.check): + parser.error("nothing to do: give --outDir, --venn, --index or --check") + + regs = rd.loadRegistries() + ok = rd.checkShared(regs) + + if args.check and not (args.outDir or args.venn or args.index): + sys.exit(0 if ok else 1) + + counts = rd.regionCounts(regs) + shared = rd.computeShared(regs) + coll = rd.computeCollisions(regs) + baseline = rd.baselineCounts() + audit = rd.sessionAudit() if args.audit else None + today = args.date or datetime.date.today().isoformat() + + vennPath = args.venn + indexPath = args.index + if args.outDir: + vennPath = vennPath or os.path.join(args.outDir, "registryVenn.html") + indexPath = indexPath or os.path.join(args.outDir, "registryIndex.html") + + vennName = args.vennLink or (os.path.basename(vennPath) if vennPath + else "registryVenn.html") + indexName = args.indexLink or (os.path.basename(indexPath) if indexPath + else "registryIndex.html") + + if vennPath: + with open(vennPath, "w") as f: + f.write(vennPage(regs, counts, shared, coll, baseline, audit, indexName, today)) + print("wrote %s" % vennPath) + if indexPath: + with open(indexPath, "w") as f: + f.write(indexPage(regs, shared, vennName, today)) + print("wrote %s" % indexPath) + + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main()