fece8651e5d986dc199aa4fa231b35863937ca4a braney Sat Aug 1 12:05:09 2026 -0700 hgConfCatalog: make --reconcile fit for a nightly run refs #37925 --reconcile printed its whole report whether or not anything had changed, so as cron mail it would have been seventeen lines of standing drift every night with the one new setting buried in it. It now prints only what somebody has to act on, which is a setting the tree reads that the catalog has not classified, and a boolean flag filed neither gate nor knob. The drift moves behind --verbose. Silent and exit 0 means nothing new. Adds the missing row for blatNewPageBanner, which was the one real finding sitting in the report, and files it as a gate with the argument for calling it a knob recorded. A scan that finds almost nothing now fails instead of reporting all clear, since pointed at an empty tree every check below it comes up empty, and KENT_SRC lets a nightly aim at a pristine checkout rather than a working tree where a stray .c file would show up as a finding. diff --git src/hg/utils/hgConfCatalog/hgConfCatalog.py src/hg/utils/hgConfCatalog/hgConfCatalog.py index bc4d6ab6272..982d0e5fa16 100755 --- src/hg/utils/hgConfCatalog/hgConfCatalog.py +++ src/hg/utils/hgConfCatalog/hgConfCatalog.py @@ -51,43 +51,55 @@ it along with every branch that reads it. QA_GRACE = 6 a gate still defaulting FALSE this long after it landed is not being gated, it is being forgotten. Either turn it on or delete the feature. Verification status. Rows carry verified=True only where the classification was confirmed by reading the code at the cited file:line. --check counts what is left, because a variable described as a permanent knob when it is really a forgotten gate defeats the purpose of the exercise. Usage: hgConfCatalog.py --json out.json hgConfCatalog.py --html out.html hgConfCatalog.py --check # counts and internal consistency hgConfCatalog.py --reconcile # diff the catalog against the tree + hgConfCatalog.py --reconcile --verbose # ... with the standing drift hgConfCatalog.py --sunset # what should be deleted, and when + +--reconcile is the mode meant for a nightly cron: it prints nothing and exits 0 +when the tree holds no setting the catalog has not classified, and exits 1 with +the list when somebody has added one. Nothing else belongs in a cron. --sunset +in particular reports every overdue gate on every run whether or not anything +changed, so as cron mail it is noise; it is a report a person reads at release +time. """ import argparse import html import json import os import sys # Sunset policy, in releases. See the module docstring. KEEP_AFTER_FLIP = 4 QA_GRACE = 6 +# Floor on how many settings a working scan finds; well under the real count. +# See the check in reconcile(). +MIN_TREE_NAMES = 150 + REDMINE = "https://redmine.gi.ucsc.edu/issues/%d" # --------------------------------------------------------------------------- # helpers # --------------------------------------------------------------------------- def h(name, kind, src, default=None, note=None, public=False, verified=False, role=None, sunset=None, env=None, deprecated=False, family=None, required=False, ticket=None, debatable=None): """One catalog entry. name the hg.conf setting name kind what sort of setting: path, table, profile, credential, url, email, limit, flag, branding, debug, internal, dead @@ -214,30 +226,42 @@ debatable="Reads as much like a knob as a gate: whether a machine's " "trackDb can be trusted is a property of that machine, " "not of a feature waiting to ship. If it is a knob it " "should stop appearing in the stalled list."), h("hgSession.shortLink", "flag", "hg/hgSession/hgSession.c:175", default="FALSE", role="gate", verified=True, note="Short session links. Added v374 and never flipped, which is " "the longest-running dark feature here."), h("showHubApiKey", "flag", "hg/hgHubConnect/hgHubConnect.c:571", default="FALSE", role="gate", verified=True, note="Expose the hub API key UI. Shares its call site with " "storeUserFiles, so the two should be retired together."), h("blatShowLocus", "flag", "hg/hgBlat/hgBlat.c:784", default="FALSE", role="gate", verified=True, note="Show the genomic locus alongside BLAT results."), + h("blatNewPageBanner", "flag", "hg/hgBlat/hgBlat.c:741", default="TRUE", + role="gate", verified=True, + note="The banner on the classic BLAT results page that offers a " + "one-click switch to the new sortable table display. Guards " + "the advertisement, not the feature: turning it off stops the " + "browser recommending the new page without releasing new CGIs, " + "and users who already opted in or follow a direct link still " + "get it. Goes away with the banner, once the new page is the " + "default.", + debatable="Born TRUE, so like sleepOn429 it never held anything " + "back, and a mirror might want to stop advertising a new " + "page permanently, which would make it a knob."), h("genarkLiftOver", "flag", "hg/lib/genark.c:413", default="FALSE", role="gate", verified=True, note="Offer liftOver between GenArk assemblies. Four call sites in " "genark.c and hdb.c."), h("showIgv", "flag", "hg/hgTracks/hgTracks.c:12116", default="FALSE", role="gate", verified=True, note="An IGV link in the track hamburger menus."), h("showLiftRequest", "flag", "hg/hgConvert/hgConvert.c:178", default="FALSE", role="gate", verified=True, ticket="37973", note="A link from the Convert page to liftRequest.html, the page " "that requests a new whole-genome alignment. The assembly " "list only offers targets that already have a chain from the " "source, so the Convert page is where a user finds out theirs " "is missing, but nothing in the tree linked to the request " "page. Off until the request pipeline is confirmed ready to " @@ -1474,144 +1498,171 @@ def wrap_text(s, width): """Wrap without pulling in textwrap for one caller.""" words = s.split() lines, cur = [], "" for w in words: if cur and len(cur) + 1 + len(w) > width: lines.append(cur) cur = w else: cur = (cur + " " + w).strip() if cur: lines.append(cur) return lines -def reconcile(cat, out=sys.stdout): +def reconcile(cat, out=sys.stdout, verbose=False): """Diff the catalog against what the tree actually reads. Three questions: 1. does the catalog list something the tree no longer reads 2. does the tree read something the catalog has not classified 3. is every boolean flag in the tree classified gate or knob The third is the one that keeps the sunset report honest: an unclassified flag is one nobody has decided the fate of. + + Only 2 and 3 count as problems, because only those two mean somebody added + a setting and did not write it down, which is the one thing a person has to + act on. The rest is drift that has been there for years (documentation for + a feature that was deleted, a read that moved into a helper), and printing + it on every run is what would turn a nightly cron into mail nobody reads. + So it goes out only under --verbose. Silent and 0 means nothing new. """ hh = load_harvester() if hh is None: print("harvestHgConf.py not importable; cannot reconcile", file=out) return 1 found, _ = hh.harvest() tree = hh.by_name(found) cataloged = by_name(cat) problems = 0 + # A scan that finds almost nothing is a broken scan, not a clean tree, and + # the difference matters: pointed at the wrong KENT_SRC or an empty clone, + # everything below would come up empty and report all clear forever. The + # real number is in the hundreds. + if len(tree) < MIN_TREE_NAMES: + print("only %d settings found under %s: expected at least %d, so the " + "scan is\nbroken rather than the tree being clean. Check " + "KENT_SRC." % (len(tree), hh.ROOT, MIN_TREE_NAMES), file=out) + return 1 + # Three kinds of name legitimately have no literal read to point at, and # all three have to be excused or the report is nothing but false alarms: # profile members (read through cfgOption2 with a runtime prefix), prefix # families (enumerated with cfgNamesWithPrefix), and the members of such a # family as spelled out in the example configs. suffixes = set(PROFILE_SUFFIXES["suffixes"]) prefix_scans = set(found["prefixScans"]) def is_profile_member(name): return "." in name and name.rsplit(".", 1)[1] in suffixes def is_prefix_family(name): if name in prefix_scans or name.endswith("."): return True return any(name.startswith(p) for p in prefix_scans) + if verbose: print("=== catalog vs tree ===", file=out) missing = sorted(n for n in tree if n not in cataloged and not n.startswith("{")) # {ident} names are carried in the catalog with the braces, so match those # separately. missing += sorted(n for n in tree if n.startswith("{") and n not in cataloged) if missing: problems += len(missing) print("\nread by the tree, not in the catalog (%d):" % len(missing), file=out) for n in missing: print(" %-40s %s" % (n, sorted(tree[n]["sites"])[0]), file=out) stale = sorted(n for n in cataloged if n not in tree and not is_profile_member(n) and not is_prefix_family(n)) - if stale: + if stale and verbose: print("\nin the catalog, no literal read found (%d):" % len(stale), file=out) print(" (a prefix family or a profile member is expected here; " "anything else\n is a catalog row whose read has gone away)", file=out) for n in stale: print(" %-40s %s" % (n, cataloged[n]["src"]), file=out) + if verbose: print("\n=== boolean flags: every one must be a gate or a knob ===", file=out) tree_flags = {n for n, d in tree.items() if d["boolean"]} classified = {v["name"] for v in gates(cat)} | {v["name"] for v in knobs(cat)} unclassified = sorted(tree_flags - classified) if unclassified: problems += len(unclassified) - print("unclassified (%d): nobody has decided whether these are " - "temporary" % len(unclassified), file=out) + print("\nboolean flag classified neither gate nor knob (%d): nobody " + "has decided\nwhether these are temporary:" % len(unclassified), + file=out) for n in unclassified: print(" %-40s %s" % (n, sorted(tree[n]["sites"])[0]), file=out) - else: + elif verbose: print("all %d boolean flags in the tree are classified" % len(tree_flags), file=out) phantom = sorted(classified - tree_flags) - if phantom: + if phantom and verbose: print("\nclassified as a flag but not read with " "cfgOptionBooleanDefault (%d):" % len(phantom), file=out) for n in phantom: print(" %-40s %s" % (n, cataloged[n]["src"]), file=out) + if verbose: print("\n=== product/ex.hg.conf ===", file=out) docs = hh.parse_doc_files() pub = {v["name"] for v in all_vars(cat) if v["public"]} # A prefix family counts as documented if any member of it is. def documented(name): if name in docs: return True return name.endswith(".") and any(d.startswith(name) for d in docs) undocumented = sorted(n for n in pub if not documented(n)) - if undocumented: + if undocumented and verbose: print("marked public in the catalog, absent from the example configs " "(%d):" % len(undocumented), file=out) print(" (these are settings a mirror operator would want and has no " "way to\n discover)", file=out) for n in undocumented: print(" %-40s %s" % (n, cataloged[n]["src"]), file=out) only_docs = sorted(n for n in docs if n not in cataloged and not is_profile_member(n) and not is_prefix_family(n)) - if only_docs: + if only_docs and verbose: print("\nin the example configs, nothing in the tree reads them (%d):" % len(only_docs), file=out) print(" (either the feature was deleted and the documentation was " "not, or the\n documented spelling is wrong, in which case a " "mirror that sets it is\n silently ignored)", file=out) for n in only_docs: print(" %-40s %s" % (n, docs[n]["sites"][0]), file=out) - print("\nproblems: %d" % problems, file=out) + if problems: + print("\nproblems: %d. Add a row to hgConfCatalog.py for each, in " + "the same commit\nas the code that reads it; see the " + "'Registering a new hg.conf variable'\nsection of the " + "edit-kent-code skill." % problems, file=out) + elif verbose: + print("\nproblems: 0", file=out) return problems # --------------------------------------------------------------------------- # HTML # --------------------------------------------------------------------------- CSS = """ body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif; margin: 0 auto; max-width: 1180px; padding: 1em 2em; color: #222; line-height: 1.45; } h1 { font-size: 1.6em; margin-bottom: 0.2em; } h2 { font-size: 1.2em; margin-top: 1.8em; border-bottom: 2px solid #4b6c9e; padding-bottom: 0.2em; color: #1a3a6b; } h3 { font-size: 1.05em; margin-top: 1.4em; color: #1a3a6b; } @@ -1819,50 +1870,53 @@ # --------------------------------------------------------------------------- # main # --------------------------------------------------------------------------- def main(): ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--json") ap.add_argument("--html") ap.add_argument("--check", action="store_true") ap.add_argument("--reconcile", action="store_true") ap.add_argument("--sunset", action="store_true") + ap.add_argument("--verbose", action="store_true", + help="with --reconcile, also print the standing drift " + "that needs no action") args = ap.parse_args() cat = build() ages = None sites = None if args.sunset or args.html or args.json: hh = load_harvester() if hh is None: print("harvestHgConf.py not importable", file=sys.stderr) return 1 found, _ = hh.harvest() ages = hh.harvest_ages(names=sorted(hh.by_name(found))) sites = {n: d["sites"] for n, d in hh.by_name(found).items()} rc = 0 if args.check: rc |= 1 if check(cat) else 0 if args.reconcile: - rc |= 1 if reconcile(cat) else 0 + rc |= 1 if reconcile(cat, verbose=args.verbose) else 0 if args.sunset: sunset_report(cat, ages, sites) if args.json: # Attribution rides along with each setting rather than in a table of # its own, so a consumer reading one entry sees where it came from. tmap = ticket_map(cat, ages) for v in all_vars(cat): if v["name"] in tmap: v["tickets"], v["ticketFrom"] = tmap[v["name"]] with open(args.json, "w") as f: json.dump(cat, f, indent=1) print("wrote %s" % args.json) if args.html: with open(args.html, "w") as f: f.write(render_html(cat, ages, sites))