d2cc543ed2c0fc032bc0cf8527ecff098b075aef braney Sun Jul 26 17:24:31 2026 -0700 record the ticket that introduced each hg.conf variable refs #37925 Each setting now carries the Redmine ticket cited by the commit that added its read, or for a flag the commit that turned its default on. --sunset prints it per gate, --html links it, and harvestHgConf.py --tickets groups the settings by whether git can attribute them at all. Nothing looser is used: a commit that merely edits a line is not a commit about the setting on it. A second history walk dates the names that reach cfgOption through a macro, so hgConfAges.json is rebuilt and a refresh now costs about four minutes. No CGI behaviour changes. diff --git src/hg/utils/hgConfCatalog/hgConfCatalog.py src/hg/utils/hgConfCatalog/hgConfCatalog.py index 71556cb654c..127d00e9b59 100755 --- src/hg/utils/hgConfCatalog/hgConfCatalog.py +++ src/hg/utils/hgConfCatalog/hgConfCatalog.py @@ -64,30 +64,32 @@ hgConfCatalog.py --check # counts and internal consistency hgConfCatalog.py --reconcile # diff the catalog against the tree hgConfCatalog.py --sunset # what should be deleted, and when """ import argparse import html import json import os import sys # Sunset policy, in releases. See the module docstring. KEEP_AFTER_FLIP = 4 QA_GRACE = 6 +REDMINE = "https://redmine.gi.ucsc.edu/issues/%d" + # --------------------------------------------------------------------------- # helpers # --------------------------------------------------------------------------- def h(name, kind, src, default=None, note=None, public=False, verified=False, role=None, sunset=None, env=None, deprecated=False, family=None, required=False, ticket=None, debatable=None): """One catalog entry. name the hg.conf setting name kind what sort of setting: path, table, profile, credential, url, email, limit, flag, branding, debug, internal, dead src file:line where the tree reads it default compiled-in default if the read supplies one @@ -1277,87 +1279,103 @@ # --------------------------------------------------------------------------- # sunset report # --------------------------------------------------------------------------- def gate_lifecycle(cat, ages): """Join each gate with the version history. Returns a record per gate carrying what the tree knows (added, flipped, current default) and what the catalog decided (sunset). Everything the report says follows from this join, so a gate cannot be described as healthy just because nobody updated its entry. """ first = ages.get("first", {}) first_true = ages.get("firstTrue", {}) cur = ages.get("current") + hh = load_harvester() out = [] for v in gates(cat): name = v["name"] added = (first.get(name) or {}).get("version") flipped = (first_true.get(name) or {}).get("version") + tickets, ticket_from = ([], None) + if hh and hasattr(hh, "ticket_for"): + tickets, ticket_from = hh.ticket_for(name, ages) shipped = v.get("default") == "TRUE" # A flip date with a FALSE default now means the flip was reverted, so # the flag is back to gating and the flip date must not drive a # deadline. reverted = bool(flipped) and not shipped sunset = v.get("sunset") if sunset is None and shipped and flipped: sunset = flipped + KEEP_AFTER_FLIP out.append({ "name": name, "src": v["src"], "default": v.get("default"), "note": v.get("note"), "added": added, "flipped": flipped, "shipped": shipped, "reverted": reverted, "sunset": sunset, "current": cur, "age": (cur - added) if (cur and added) else None, + "tickets": tickets, "ticketFrom": ticket_from, }) return out def sunset_report(cat, ages, sites=None, out=sys.stdout): """What should be deleted, what has no deadline, what is stuck in QA.""" cur = ages.get("current") life = gate_lifecycle(cat, ages) print("current tree version: v%s" % cur, file=out) if ages.get("stale"): print("\nWARNING: the age cache was built at v%s and the tree is now " "at v%s.\nDeadlines below are still correct, but any flag added " "since v%s has no date\nand will show as 'age unknown' rather " "than being reported. Rebuild with\nharvestHgConf.py --age " "--refresh.\n" % (ages.get("cachedAt"), cur, ages.get("cachedAt")), file=out) undated = [g for g in life if g["added"] is None] if undated: print("\n%d gate(s) could not be dated from history, so no deadline " "applies to them:\n %s\n" % (len(undated), ", ".join(sorted(g["name"] for g in undated))), file=out) print("policy: keep a flag %d releases after its default flips TRUE; " "a gate\nstill defaulting FALSE after %d releases is stalled.\n" % (KEEP_AFTER_FLIP, QA_GRACE), file=out) + noticket = [g["name"] for g in life if not g["tickets"]] + if noticket: + print("The ticket column is the ticket cited by the commit that added " + "the flag, or\nby the commit that turned it on, marked (flip). " + "%d of %d gates have neither\nand are blank: %s\n" + % (len(noticket), len(life), ", ".join(sorted(noticket))), + file=out) def line(g): bits = [] if g["added"]: bits.append("added v%d" % g["added"]) if g["flipped"] and not g["reverted"]: bits.append("flipped v%d" % g["flipped"]) if g["reverted"]: bits.append("flip v%d reverted" % g["flipped"]) if g["sunset"]: bits.append("sunset v%d" % g["sunset"]) n = len((sites or {}).get(g["name"], [])) or None tail = "%d call site%s" % (n, "" if n == 1 else "s") if n else "" - return " %-26s %-46s %s" % (g["name"], ", ".join(bits), tail) + tik = ", ".join("#%d" % t for t in g["tickets"]) + if tik and g["ticketFrom"] == "flip": + tik += " (flip)" + return " %-26s %-46s %-16s %s" % (g["name"], ", ".join(bits), + tik, tail) overdue = sorted([g for g in life if g["sunset"] and cur and g["sunset"] <= cur], key=lambda g: g["sunset"]) print("OVERDUE (delete the flag and every branch that reads it): %d" % len(overdue), file=out) for g in overdue: print(line(g), file=out) due = sorted([g for g in life if g["sunset"] and cur and g["sunset"] > cur], key=lambda g: g["sunset"]) print("\nSCHEDULED (shipped, deadline not yet reached): %d" % len(due), file=out) for g in due: print(line(g), file=out) @@ -1614,186 +1632,240 @@ div.arguable { color: #6b4a00; background: #fdf6e3; border-left: 3px solid #d9a441; padding: 3px 7px; margin-top: 4px; font-size: 0.93em; } span.arguable { background: #fdf0c0; color: #6b5300; } div.box { background: #f6f8fb; border-left: 4px solid #4b6c9e; padding: 0.7em 1em; margin: 1em 0; } div.policy { background: #fff8ec; border-left: 4px solid #d9a441; padding: 0.7em 1em; margin: 1em 0; } ul.toc { columns: 3; list-style: none; padding-left: 0; font-size: 0.92em; } """ def esc(s): return html.escape(str(s), quote=False) -def var_rows(vs, life_by_name=None): +def ticket_map(cat, ages): + """name -> (tickets, kind) for every setting git can attribute. + + kind is "introduced" when the commit that added the read cited the ticket + and "flip" when only the commit that turned a flag on did. A setting + missing from this map has no ticket in either commit, which for anything + added before about v270 means the tree predates Redmine. + """ + hh = load_harvester() + if not (ages and hh and hasattr(hh, "ticket_for")): + return {} + out = {} + for v in all_vars(cat): + tickets, kind = hh.ticket_for(v["name"], ages) + if tickets: + out[v["name"]] = (tickets, kind) + return out + + +def ticket_links(rec): + tickets, kind = rec + links = " ".join('#%d' % (REDMINE % t, t) for t in tickets) + if kind == "flip": + return "turned on by %s" % links + return "introduced by %s" % links + + +def var_rows(vs, life_by_name=None, tickets=None): rows = [] for v in sorted(vs, key=lambda x: x["name"].lower()): tags = ['%s' % esc(v["kind"])] role = v.get("role") if role: tags.append('%s' % (role, role)) if v.get("required"): tags.append('required') if v.get("deprecated"): tags.append('retired') life = (life_by_name or {}).get(v["name"]) if life: cur = life.get("current") if life.get("sunset") and cur and life["sunset"] <= cur: tags.append('overdue v%d' % life["sunset"]) elif life.get("sunset"): tags.append('sunset v%d' % life["sunset"]) if (not life.get("shipped") and life.get("age") is not None and life["age"] > QA_GRACE): tags.append('stalled %d' % life["age"]) extra = "" if life and life.get("added"): extra = "added v%d" % life["added"] if life.get("flipped"): extra += ", flipped v%d" % life["flipped"] + tik = (tickets or {}).get(v["name"]) + if tik: + extra += (", " if extra else "") + ticket_links(tik) note = "" if v.get("note"): note = '
%s
' % esc(v["note"]) if v.get("debatable"): tags.append('gate or knob?') note += ('
Arguable: %s
' % esc(v["debatable"])) env = "" if v.get("env"): env = '
environment: %s
' \ % esc(v["env"]) default = esc(v.get("default") or "") rows.append( "%s%s%s" "%s%s" "%s%s" % (esc(v["name"]), note, env, " ".join(tags), default, esc(v["src"]), ("
%s
" % extra) if extra else "")) return "\n".join(rows) -def table_of(vs, life_by_name=None): +def table_of(vs, life_by_name=None, tickets=None): return ("" "\n%s\n
settingkinddefaultread at
" - % var_rows(vs, life_by_name)) + % var_rows(vs, life_by_name, tickets)) def render_html(cat, ages=None, sites=None): life_by_name = {} sunset_html = "" + tickets = ticket_map(cat, ages) if ages: life = gate_lifecycle(cat, ages) life_by_name = {g["name"]: g for g in life} cur = ages.get("current") overdue = [g for g in life if g["sunset"] and cur and g["sunset"] <= cur] stalled = [g for g in life if not g["shipped"] and g["age"] is not None and g["age"] > QA_GRACE] sunset_html = ( '
Sunset status at v%s. ' '%d shipped gates are past their removal deadline and %d have been ' 'sitting at a FALSE default for more than %d releases. ' 'Policy: keep a flag %d releases after its default flips TRUE, ' 'then delete it and every branch that reads it. ' 'hgConfCatalog.py --sunset prints the working list.' '
' % (cur, len(overdue), len(stalled), QA_GRACE, KEEP_AFTER_FLIP)) c = counts(cat) parts = ["

Genome Browser hg.conf variables

", "

%d settings the CGIs read from hg.conf, " "generated from hg/utils/hgConfCatalog/. " "%d are release gates and %d are permanent deployment knobs." "

" % (c["distinctNames"], c["gates"], c["knobs"]), '
%s
' % esc(cat["boundary"]), sunset_html] + if tickets: + intro = len([1 for _, k in tickets.values() if k == "introduced"]) + parts.append( + '
Where a setting came from. Each entry ' + 'below carries the Redmine ticket cited by the commit that added ' + 'the read, or for a flag the commit that turned its default on, ' + 'marked there as "turned on by". %d of %d settings are attributed ' + 'that way. The rest are blank on purpose: no commit in the chain ' + 'names a ticket, and for anything added before about v270 the tree ' + 'predates our use of Redmine, so there is nothing to find. Nothing ' + 'here is inferred from a later commit that merely edited the line, ' + 'which would have credited half the file to whatever refactor ' + 'touched it last. harvestHgConf.py --tickets lists ' + 'the unattributed settings, separating the ones old enough to be ' + 'hopeless from the ones somebody could still fill in.
' + % (intro, c["distinctNames"])) + parts.append("

Contents

") parts.append("

How a setting is read

") parts.append("") for fn, what in sorted(ACCESSORS.items()): parts.append("" % (esc(fn), esc(what))) parts.append("
accessorbehaviour
%s%s
") ps = cat["profileSuffixes"] parts.append("

Database profile suffixes

") parts.append("

%s

" % esc(ps["what"])) parts.append("

Suffixes: %s

" % ", ".join("%s" % esc(s) for s in ps["suffixes"])) parts.append("

Profiles in use: %s

" % ", ".join("%s." % esc(s) for s in ps["knownProfiles"])) for sec in cat["sections"]: parts.append("

%s

" % (esc(sec["title"].replace(" ", "-")), esc(sec["title"]))) parts.append("

%s

" % esc(sec["what"])) - parts.append(table_of(sec["vars"], life_by_name)) + parts.append(table_of(sec["vars"], life_by_name, tickets)) return ("\n" "hg.conf variables" "\n%s\n\n" % (CSS, "\n".join(parts))) # --------------------------------------------------------------------------- # main # --------------------------------------------------------------------------- def main(): ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--json") ap.add_argument("--html") ap.add_argument("--check", action="store_true") ap.add_argument("--reconcile", action="store_true") ap.add_argument("--sunset", action="store_true") args = ap.parse_args() cat = build() ages = None sites = None - if args.sunset or args.html: + if args.sunset or args.html or args.json: hh = load_harvester() if hh is None: print("harvestHgConf.py not importable", file=sys.stderr) return 1 - ages = hh.harvest_ages() found, _ = hh.harvest() + ages = hh.harvest_ages(names=sorted(hh.by_name(found))) sites = {n: d["sites"] for n, d in hh.by_name(found).items()} rc = 0 if args.check: rc |= 1 if check(cat) else 0 if args.reconcile: rc |= 1 if reconcile(cat) else 0 if args.sunset: sunset_report(cat, ages, sites) if args.json: + # Attribution rides along with each setting rather than in a table of + # its own, so a consumer reading one entry sees where it came from. + tmap = ticket_map(cat, ages) + for v in all_vars(cat): + if v["name"] in tmap: + v["tickets"], v["ticketFrom"] = tmap[v["name"]] with open(args.json, "w") as f: json.dump(cat, f, indent=1) print("wrote %s" % args.json) if args.html: with open(args.html, "w") as f: f.write(render_html(cat, ages, sites)) print("wrote %s" % args.html) if not any([args.check, args.reconcile, args.sunset, args.json, args.html]): for k, v in sorted(counts(cat).items()): print("%-16s %s" % (k, v)) return rc if __name__ == "__main__":