b1a4f0619269b4c813036fc592c97471de310515 braney Sun Sep 20 17:53:28 2026 -0700 registry: hold every v504 ticket, including the ones a unit test is wrong for The table held only the tickets that want a unit test, 37 of the release's 66. That cannot answer "how much of v504 is tested", because the 29 it left out are indistinguishable from tickets nobody has looked at: both are absent. So all 66 are here now, and the 29 carry a new why, "page" -- what changed is what a page says or does, so the docent suite is the right test and no unit test is wanted. Each says what changed and where the fix is, so the claim is checkable rather than asserted; "this one does not need a unit test" was a judgement living in one head, and now it is a row somebody can argue with. A page row may not name a unit test, which check enforces. `needed` leaves them out, since they are not waiting for anything. `unwatched` does not: a ticket with no test of any kind is worth seeing whichever kind it should have had, and that list is 19 now rather than 4 -- the four blocked ones plus fifteen that nothing watches and nobody had recorded. refs #38391 diff --git src/utils/testRegistry/testRegistry src/utils/testRegistry/testRegistry index 5cbe8ed139f..30fe79c42dc 100755 --- src/utils/testRegistry/testRegistry +++ src/utils/testRegistry/testRegistry @@ -70,31 +70,41 @@ # compatible on purpose, so one reader can compare a unit row with a docent one. # "unrecorded" is not in proof.js; it is this file's way of saying nobody has # written down what was seen. LEVELS = [ "unrecorded", "assertion-only", "sandbox-ab", "release-ab", "caught-regression", ] # Why a unit test, and not a browser test. registry.tsv's header describes each # one. The vocabulary is short on purpose: a fourth reason should be argued for # on the ticket before it is added here, because every one of these is a claim # about what browser tests cannot do. -WHYS = ["invisible", "perf", "library"] +UNIT_WHYS = ["invisible", "perf", "library"] + +# The opposite claim: what changed is what a page says or does, so the docent +# suite is the right test and no unit test is wanted. These rows are here so the +# judgement is written down and can be argued with, rather than living in +# somebody's head as "that one does not need a unit test". `needed` leaves them +# out; `unwatched` does not, because a ticket with no test of any kind is worth +# seeing whichever kind it should have had. +NO_UNIT_WHY = "page" + +WHYS = UNIT_WHYS + [NO_UNIT_WHY] # Where the browser-side suite keeps its scripts, refs #38252. Named here so the # docent column is checked for rot the same way the test column is. DOCENT_DIR = "hg/utils/docent/tests/regress" HERE = os.path.dirname(os.path.abspath(__file__)) REGISTRY = os.path.join(HERE, "registry.tsv") # utils/testRegistry/ -> src/. Every path in the table is written from src, # because that is how the tree is talked about everywhere else. KENT_SRC = os.path.normpath(os.path.join(HERE, "..", "..")) class Row: """One line of registry.tsv.""" @@ -160,30 +170,32 @@ "not %s" % (row.ticket, row.docent)) elif not os.path.exists(os.path.join(KENT_SRC, DOCENT_DIR, row.docent)): bad(row, "%s/%s is not in the tree" % (DOCENT_DIR, row.docent)) if row.why not in WHYS: bad(row, "why '%s' is not one of %s" % (row.why, ", ".join(WHYS))) if not row.note.strip(): bad(row, "no note, so nobody can tell what the test holds down") if row.test == "-": # A ticket waiting for a test. There is no file to check, and any # evidence level would be a claim about a test that is not there. if row.evidence != "-": bad(row, "no test, so the evidence column must be -") continue + if row.why == NO_UNIT_WHY: + bad(row, "why is '%s', so this row cannot name a unit test" % NO_UNIT_WHY) if row.evidence not in LEVELS: bad(row, "evidence '%s' is not one of %s" % (row.evidence, ", ".join(LEVELS))) key = (row.ticket, row.test) if key in seen: bad(row, "already on line %d" % seen[key]) seen[key] = row.lineNo if not os.path.exists(row.path()): # The rot this file exists to catch. bad(row, "%s is not in the tree" % row.fileName()) for ticket, group in byTicketMap(rows).items(): if len(group) > 1 and any(r.test == "-" for r in group): @@ -217,46 +229,51 @@ for line in problems: print(line) return len(problems) def byTicketMap(rows): """Rows grouped by ticket.""" out = {} for row in rows: out.setdefault(row.ticket, []).append(row) return out def summary(rows): covered = {r.ticket for r in rows if r.test != "-"} - waiting = {r.ticket for r in rows if r.test == "-"} + waiting = {r.ticket for r in rows + if r.test == "-" and r.why in UNIT_WHYS} tests = {r.fileName() for r in rows if r.test != "-"} print("%d tickets covered by %d tests, %d waiting for one" % (len(covered), len(tests), len(waiting))) releases = {} for row in rows: releases.setdefault(row.release, set()).add(row.ticket) for release in sorted(releases, key=lambda v: (v == "-", v)): print(" v%-6s %3d tickets" % (release, len(releases[release]))) + noUnit = {r.ticket for r in rows if r.why == NO_UNIT_WHY} + if noUnit: + print(" %d more want no unit test at all: a page test is the right one" + % len(noUnit)) watched = {r.ticket for r in rows if r.docent != "-"} nothing = {r.ticket for r in rows if r.test == "-" and r.docent == "-"} print("%d also have a docent script, %d have no test of any kind" % (len(watched), len(nothing))) print("why a unit test:") - for why in WHYS: + for why in UNIT_WHYS: n = len({r.ticket for r in rows if r.why == why}) if n: print(" %-10s %3d tickets" % (why, n)) print("evidence, weakest first:") for level in LEVELS: n = len([r for r in rows if r.evidence == level]) if n: print(" %-18s %3d rows" % (level, n)) def show(rows): for row in rows: print("%-6s v%-5s %-42s %-26s %-10s %-15s %s" % (row.ticket, row.release, row.test, row.docent, row.why, row.evidence, row.note)) @@ -273,59 +290,60 @@ return 1 for row in hits: if row.test == "-": print("#%s v%s no unit test yet (%s) -- %s" % (row.ticket, row.release, row.why, row.note)) else: print("#%s v%s %s (%s, %s) -- %s" % (row.ticket, row.release, row.test, row.why, row.evidence, row.note)) if row.docent != "-": print(" docent: %s/%s" % (DOCENT_DIR, row.docent)) return 0 def needed(rows): - hits = [r for r in rows if r.test == "-"] + hits = [r for r in rows if r.test == "-" and r.why in UNIT_WHYS] if not hits: print("every ticket in the registry has a test") return 0 - for why in WHYS: + for why in UNIT_WHYS: group = [r for r in hits if r.why == why] if not group: continue print("\n%s" % why) for row in group: print(" #%s v%s %s%s" % (row.ticket, row.release, "" if row.docent == "-" else "[%s] " % row.docent, row.note)) print("\n%d tickets waiting for a unit test" % len(hits)) return 0 def unwatched(rows): """The tickets with neither a unit test nor a docent script. This is the queue: nothing anywhere would go red if one of these came back.""" hits = [r for r in rows if r.test == "-" and r.docent == "-"] if not hits: print("every ticket in the registry has a test of some kind") return 0 for why in WHYS: group = [r for r in hits if r.why == why] if not group: continue - print("\n%s" % why) + print("\n%s%s" % (why, " (a page test is the right one, not a unit test)" + if why == NO_UNIT_WHY else "")) for row in group: print(" #%s v%s %s" % (row.ticket, row.release, row.note)) print("\n%d tickets with nothing watching them" % len(hits)) return 0 def byWhy(rows, why): hits = [r for r in rows if r.why == why] if not hits: print("nothing in the registry is marked '%s'" % why) return 1 show(hits) return 0