4697bbddd881c72cccb85b9ff0aacd769396b9d0 braney Thu Sep 10 07:34:35 2026 -0700 docent: record what evidence each regression test has, and count it A regression test written after the fix asserts the right answer, but nobody has watched it fail for the reason it exists, and a loose assertion in that state is indistinguishable from no test at all. Four of the 37 scripts here have actually been watched to flip. That was recorded only as prose in each script's header, so answering "how many of these are real regression tests" meant a grep and a read, and the number could not be quoted. Every script now carries a top-level `proof:` key, one quoted line per piece of evidence, `<level> <YYYY-MM-DD> -- <what was seen>`. docent.js reads only the keys it names off the parsed document, so this costs a run nothing. tests/proof.js reads them and tallies, wired up as `make proof` in the shared docentTest.mk. It exits 1 on a malformed line, an unknown level, or a line left unquoted -- that last one because nearly every note names a ticket and a bare # in an unquoted YAML scalar silently truncates the sentence at the ticket number, which is how the first pass of this change lost half its text. The levels, weakest first: assertion-only, xfail, sandbox-ab, server-flip, caught-regression. Today that reads 31 / 2 / 0 / 3 / 1. nightly.sh now records the flips it finds. An xfail that PASSES is the best evidence this suite produces -- the same server, the same fixtures, the same script, one real build apart -- and until now it arrived as a red mail and was thrown away with the log 60 days later. It is appended to /hive/users/braney/docentNightly/flips.log, one line per script ever, outside the checkout because --update resets the tree. The mail says what to do with it. The three flips that already happened (rm38272 2026-09-06, rm36212 2026-09-09, rm38310 2026-09-10) were recovered from the old logs and seeded there by hand. Full suite run after the change: 37 scripts, all ok. refs #38252 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> diff --git src/hg/utils/docent/tests/docentTest.mk src/hg/utils/docent/tests/docentTest.mk index 6a7b08adf14..b74d6158b18 100644 --- src/hg/utils/docent/tests/docentTest.mk +++ src/hg/utils/docent/tests/docentTest.mk @@ -1,127 +1,140 @@ # Shared rules for a directory of Docent tests. Included by tests/makefile and by # tests/regress/makefile, so both directories run the same code rather than a copy of it. # # An including makefile sets, before the include: # DOCENT path to docent.js from THIS directory (required) # PREFLIGHT path to preflight.js from THIS directory (required) +# PROOF path to proof.js from THIS directory (default: beside PREFLIGHT) # PARITY which script `make parity` runs (default: the first one found) # # Everything else -- which scripts are tests, which have derive baselines -- comes from # what is on disk here, so a new *.docent.yaml is picked up with no edit. ifndef DOCENT $(error include docentTest.mk only after setting DOCENT, e.g. DOCENT = ../docent.js) endif ifndef PREFLIGHT $(error include docentTest.mk only after setting PREFLIGHT, e.g. PREFLIGHT = ./preflight.js) endif PW_DIR ?= /hive/groups/browser/uiTest/pw PW_ENV ?= PLAYWRIGHT_BROWSERS_PATH=$(PW_DIR)/browsers NODE_PATH=$(PW_DIR)/node_modules T ?= # `make parity` needs one script that is expected to PASS, so an .xfail one is no use as # the default. An including makefile can name a better one. PASSING := $(filter-out %.xfail,$(patsubst %.docent.yaml,%,$(wildcard *.docent.yaml))) PARITY ?= $(firstword $(PASSING)) TESTS := $(if $(T),$(addsuffix .docent.yaml,$(T)),$(wildcard *.docent.yaml)) +PROOF ?= $(dir $(PREFLIGHT))proof.js -.PHONY: test parity clean preflight +.PHONY: test parity clean preflight proof # The fixtures the scripts here name but do not contain: saved sessions, hub URLs, the # server itself. No browser, so this is seconds, and it is what separates "the fixtures # went away" from "a bug came back" -- which are the same red without it. Run it before # the suite, and on its own as often as you like. preflight: @$(PW_ENV) node $(PREFLIGHT) . +# What evidence each script has that it would catch its bug, and the tally. No browser +# and no network, so it costs nothing to run and the number can go straight into a commit +# message or a ticket. It is a separate target and not part of `make test` on purpose: a +# script with no proof is not a failure, it is a script whose evidence has not been +# collected yet, and the two must not arrive as the same red. +# +# It DOES fail on a malformed or unknown proof line, because a vocabulary nobody enforces +# turns into free text and free text cannot be counted. +proof: + @$(PW_ENV) node $(PROOF) . $(T) + test: @if [ -z "$(strip $(TESTS))" ]; then \ echo "no *.docent.yaml here -- nothing was tested"; exit 1; fi @fail=0; \ for f in $(TESTS); do \ b=$${f%.docent.yaml}; want=0; \ case $$b in *.xfail) want=1;; esac; \ if [ $$want = 1 ]; then printf '=== %s (expected to fail)\n' "$$b"; \ else printf '=== %s\n' "$$b"; fi; \ $(PW_ENV) node $(DOCENT) $$f > $$b.log 2>&1; got=$$?; \ if [ $$got -ne 0 ] && [ $$want -eq 0 ]; then \ echo " FAILED -- run said:"; sed 's/^/ /' $$b.log; fail=1; \ elif [ $$got -eq 0 ] && [ $$want -eq 1 ]; then \ echo " FAILED -- this was supposed to fail, and it passed"; fail=1; \ else echo " ok"; fi; \ done; \ if [ $$fail -eq 0 ]; then echo "docent tests passed"; else echo "docent tests FAILED"; exit 1; fi # Two invariants that need the same script run more than once, so they cannot be # written as a script of their own: # FAST parity -- FAST drops the dwells, the cursor animation and the recording. # It must not change what the page ends up showing. # rerun stability -- a second run in the same directory must reach the same state. # Cart bleed between runs would show up here and nowhere else. parity: @echo "=== $(PARITY) fast"; \ DOCENT_FAST=1 $(PW_ENV) node $(DOCENT) $(PARITY).docent.yaml > parity.fast.log 2>&1 \ || { sed 's/^/ /' parity.fast.log; exit 1; } @echo "=== $(PARITY) slow (records an mp4, so this one is not quick)"; \ $(PW_ENV) node $(DOCENT) $(PARITY).docent.yaml > parity.slow.log 2>&1 \ || { sed 's/^/ /' parity.slow.log; exit 1; } @echo "=== $(PARITY) again, to catch state left behind by the last run"; \ DOCENT_FAST=1 $(PW_ENV) node $(DOCENT) $(PARITY).docent.yaml > parity.rerun.log 2>&1 \ || { sed 's/^/ /' parity.rerun.log; exit 1; } @echo "parity passed" # The derivation on its own: DOCENT_DERIVE=1 resolves each `track:` step against the # server's trackDb and prints the cart variables, with no browser and no navigation. That # is where Docent's own decisions are, and it runs in about a second, so it is worth # checking against a baseline. # # Only the scripts with a file in expected/ are checked. The output depends on LIVE # trackDb, so a baseline can go stale for an honest reason -- a new member of a superTrack, # a retired subtrack. When that happens, read the diff before believing it: # # make derive # diff every baseline # make derive-accept # rewrite the baselines, then `git diff` them # # Scripts whose derivation is large and churny (views, 188 variables from one view-level # hideKids) deliberately have NO baseline: it would fail every time ENCODE gained a cell # line, and the browser test already covers the behaviour. # # One line has to be stripped before the diff. docent.js caches the trackDb listing in # $TMPDIR for a day, and prints `trackDb: N tracks for DB from .../hubApi` only when it # actually fetches. So the first run of the day carries a line that every run after it # does not, and a baseline captured warm would fail against a cold run for a reason that # is not about trackDb at all. Both targets strip exactly that line, so it cannot get # into a baseline either. The other two trackDb lines -- a hub genome, an unreachable # hubApi -- are real news about the derivation and are left in. DERIVE_ENV = DOCENT_DERIVE=1 $(PW_ENV) DERIVE_FILTER = sed '/^trackDb: [0-9][0-9]* tracks for /d' BASELINES := $(patsubst expected/%.derive,%,$(wildcard expected/*.derive)) .PHONY: derive derive-accept derive: @if [ -z "$(strip $(BASELINES))" ]; then \ echo "no baselines in expected/ -- nothing was checked"; exit 1; fi @fail=0; \ for b in $(BASELINES); do \ $(DERIVE_ENV) node $(DOCENT) $$b.docent.yaml 2>&1 | $(DERIVE_FILTER) > $$b.derive.out; \ if diff -u expected/$$b.derive $$b.derive.out > $$b.derive.diff; then \ echo "=== $$b"; echo " ok"; rm -f $$b.derive.diff; \ else \ echo "=== $$b"; echo " CHANGED -- read this before accepting it:"; \ sed 's/^/ /' $$b.derive.diff; fail=1; \ fi; \ rm -f $$b.derive.out; \ done; \ if [ $$fail -eq 0 ]; then echo "derivation baselines match"; else echo "derivation CHANGED"; exit 1; fi derive-accept: @mkdir -p expected @for b in $(BASELINES); do \ $(DERIVE_ENV) node $(DOCENT) $$b.docent.yaml 2>&1 | $(DERIVE_FILTER) > expected/$$b.derive; \ echo "rewrote expected/$$b.derive"; \ done @echo "now read: git diff expected/" clean: rm -rf stills sessions *.log *.derive.out *.derive.diff *.mp4