91008f116877a4ac9de458dec4abb98ca3249dbf braney Wed Sep 23 17:15:55 2026 -0700 docent status page: try-it links for sessions loaded by URL, none on hgwbeta for named ones, refs #38252 A named session lives in one machine's hgcentral, so its hgwbeta link only answered "Could not find session"; those rows now offer genome-test alone and say why. A session loaded from a text file by URL works on any server, so a script that loads one now gets a link, with both servers. diff --git src/hg/utils/docent/tests/regress/nightlyStatus.py src/hg/utils/docent/tests/regress/nightlyStatus.py index 1bf1cd7d074..686ebe6135e 100755 --- src/hg/utils/docent/tests/regress/nightlyStatus.py +++ src/hg/utils/docent/tests/regress/nightlyStatus.py @@ -1,435 +1,447 @@ #!/usr/bin/env python3 """Write the Docent nightly status page. refs #38252 nightly.sh runs this after the tests. It reads what the job already keeps: the nightly logs (one per run, named YYYY-MM-DD_HHMM.txt), flips.log, and each committed script's proof: lines. It writes one static page with a row per script, anchored by the script's name, so a ticket comment can link to <url>#rm38236. Each row carries a reproduce-it-yourself link: the script's steps, as far as one URL can rebuild them. track: steps are resolved by docent.js's own DOCENT_DERIVE mode, so the URL turns on the same containers the script does. Every link is fetched from genome-test with a fresh cart before it is offered, and only a link that draws what its expect: step names is called verified. Nothing here decides pass or fail; the page only reports what the log says. A problem in this script must never change the night's verdict, so nightly.sh treats its exit status as a note in the mail and nothing more. """ import argparse import concurrent.futures import datetime import glob import html import json import os import re import subprocess import sys import urllib.parse import urllib.request import yaml HERE = os.path.dirname(os.path.abspath(__file__)) DOCENT = os.path.join(HERE, '..', '..', 'docent.js') PW = '/hive/groups/browser/uiTest/pw' GITHUB = 'https://github.com/ucscGenomeBrowser/kent' REDMINE = 'https://redmine.gi.ucsc.edu/issues' TEST = 'https://genome-test.gi.ucsc.edu' BETA = 'https://hgwbeta.soe.ucsc.edu' NIGHTS = 14 PIX = 'pix=1100' # nightly.sh writes this line into the log of every --update run, which is what cron runs. MARK = 'updated: this checkout was reset to origin/master' # Weakest first, the same order as proof.js. LEVELS = ['assertion-only', 'xfail', 'sandbox-ab', 'release-ab', 'server-flip', 'caught-regression'] PLAIN = { 'assertion-only': 'Checks the fixed behavior. Never seen to fail.', 'xfail': 'Fails now, as expected. The fix has not reached genome-test.', 'sandbox-ab': 'Seen to fail on a build without the fix, and pass with it.', 'release-ab': 'Seen to fail on the release before the fix, and pass on the release ' 'with it.', 'server-flip': 'Seen to fail on genome-test, then pass when the fix arrived.', 'caught-regression': 'Went red for a real regression, which was then fixed.', } # Verbs that change nothing a URL has to carry. PASSIVE = ('shot', 'caption', 'say', 'pause', 'wait', 'note', 'title', 'session') def parseLog(path): """Return (commit, {script: 'pass'|'fail'}, {script: text of its failure}).""" commit, res, text, cur, buf = None, {}, {}, None, [] def close(): if cur and res.get(cur) == 'fail': text[cur] = ''.join(buf) for line in open(path, errors='replace'): m = re.match(r'commit:\s+(\S+)', line) if m and commit is None: commit = m.group(1) m = re.match(r'=== (\S+)', line) if m: close() cur, buf = m.group(1), [] continue if cur is None: continue if line.startswith(('docent tests', 'make:', 'full log', '--- ')): close() cur = None continue buf.append(line) if cur not in res: if line.startswith(' ok'): res[cur] = 'pass' elif line.startswith(' FAILED'): res[cur] = 'fail' close() return commit, res, text def strongest(proofs): best = -1 for p in proofs or []: words = str(p).split() if words and words[0] in LEVELS: best = max(best, LEVELS.index(words[0])) return LEVELS[best] if best >= 0 else None def norm(step): if isinstance(step, str): verb, *rest = step.split() return verb, ' '.join(rest) if rest else True verb = next(iter(step)) return verb, step[verb] def enc(s): return urllib.parse.quote(str(s), safe=':,/') def derive(yamlPath): """{step number: {var: value}} for the script's track: steps, every round merged.""" env = dict(os.environ, DOCENT_DERIVE='1', NODE_PATH=PW + '/node_modules') out = subprocess.run(['node', DOCENT, yamlPath], env=env, capture_output=True, text=True, timeout=120).stdout steps, cur = {}, None for line in out.splitlines(): m = re.match(r'step (\d+) track ', line) if m: cur = int(m.group(1)) steps[cur] = {} continue m = re.match(r'\s+round \d+ \(\d+ vars\): (.*)', line) if m and cur: for kv in m.group(1).split(): k, _, v = kv.partition('=') steps[cur][k] = v return steps def urlOf(db, pos, base, cgiVars): v = dict(cgiVars) # hideTracks=1 in the same request as hubUrl hides the hub's own rows too, so a hub # keeps the default tracks rather than losing the ones the script is about. if 'hubUrl' in v: v.pop('hideTracks', None) if base is None: q = [f'db={enc(db)}'] + ([f'position={enc(pos)}'] if pos else []) base = '/cgi-bin/hgTracks?' + '&'.join(q) extra = '&'.join(f'{k}={enc(x)}' for k, x in v.items()) url = base + ('&' if '?' in base else '?') + extra if extra else base url = url.replace('&&', '&').rstrip('&') if url.startswith('/cgi-bin/hgTracks') and 'pix=' not in url: url += '&' + PIX return url def candidates(yamlPath, doc): """One URL per expect: step that a single request can reach, in step order.""" db = doc.get('db') or 'hg38' pos = doc.get('position') steps = doc.get('steps') or [] nExpect = sum(norm(s)[0] == 'expect' for s in steps) base, cgiVars, tracks, cands, seen, stop = None, {}, None, [], 0, None + session = False for i, s in enumerate(steps, 1): verb, arg = norm(s) if verb == 'expect': seen += 1 cands.append({'url': urlOf(db, pos, base, cgiVars), 'reached': seen, + 'session': session, 'expect': arg if isinstance(arg, dict) else {}}) continue if verb in PASSIVE: continue if verb == 'go': if arg is True: continue pos, base = arg, None elif verb == 'goto': base, cgiVars = str(arg), {} elif verb == 'hide' and arg in ('all', True): cgiVars['hideTracks'] = '1' elif verb == 'track' and isinstance(arg, dict): if tracks is None: tracks = derive(yamlPath) cgiVars.update(tracks.get(i, {})) elif verb == 'hub': o = arg if isinstance(arg, dict) else {'url': arg} cgiVars['hubUrl'] = o['url'] db = o.get('db', db) pos = o.get('position', pos) elif verb == 'loadSession' and isinstance(arg, dict) and arg.get('user'): cgiVars.update({'hgS_doOtherUser': 'submit', 'hgS_otherUserName': arg['user'], 'hgS_otherUserSessionName': arg.get('name')}) + session = True + elif verb == 'loadSession' and isinstance(arg, str) and re.match(r'https?://', arg): + # A session file by URL (sessionFiles/), which any server can load. + cgiVars.update({'hgS_doLoadUrl': 'submit', 'hgS_loadUrlName': arg}) else: stop = verb break for c in cands: c['total'], c['stop'] = nExpect, stop if not cands: return None, (f'"{stop}:" comes before the first check' if stop else 'it has no expect: step') return cands, None def verify(c): """Fetch with a fresh cart and look for what the expect: step names.""" exp = c.pop('expect') rows = exp.get('rows') or [] texts = [t for t in [exp.get('text')] if isinstance(t, str)] if not rows and not texts: c['ok'] = None return c try: page = urllib.request.urlopen(TEST + c['url'], timeout=90).read() page = page.decode('utf-8', 'replace') except Exception: c['ok'] = None return c drawn = set(re.findall(r"id=['\"]tr_([A-Za-z0-9_]+)", page)) missRow = [t for t in rows if not any(d == t or d.endswith('_' + t) for d in drawn)] missText = [t for t in texts if t not in page] c['ok'] = not missRow and not missText return c def repro(yamlPath, doc): """The deepest check one URL really reproduces, or the reason there is none.""" cands, why = candidates(yamlPath, doc) if cands is None: return {'reason': why} last = None for c in reversed(cands): last = verify(c) if last['ok'] is not False: return last return last def tryIt(name, r): e = html.escape if 'url' not in r: return f'<span class="note">Run the script: {e(r["reason"])}.</span>' u = e(r['url']) + # A saved session lives in one machine's central, and hgwbeta reads a different one, + # so a link that loads one would only answer "Could not find session" there. + if r.get('session'): + links = f'<a href="{TEST}{u}">genome-test</a> <span class="note">(a saved ' \ + f'session, so not on hgwbeta)</span>' + else: links = f'<a href="{TEST}{u}">genome-test</a> · <a href="{BETA}{u}">hgwbeta</a>' n, m = r['reached'], r['total'] if n == m: reach = f'Reaches the last check ({n} of {m}).' elif r.get('stop'): reach = f'Reaches check {n} of {m}. Then "{e(r["stop"])}:" needs the browser UI.' else: reach = f'Reaches check {n} of {m}. Later checks need more than one request.' if name.endswith('.xfail'): return f'{links}<div class="note">Shows the open bug. The fix is not on ' \ f'genome-test yet.</div>' if r['ok'] is True: return f'{links}<div class="note">{reach} Verified on genome-test.</div>' if r['ok'] is False: return '<span class="note">Run the script: one URL does not rebuild what its ' \ 'steps do.</span>' return f'{links}<div class="note">{reach} Not verified: nothing a fetch can see.</div>' def render(scripts, days, lastRun, commit, mediaUrl): e = html.escape nPass = sum(s['state'] == 'pass' for s in scripts) nFail = sum(s['state'] == 'fail' for s in scripts) byLevel = ' · '.join(f'{l} {sum(s["level"] == l for s in scripts)}' for l in LEVELS) rows = [] for s in scripts: cells = ''.join( f'<span class="c {h}" title="{d}: {h.replace("none", "not in suite")}"></span>' for d, h in zip(days, s['hist'])) if s['fail']: state = f'<a href="#fail-{s["name"]}" class="st fail">FAIL</a>' else: state = f'<span class="st {s["state"]}">{e(s["state"])}</span>' conf = PLAIN.get(s['level'], 'No proof line.') if s['flip']: conf += f' Flipped on genome-test {s["flip"]}.' proofs = ''.join(f'<li>{e(str(p))}</li>' for p in s['proofs']) links = [f'<a href="{GITHUB}/blob/{commit}/src/hg/utils/docent/tests/regress/' f'{s["file"]}">script</a>'] if s['media']: links.append(f'<a href="{mediaUrl}/rm{s["ticket"]}compare.png">before/after</a>') rows.append(f'''<tr id="{s["name"]}" class="{s["state"]}"> <td class="nowrap">{s["name"]} <a href="#{s["name"]}" class="anchor" title="link to this row">#</a></td> <td><a href="{REDMINE}/{s["ticket"]}">#{s["ticket"]}</a></td> <td>{state}</td><td class="nowrap">{s["since"]}</td> <td class="nowrap">{cells}</td> <td><span class="lvl">{e(s["level"] or "none")}</span> {e(conf)} <details><summary>proof lines</summary><ul>{proofs}</ul></details></td> <td>{tryIt(s["name"], s["repro"])}</td> <td class="nowrap">{" · ".join(links)}</td></tr>''') fails = ''.join(f'<h3 id="fail-{s["name"]}">{s["name"]}, failing since {s["since"]}</h3>' f'<pre>{e(s["fail"])}</pre>' for s in scripts if s['fail']) made = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') page = f'''<!DOCTYPE html> <html lang="en"><head><meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>Docent Nightly Status</title> <style> body {{ max-width: 1400px; font-family: Arial, Helvetica, sans-serif; font-size: 14px; margin: 16px; color: #000; background: #fff; }} h1 {{ font-size: 20px; margin-bottom: 4px; }} .sub {{ color: #444; margin-bottom: 12px; line-height: 1.5; }} table {{ border-collapse: collapse; width: 100%; }} th, td {{ border-bottom: 1px solid #ddd; padding: 4px 6px; text-align: left; vertical-align: top; }} th {{ background: #e8e8f0; position: sticky; top: 0; }} tr:target {{ background: #fff6c8; }} tr.fail {{ background: #fdecea; }} .nowrap {{ white-space: nowrap; }} .st.pass {{ color: #1a7f37; font-weight: bold; }} .st.fail {{ color: #b3261e; font-weight: bold; }} .c {{ display: inline-block; width: 9px; height: 14px; margin-right: 1px; }} .c.pass {{ background: #3fa35b; }} .c.fail {{ background: #d9443b; }} .c.none {{ background: #eee; }} .c.norun {{ background: repeating-linear-gradient(45deg, #fff, #fff 2px, #bbb 2px, #bbb 4px); }} .anchor {{ color: #999; text-decoration: none; font-size: 12px; }} .anchor:hover {{ color: #00c; }} .note {{ color: #555; font-size: 12px; }} .lvl {{ font-family: monospace; background: #eef; padding: 0 3px; }} details summary {{ color: #555; cursor: pointer; font-size: 12px; }} details ul {{ margin: 4px 0; padding-left: 18px; font-size: 12px; }} pre {{ background: #f6f6f6; padding: 8px; overflow-x: auto; font-size: 12px; }} a {{ color: #00c; }} .wrap {{ overflow-x: auto; }} </style></head><body> <h1>Docent Nightly Status</h1> <div class="sub">Run of {lastRun} against genome-test, at commit <a href="{GITHUB}/commit/{commit}">{commit[:11]}</a>. {len(scripts)} scripts: <b>{nPass} pass</b>, <b style="color:#b3261e">{nFail} fail</b>.<br> The squares are the last {len(days)} days, oldest on the left. A striped square is a night with no run.<br> Confidence, the strongest proof: level per script: {byLevel}.<br> Written {made} by nightlyStatus.py after the nightly run (refs <a href="{REDMINE}/38252">#38252</a>).</div> <div class="wrap"><table> <tr><th>script</th><th>ticket</th><th>state</th><th>since</th><th>last {len(days)} days</th> <th>confidence</th><th>try it</th><th>links</th></tr> {"".join(rows)} </table></div> <h2>Failures in the last run</h2> {fails or "<p>None.</p>"} </body></html> ''' # Every link that leaves the page opens a new tab; the in-page row links do not. return re.sub(r'<a href="(https?:)', r'<a target="_blank" rel="noopener" href="\1', page) def main(): ap = argparse.ArgumentParser(description=__doc__.split('\n')[0]) ap.add_argument('--logs', required=True, help='directory of nightly logs') ap.add_argument('--flips', required=True, help='flips.log') ap.add_argument('--out', required=True, help='directory to write index.html into') ap.add_argument('--media-dir', default=os.path.expanduser('~/docentTours'), help='where rm<ticket>compare.png pictures live') ap.add_argument('--media-url', default='https://hgwdev.gi.ucsc.edu/~braney/docentTours', help='the URL that serves --media-dir') args = ap.parse_args() # A night is a log written by the cron, and the cron always runs with --update, which # stamps its log. Not the log's name: a run can start minutes late (09-14 began at # 04:36), and a hand run of the script without --update is not a night. The last such # log of each day stands for that day. latest = {} for p in sorted(glob.glob(os.path.join(args.logs, '*.txt'))): with open(p, errors='replace') as fh: if MARK in fh.read(4096): latest[os.path.basename(p)[:10]] = p runs = [] for day in sorted(latest): commit, res, text = parseLog(latest[day]) runs.append((day, commit, res, text)) if not runs: sys.exit(f'no nightly logs in {args.logs}') lastDay, lastCommit, lastRes, lastText = runs[-1] byDay = {d: r for d, _, r, _ in runs} end = datetime.date.fromisoformat(lastDay) days = [str(end - datetime.timedelta(n)) for n in range(NIGHTS - 1, -1, -1)] flips = {} if os.path.exists(args.flips): for line in open(args.flips): if line.strip() and not line.startswith('#'): f = line.split() flips[f[1].replace('.xfail', '')] = f[0] # The committed scripts, as the nightly run lists them. files = subprocess.run(['git', 'ls-files', '*.docent.yaml'], cwd=HERE, capture_output=True, text=True).stdout.split() docs = {} for f in files: name = f[:-len('.docent.yaml')] docs[name] = (f, yaml.safe_load(open(os.path.join(HERE, f))) or {}) with concurrent.futures.ThreadPoolExecutor(6) as ex: repros = dict(zip(docs, ex.map(lambda n: repro(os.path.join(HERE, docs[n][0]), docs[n][1]), docs))) scripts = [] for name, (f, doc) in docs.items(): ticket = re.match(r'rm(\d+)', name).group(1) if name.startswith('rm') else '' state = lastRes.get(name, 'not run') since = lastDay for d, _, r, _ in reversed(runs): if r.get(name) != state: break since = d hist = [] for d in days: if d not in byDay: hist.append('norun') else: hist.append(byDay[d].get(name, 'none')) proofs = doc.get('proof') or [] scripts.append(dict( name=name, ticket=ticket, state=state, since=since, hist=hist, level=strongest(proofs), proofs=proofs, fail=lastText.get(name), flip=flips.get(name.replace('.xfail', '')), file=f, repro=repros[name], media=os.path.exists(os.path.join(args.media_dir, f'rm{ticket}compare.png')))) # Failures first, then weakest confidence first, so what needs a person is on top. scripts.sort(key=lambda s: (s['state'] != 'fail', LEVELS.index(s['level']) if s['level'] else -1, s['name'])) page = render(scripts, days, lastDay, lastCommit or 'master', args.media_url) os.makedirs(args.out, exist_ok=True) tmp = os.path.join(args.out, '.index.html.tmp') with open(tmp, 'w') as fh: fh.write(page) os.chmod(tmp, 0o664) os.replace(tmp, os.path.join(args.out, 'index.html')) print(f'{len(scripts)} scripts, {sum(s["state"] == "fail" for s in scripts)} failing, ' f'{sum(s["repro"].get("ok") is True for s in scripts)} verified try-it links') if __name__ == '__main__': main()