fbbc2d15114548cc241e4b99dc83c0c9e79310ac braney Tue Jul 28 10:20:22 2026 -0700 Docent: take track state from trackDb, add goShow, search the Convert target visibly refs #37892 Track visibility no longer comes from a table inside the renderer. docent.js used to carry per-composite cart parameters (mane/dbSnp155/clinvar), which pinned clinvarMain to dense for every clinvar step and quietly aliased dbSnp155 to a subtrack it does not name. It now reads the trackDb of the server it is driving (hubApi /list/tracks, cached a day in $TMPDIR) and derives what a step needs: the containers above a track, and the _sel checkbox that actually decides a composite child. Nothing is pushed downward, since a container's visibility already reaches its selected children, so a script names only its deviations from trackDb. Scripts use real trackDb names; names trackDb does not have (hubs, custom tracks, a quickLift target) are sent as a literal name=mode. Two hgTracks mechanics this had to learn: a bare clinvarCnv=hide is dropped when the container's visibility is in the same request (the cart keeps clinvarCnv_sel=1 and the row still draws), so a step naming both a composite and a child of it is applied in two requests, container first. New goShow verb: types a position or a GENE NAME into the position bar and lets the page finish -- Search on hgTracks, the arrow on hgGateway. A gene name goes through the browser's own suggestion menu, so it lands on the gene rather than the search-results page. pick: disambiguates the menu. convert: now finds the target the way a user does, by typing it into the Convert page's own genome search bar and clicking the suggestion, and accepts shot: for the Convert page itself (opened / filled / result), which no other verb can reach. Stills of pages that are not hgTracks (an hgc detail page, an external page a link led to) are the viewport -- the top of the page -- instead of an element shot of the whole scrolling document, which ran to 4400px. A fading click ripple is removed before a still: it belongs to the video, not to a figure. Speed: FAST=1 (make FAST=1 BP1, DOCENT_FAST=1, or fast: true) drops everything that exists only for the video -- dwells, cursor animation, dropdown theatrics, screen recording, mp4 transcode -- taking BP1 from 64s to 24s for the same figures. docent.mk documents make -j for parallel scenarios; the trackDb cache is written via rename so concurrent runs cannot read a partial file, and holds the derived index (3MB) rather than hubApi's reply (30MB). DOCENT_TIME=1 prints where the wall clock went, DOCENT_ROWS=1 the rows hgTracks actually drew. Trimming the dwells exposed a real race in mouseover: dismissing the previous tooltip with a sleep and then waiting for "a tooltip is visible" was satisfied by the stale one, so a back-to-back pinned mouseover captured the previous item's text. It now waits for the old tooltip to be gone and for a new one whose content differs. Co-Authored-By: Claude Opus 5 (1M context) diff --git src/hg/utils/docent/docent.js src/hg/utils/docent/docent.js index e07bf093564..5c27819655b 100755 --- src/hg/utils/docent/docent.js +++ src/hg/utils/docent/docent.js @@ -9,30 +9,31 @@ * * PLAYWRIGHT_BROWSERS_PATH=~/pwrec/browsers NODE_PATH=~/pwrec/node_modules \ * node docent.js AP1.docent.yaml * * The high-level verbs bake in the quickLift/Convert mechanics (dbSNP composite * params, the hideDefaults-reverts-on-assembly-change bug, target lookup) so the * author writes intent, not selectors. See README.md for the language. * * The surface syntax is YAML so ordinary editors highlight it; the language is the * verb vocabulary layered on top, not the serialization. Scripts are named * .docent.yaml (a bare .docent also works). */ const { chromium } = require('playwright'); const yaml = require('js-yaml'); const fs = require('fs'); +const os = require('os'); const path = require('path'); const { execFileSync } = require('child_process'); // ---------- parse script + config ---------- const SCRIPT = process.argv[2]; if (!SCRIPT) { console.error('usage: node docent.js SCRIPT.docent.yaml [OUT.mp4]'); process.exit(2); } const doc = yaml.load(fs.readFileSync(SCRIPT, 'utf8')) || {}; // Lint: in a YAML flow map a colon needs a trailing space, so `{item:name5568747}` // parses as ONE key "item:name5568747" (value null) and the intended `item:` arg is // silently dropped -- the verb then quietly falls back to a default. Catch that here // (before the long browser run) by flagging any arg key that contains a ':'. (function lintSteps(steps) { let n = 0; const scan = (obj, where) => { @@ -64,193 +65,464 @@ const SERVERS = { 'rr': 'https://genome.ucsc.edu/cgi-bin', 'genome-test': 'https://genome-test.gi.ucsc.edu/cgi-bin', 'hgwdev': 'https://hgwdev.gi.ucsc.edu/cgi-bin', 'hgwbeta': 'https://hgwbeta.soe.ucsc.edu/cgi-bin', }; const resolveTarget = t => { if (!t) return SERVERS['genome-test']; if (SERVERS[t]) return SERVERS[t]; if (/^hgwdev-[a-z0-9._-]+$/i.test(t)) return `https://${t}.gi.ucsc.edu/cgi-bin`; // personal sandbox return t; // full URL }; const SERVER = resolveTarget(doc.target).replace(/\/$/, ''); const [VW, VH] = doc.size || [1000, 760]; const PIX = doc.pix || 850; -const PACE = Math.round((doc.pace ?? 1.2) * 1000); // dwell after each step -const SHOTHOLD = Math.round((doc.shotHold ?? 2.2) * 1000); // extra dwell (pause) at a shot +// FAST: iterate on the FIGURES. Everything that exists only for the video is dropped -- +// the dwells, the cursor animation, the dropdown theatrics, the screen recording and the +// mp4 transcode. The stills are byte-for-byte what a full run produces, and a run costs +// roughly a third as long. `fast: true` in the script, DOCENT_FAST=1, or `make FAST=1 BP1`. +const FAST = !!(doc.fast || process.env.DOCENT_FAST); +const PACE = FAST ? 0 : Math.round((doc.pace ?? 1.2) * 1000); // dwell after each step +const SHOTHOLD = FAST ? 0 : Math.round((doc.shotHold ?? 2.2) * 1000); // extra pause at a shot -// track-name -> cart params (composite tracks expand to their learned "clean" config) -const TRACKS = { - mane: m => [`mane=${m}`], - dbSnp155: m => [`dbSnp155Composite=${m}`, `dbSnp155Common=${m}`, `dbSnp155ViewVariants=${m}`, `dbSnp155ViewErrs=hide`], - clinvar: m => [`clinvar=${m}`, `clinvarMain=dense`, `clinvarSubLolly=${m}`, `clinvarCnv=hide`], +// ---------- trackDb ---------- +// Docent carries NO table of per-track cart variables. Such a table encodes one snapshot +// of trackDb and then quietly lies when trackDb changes (this file used to pin +// `clinvarMain=dense` for every `clinvar:` step, for instance). Instead ask the server for +// the trackDb it is driving -- hubApi /list/tracks -- and derive what a step needs: +// +// * the containers above a subtrack (composite, view, superTrack) that have to be +// turned on with it, and what each of those takes (a superTrack wants show/hide), +// * whether trackDb leaves that subtrack UNSELECTED (`parent off`), in which case +// the subtrack checkbox `_sel` has to come along, +// * which leaf actually draws the pixels for a container name. +// +// Everything else -- which dropdown to open, which row to hover -- is read off the live +// page. Tracks the listing doesn't know (attached hubs, custom tracks, quickLift's own +// tracks on the target) fall back to a literal `name=mode`, which is all Docent could +// honestly do for them anyway. +const TDB_TTL = 24 * 3600 * 1000; // re-fetch a cached listing daily +const TDB_CACHE = path.join(os.tmpdir(), `docent-tdb-${SERVER.replace(/[^\w.-]+/g, '_')}`); +let tdbPending = null; // db -> Promise, once each +function tdbParse(genome) { + const idx = new Map(); + const add = (name, o) => { + const p = String(o.parent || '').trim().split(/\s+/); + idx.set(name, { + name, + parent: p[0] || null, + parentState: (p[1] || '').toLowerCase(), // on | off | a visibility | '' + vis: o.visibility || null, + view: o.view || null, + superChild: !!o.superTrack, // member of a superTrack + superTrack: false, // set below for containers + children: [], + }); + for (const [k, v] of Object.entries(o)) + if (v && typeof v === 'object' && !Array.isArray(v)) add(k, v); }; -// The control-dropdown name and the data-image id can differ from the shortcut name: -// composites are turned on via their composite cart var, and dbSNP's visible pixels -// live in the "Common" subtrack's data image. -const CTRL = { dbSnp155: 'dbSnp155Composite' }; // select[name=...] in the track controls -const DATAIMG = { dbSnp155: 'dbSnp155Common', clinvar: 'clinvarMain' }; // #img_data_... that holds the drawn items -const ctrlName = t => CTRL[t] || t; -const imgTrack = t => DATAIMG[t] || t; + for (const [k, v] of Object.entries(genome || {})) + if (v && typeof v === 'object' && !Array.isArray(v)) add(k, v); + // hubApi flattens superTrack members to the top level and never lists the superTrack + // itself, so synthesize the container from the `parent` field it points at. + for (const n of [...idx.values()]) { + if (!n.parent) continue; + if (!idx.has(n.parent)) + idx.set(n.parent, { name: n.parent, parent: null, parentState: '', vis: null, view: null, + superChild: false, superTrack: true, children: [] }); + idx.get(n.parent).children.push(n.name); + } + return idx; +} +// The cache holds the DERIVED index (a few hundred kB), not hubApi's reply (~30 MB for +// hg38): same information for our purposes, ~20x less to read and parse on every run. +function tdbFlatten(idx) { + return { docentIndex: 1, + rows: [...idx.values()].map(n => [n.name, n.parent, n.parentState, n.vis, n.view, + n.superChild ? 1 : 0, n.superTrack ? 1 : 0]) }; +} +function tdbInflate(o) { + const idx = new Map(); + for (const [name, parent, parentState, vis, view, superChild, superTrack] of o.rows) + idx.set(name, { name, parent, parentState, vis, view, + superChild: !!superChild, superTrack: !!superTrack, children: [] }); + for (const n of idx.values()) if (n.parent && idx.has(n.parent)) idx.get(n.parent).children.push(n.name); + return idx; +} +async function tdbIndex(db) { + if (!tdbPending) tdbPending = new Map(); + if (tdbPending.has(db)) return tdbPending.get(db); + const p = (async () => { + const cache = `${TDB_CACHE}-${db}.json`; + try { + const st = fs.statSync(cache); + if (Date.now() - st.mtimeMs < TDB_TTL) { + const o = JSON.parse(fs.readFileSync(cache, 'utf8')); + return o && o.docentIndex ? tdbInflate(o) : tdbParse(o); + } + } catch (e) {} + const url = `${SERVER}/hubApi/list/tracks?genome=${enc(db)}&trackLeavesOnly=0`; + try { + const r = await fetch(url); + if (!r.ok) throw new Error(`HTTP ${r.status}`); + const j = await r.json(); + const genome = j[db]; + // A hub-supplied genome (an assembly hub, or quickLift's own generated target hub) + // isn't in the server's trackDb listing at all -- expected, not a failure. + if (!genome) { + console.log(`trackDb: ${db} is not a server assembly (hub genome), ` + + `so track steps there are sent as literal name=mode`); + return null; + } + const idx = tdbParse(genome); + // Write via a unique temp file + rename so parallel builds (make -j) can't read a + // half-written cache. + try { + const tmp = `${cache}.${process.pid}.tmp`; + fs.writeFileSync(tmp, JSON.stringify(tdbFlatten(idx))); + fs.renameSync(tmp, cache); + } catch (e) {} + console.log(`trackDb: ${idx.size} tracks for ${db} from ${SERVER}/hubApi`); + return idx; + } catch (e) { + console.warn(`trackDb: could not read ${url} (${e.message}) -- ` + + `falling back to literal name=mode for every track step`); + return null; + } + })(); + tdbPending.set(db, p); + return p; +} const sleep = ms => new Promise(r => setTimeout(r, ms)); +// A pause that exists only so a viewer can follow the video: skipped entirely in FAST. +const dwell = ms => (FAST ? Promise.resolve() : sleep(ms)); +// Typing is shown on screen for the video; in FAST just put the text in the box (fill() +// still fires the input events the autocompletes listen for). +async function typeIn(pg, sel, text) { + if (FAST) await pg.fill(sel, String(text)); + else await pg.type(sel, String(text), { delay: 45 }); +} const enc = s => encodeURIComponent(String(s)); const state = { db: doc.db || 'hg38', position: doc.position || '', hgsid: '' }; // ---------- animated cursor (same technique as the walkthrough-video skill's record.js) ---------- const CURSOR_INIT = () => { const add = () => { if (document.getElementById('__cur')) return; const c = document.createElement('div'); c.id = '__cur'; c.style.cssText = 'position:fixed;left:0;top:0;z-index:2147483647;pointer-events:none;width:24px;height:24px;margin-left:-3px;margin-top:-2px;filter:drop-shadow(0 1px 1px rgba(0,0,0,.4));'; c.innerHTML = ''; document.documentElement.appendChild(c); const place = (x, y) => { c.style.transform = `translate(${x}px,${y}px)`; }; place(120, 120); document.addEventListener('mousemove', e => place(e.clientX, e.clientY), true); document.addEventListener('mousedown', e => { const r = document.createElement('div'); + r.className = '__ripple'; r.style.cssText = `position:fixed;left:${e.clientX}px;top:${e.clientY}px;z-index:2147483645;pointer-events:none;width:6px;height:6px;margin:-3px 0 0 -3px;border:3px solid rgba(225,30,30,.95);border-radius:50%;`; document.documentElement.appendChild(r); r.animate([{ transform: 'scale(1)', opacity: 1 }, { transform: 'scale(6)', opacity: 0 }], { duration: 520, easing: 'ease-out' }); setTimeout(() => r.remove(), 540); }, true); }; if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', add); else add(); }; function absurl(u) { if (/^https?:/.test(u)) return u; if (u.startsWith('/cgi-bin/')) return SERVER.replace(/\/cgi-bin$/, '') + u; if (u.startsWith('/')) return SERVER.replace(/\/cgi-bin$/, '') + u; return SERVER + '/' + u; } +const T_START = Date.now(); (async () => { fs.mkdirSync(STILLDIR, { recursive: true }); const browser = await chromium.launch({ headless: true, args: ['--force-color-profile=srgb'] }); const ctx = await browser.newContext({ viewport: { width: VW, height: VH }, deviceScaleFactor: 1, - recordVideo: { dir: path.join(HERE, '.vid_' + base), size: { width: VW, height: VH } }, + ...(FAST ? {} : { recordVideo: { dir: path.join(HERE, '.vid_' + base), size: { width: VW, height: VH } } }), }); await ctx.addInitScript(CURSOR_INIT); await ctx.addInitScript(() => { try { localStorage.setItem('hgTracks_hideTutorial', '1'); } catch (e) {} }); const page = await ctx.newPage(); const cur = { x: 120, y: 120 }; const pinnedTips = []; // recorded mouseover tooltips for the next pinShot (per view) async function captureState() { try { const u = new URL(page.url()); const h = u.searchParams.get('hgsid'); if (h) state.hgsid = h; const db = u.searchParams.get('db'); if (db) state.db = db; const p = u.searchParams.get('position'); if (p) state.position = p; } catch (e) {} // An interactive zoom / drag-select reload stores the new window in the CART, not the // URL, so read the live position straight from hgTracks when we're on a tracks page -- // otherwise a later position-based nav (e.g. turning on a track) reverts the zoom. try { const pos = await page.evaluate(() => { try { if (typeof hgTracks !== 'undefined' && hgTracks.chromName) return hgTracks.chromName + ':' + (hgTracks.winStart + 1) + '-' + hgTracks.winEnd; } catch (_) {} return null; }); if (pos) state.position = pos; } catch (e) {} + // Authoring aid: DOCENT_ROWS=1 logs the rows hgTracks actually drew, so "why is that + // subtrack still there / why is my mouseover track not shown" is one run, not a guess. + if (process.env.DOCENT_ROWS) { + const rows = await page.evaluate(() => + [...document.querySelectorAll('[id^="img_data_"]')].map(e => e.id.replace('img_data_', ''))).catch(() => null); + if (rows) console.log(' rows:', rows.join(', ') || '(none)'); + } } async function nav(u) { pinnedTips.length = 0; await page.goto(absurl(u), { waitUntil: 'load' }); await captureState(); await page.mouse.move(cur.x, cur.y); } async function glide(x, y) { + if (FAST) { await page.mouse.move(x, y); cur.x = x; cur.y = y; return; } const steps = Math.max(10, Math.round(Math.hypot(x - cur.x, y - cur.y) / 9)); for (let i = 1; i <= steps; i++) { await page.mouse.move(cur.x + (x - cur.x) * i / steps, cur.y + (y - cur.y) * i / steps); await sleep(15); } cur.x = x; cur.y = y; } async function glideTo(sel) { const b = await page.locator(sel).first().boundingBox({ timeout: 8000 }).catch(() => null); if (b) await glide(b.x + b.width / 2, b.y + b.height / 2); } async function clickGlide(sel) { await glideTo(sel); await sleep(160); await page.click(sel); } async function checkGlide(sel, want) { await glideTo(sel); await sleep(140); if (want) await page.check(sel).catch(() => {}); else await page.uncheck(sel).catch(() => {}); } // visibly open a native