fbbc2d15114548cc241e4b99dc83c0c9e79310ac braney Tue Jul 28 10:20:22 2026 -0700 Docent: take track state from trackDb, add goShow, search the Convert target visibly refs #37892 Track visibility no longer comes from a table inside the renderer. docent.js used to carry per-composite cart parameters (mane/dbSnp155/clinvar), which pinned clinvarMain to dense for every clinvar step and quietly aliased dbSnp155 to a subtrack it does not name. It now reads the trackDb of the server it is driving (hubApi /list/tracks, cached a day in $TMPDIR) and derives what a step needs: the containers above a track, and the <name>_sel checkbox that actually decides a composite child. Nothing is pushed downward, since a container's visibility already reaches its selected children, so a script names only its deviations from trackDb. Scripts use real trackDb names; names trackDb does not have (hubs, custom tracks, a quickLift target) are sent as a literal name=mode. Two hgTracks mechanics this had to learn: a bare clinvarCnv=hide is dropped when the container's visibility is in the same request (the cart keeps clinvarCnv_sel=1 and the row still draws), so a step naming both a composite and a child of it is applied in two requests, container first. New goShow verb: types a position or a GENE NAME into the position bar and lets the page finish -- Search on hgTracks, the arrow on hgGateway. A gene name goes through the browser's own suggestion menu, so it lands on the gene rather than the search-results page. pick: disambiguates the menu. convert: now finds the target the way a user does, by typing it into the Convert page's own genome search bar and clicking the suggestion, and accepts shot: for the Convert page itself (opened / filled / result), which no other verb can reach. Stills of pages that are not hgTracks (an hgc detail page, an external page a link led to) are the viewport -- the top of the page -- instead of an element shot of the whole scrolling document, which ran to 4400px. A fading click ripple is removed before a still: it belongs to the video, not to a figure. Speed: FAST=1 (make FAST=1 BP1, DOCENT_FAST=1, or fast: true) drops everything that exists only for the video -- dwells, cursor animation, dropdown theatrics, screen recording, mp4 transcode -- taking BP1 from 64s to 24s for the same figures. docent.mk documents make -j for parallel scenarios; the trackDb cache is written via rename so concurrent runs cannot read a partial file, and holds the derived index (3MB) rather than hubApi's reply (30MB). DOCENT_TIME=1 prints where the wall clock went, DOCENT_ROWS=1 the rows hgTracks actually drew. Trimming the dwells exposed a real race in mouseover: dismissing the previous tooltip with a sleep and then waiting for "a tooltip is visible" was satisfied by the stale one, so a back-to-back pinned mouseover captured the previous item's text. It now waits for the old tooltip to be gone and for a new one whose content differs. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> diff --git src/hg/utils/docent/docent.js src/hg/utils/docent/docent.js index e07bf093564..5c27819655b 100755 --- src/hg/utils/docent/docent.js +++ src/hg/utils/docent/docent.js @@ -9,30 +9,31 @@ * * PLAYWRIGHT_BROWSERS_PATH=~/pwrec/browsers NODE_PATH=~/pwrec/node_modules \ * node docent.js AP1.docent.yaml * * The high-level verbs bake in the quickLift/Convert mechanics (dbSNP composite * params, the hideDefaults-reverts-on-assembly-change bug, target lookup) so the * author writes intent, not selectors. See README.md for the language. * * The surface syntax is YAML so ordinary editors highlight it; the language is the * verb vocabulary layered on top, not the serialization. Scripts are named * <base>.docent.yaml (a bare <base>.docent also works). */ const { chromium } = require('playwright'); const yaml = require('js-yaml'); const fs = require('fs'); +const os = require('os'); const path = require('path'); const { execFileSync } = require('child_process'); // ---------- parse script + config ---------- const SCRIPT = process.argv[2]; if (!SCRIPT) { console.error('usage: node docent.js SCRIPT.docent.yaml [OUT.mp4]'); process.exit(2); } const doc = yaml.load(fs.readFileSync(SCRIPT, 'utf8')) || {}; // Lint: in a YAML flow map a colon needs a trailing space, so `{item:name5568747}` // parses as ONE key "item:name5568747" (value null) and the intended `item:` arg is // silently dropped -- the verb then quietly falls back to a default. Catch that here // (before the long browser run) by flagging any arg key that contains a ':'. (function lintSteps(steps) { let n = 0; const scan = (obj, where) => { @@ -64,193 +65,464 @@ const SERVERS = { 'rr': 'https://genome.ucsc.edu/cgi-bin', 'genome-test': 'https://genome-test.gi.ucsc.edu/cgi-bin', 'hgwdev': 'https://hgwdev.gi.ucsc.edu/cgi-bin', 'hgwbeta': 'https://hgwbeta.soe.ucsc.edu/cgi-bin', }; const resolveTarget = t => { if (!t) return SERVERS['genome-test']; if (SERVERS[t]) return SERVERS[t]; if (/^hgwdev-[a-z0-9._-]+$/i.test(t)) return `https://${t}.gi.ucsc.edu/cgi-bin`; // personal sandbox return t; // full URL }; const SERVER = resolveTarget(doc.target).replace(/\/$/, ''); const [VW, VH] = doc.size || [1000, 760]; const PIX = doc.pix || 850; -const PACE = Math.round((doc.pace ?? 1.2) * 1000); // dwell after each step -const SHOTHOLD = Math.round((doc.shotHold ?? 2.2) * 1000); // extra dwell (pause) at a shot +// FAST: iterate on the FIGURES. Everything that exists only for the video is dropped -- +// the dwells, the cursor animation, the dropdown theatrics, the screen recording and the +// mp4 transcode. The stills are byte-for-byte what a full run produces, and a run costs +// roughly a third as long. `fast: true` in the script, DOCENT_FAST=1, or `make FAST=1 BP1`. +const FAST = !!(doc.fast || process.env.DOCENT_FAST); +const PACE = FAST ? 0 : Math.round((doc.pace ?? 1.2) * 1000); // dwell after each step +const SHOTHOLD = FAST ? 0 : Math.round((doc.shotHold ?? 2.2) * 1000); // extra pause at a shot -// track-name -> cart params (composite tracks expand to their learned "clean" config) -const TRACKS = { - mane: m => [`mane=${m}`], - dbSnp155: m => [`dbSnp155Composite=${m}`, `dbSnp155Common=${m}`, `dbSnp155ViewVariants=${m}`, `dbSnp155ViewErrs=hide`], - clinvar: m => [`clinvar=${m}`, `clinvarMain=dense`, `clinvarSubLolly=${m}`, `clinvarCnv=hide`], +// ---------- trackDb ---------- +// Docent carries NO table of per-track cart variables. Such a table encodes one snapshot +// of trackDb and then quietly lies when trackDb changes (this file used to pin +// `clinvarMain=dense` for every `clinvar:` step, for instance). Instead ask the server for +// the trackDb it is driving -- hubApi /list/tracks -- and derive what a step needs: +// +// * the containers above a subtrack (composite, view, superTrack) that have to be +// turned on with it, and what each of those takes (a superTrack wants show/hide), +// * whether trackDb leaves that subtrack UNSELECTED (`parent <c> off`), in which case +// the subtrack checkbox `<name>_sel` has to come along, +// * which leaf actually draws the pixels for a container name. +// +// Everything else -- which dropdown to open, which row to hover -- is read off the live +// page. Tracks the listing doesn't know (attached hubs, custom tracks, quickLift's own +// tracks on the target) fall back to a literal `name=mode`, which is all Docent could +// honestly do for them anyway. +const TDB_TTL = 24 * 3600 * 1000; // re-fetch a cached listing daily +const TDB_CACHE = path.join(os.tmpdir(), `docent-tdb-${SERVER.replace(/[^\w.-]+/g, '_')}`); +let tdbPending = null; // db -> Promise<Map|null>, once each +function tdbParse(genome) { + const idx = new Map(); + const add = (name, o) => { + const p = String(o.parent || '').trim().split(/\s+/); + idx.set(name, { + name, + parent: p[0] || null, + parentState: (p[1] || '').toLowerCase(), // on | off | a visibility | '' + vis: o.visibility || null, + view: o.view || null, + superChild: !!o.superTrack, // member of a superTrack + superTrack: false, // set below for containers + children: [], + }); + for (const [k, v] of Object.entries(o)) + if (v && typeof v === 'object' && !Array.isArray(v)) add(k, v); }; -// The control-dropdown name and the data-image id can differ from the shortcut name: -// composites are turned on via their composite cart var, and dbSNP's visible pixels -// live in the "Common" subtrack's data image. -const CTRL = { dbSnp155: 'dbSnp155Composite' }; // select[name=...] in the track controls -const DATAIMG = { dbSnp155: 'dbSnp155Common', clinvar: 'clinvarMain' }; // #img_data_... that holds the drawn items -const ctrlName = t => CTRL[t] || t; -const imgTrack = t => DATAIMG[t] || t; + for (const [k, v] of Object.entries(genome || {})) + if (v && typeof v === 'object' && !Array.isArray(v)) add(k, v); + // hubApi flattens superTrack members to the top level and never lists the superTrack + // itself, so synthesize the container from the `parent` field it points at. + for (const n of [...idx.values()]) { + if (!n.parent) continue; + if (!idx.has(n.parent)) + idx.set(n.parent, { name: n.parent, parent: null, parentState: '', vis: null, view: null, + superChild: false, superTrack: true, children: [] }); + idx.get(n.parent).children.push(n.name); + } + return idx; +} +// The cache holds the DERIVED index (a few hundred kB), not hubApi's reply (~30 MB for +// hg38): same information for our purposes, ~20x less to read and parse on every run. +function tdbFlatten(idx) { + return { docentIndex: 1, + rows: [...idx.values()].map(n => [n.name, n.parent, n.parentState, n.vis, n.view, + n.superChild ? 1 : 0, n.superTrack ? 1 : 0]) }; +} +function tdbInflate(o) { + const idx = new Map(); + for (const [name, parent, parentState, vis, view, superChild, superTrack] of o.rows) + idx.set(name, { name, parent, parentState, vis, view, + superChild: !!superChild, superTrack: !!superTrack, children: [] }); + for (const n of idx.values()) if (n.parent && idx.has(n.parent)) idx.get(n.parent).children.push(n.name); + return idx; +} +async function tdbIndex(db) { + if (!tdbPending) tdbPending = new Map(); + if (tdbPending.has(db)) return tdbPending.get(db); + const p = (async () => { + const cache = `${TDB_CACHE}-${db}.json`; + try { + const st = fs.statSync(cache); + if (Date.now() - st.mtimeMs < TDB_TTL) { + const o = JSON.parse(fs.readFileSync(cache, 'utf8')); + return o && o.docentIndex ? tdbInflate(o) : tdbParse(o); + } + } catch (e) {} + const url = `${SERVER}/hubApi/list/tracks?genome=${enc(db)}&trackLeavesOnly=0`; + try { + const r = await fetch(url); + if (!r.ok) throw new Error(`HTTP ${r.status}`); + const j = await r.json(); + const genome = j[db]; + // A hub-supplied genome (an assembly hub, or quickLift's own generated target hub) + // isn't in the server's trackDb listing at all -- expected, not a failure. + if (!genome) { + console.log(`trackDb: ${db} is not a server assembly (hub genome), ` + + `so track steps there are sent as literal name=mode`); + return null; + } + const idx = tdbParse(genome); + // Write via a unique temp file + rename so parallel builds (make -j) can't read a + // half-written cache. + try { + const tmp = `${cache}.${process.pid}.tmp`; + fs.writeFileSync(tmp, JSON.stringify(tdbFlatten(idx))); + fs.renameSync(tmp, cache); + } catch (e) {} + console.log(`trackDb: ${idx.size} tracks for ${db} from ${SERVER}/hubApi`); + return idx; + } catch (e) { + console.warn(`trackDb: could not read ${url} (${e.message}) -- ` + + `falling back to literal name=mode for every track step`); + return null; + } + })(); + tdbPending.set(db, p); + return p; +} const sleep = ms => new Promise(r => setTimeout(r, ms)); +// A pause that exists only so a viewer can follow the video: skipped entirely in FAST. +const dwell = ms => (FAST ? Promise.resolve() : sleep(ms)); +// Typing is shown on screen for the video; in FAST just put the text in the box (fill() +// still fires the input events the autocompletes listen for). +async function typeIn(pg, sel, text) { + if (FAST) await pg.fill(sel, String(text)); + else await pg.type(sel, String(text), { delay: 45 }); +} const enc = s => encodeURIComponent(String(s)); const state = { db: doc.db || 'hg38', position: doc.position || '', hgsid: '' }; // ---------- animated cursor (same technique as the walkthrough-video skill's record.js) ---------- const CURSOR_INIT = () => { const add = () => { if (document.getElementById('__cur')) return; const c = document.createElement('div'); c.id = '__cur'; c.style.cssText = 'position:fixed;left:0;top:0;z-index:2147483647;pointer-events:none;width:24px;height:24px;margin-left:-3px;margin-top:-2px;filter:drop-shadow(0 1px 1px rgba(0,0,0,.4));'; c.innerHTML = '<svg width="24" height="24" viewBox="0 0 24 24"><path d="M3 2 L3 19 L7.5 14.5 L10.5 21.5 L13.5 20.2 L10.6 13.5 L17 13.5 Z" fill="#111" stroke="#fff" stroke-width="1.3"/></svg>'; document.documentElement.appendChild(c); const place = (x, y) => { c.style.transform = `translate(${x}px,${y}px)`; }; place(120, 120); document.addEventListener('mousemove', e => place(e.clientX, e.clientY), true); document.addEventListener('mousedown', e => { const r = document.createElement('div'); + r.className = '__ripple'; r.style.cssText = `position:fixed;left:${e.clientX}px;top:${e.clientY}px;z-index:2147483645;pointer-events:none;width:6px;height:6px;margin:-3px 0 0 -3px;border:3px solid rgba(225,30,30,.95);border-radius:50%;`; document.documentElement.appendChild(r); r.animate([{ transform: 'scale(1)', opacity: 1 }, { transform: 'scale(6)', opacity: 0 }], { duration: 520, easing: 'ease-out' }); setTimeout(() => r.remove(), 540); }, true); }; if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', add); else add(); }; function absurl(u) { if (/^https?:/.test(u)) return u; if (u.startsWith('/cgi-bin/')) return SERVER.replace(/\/cgi-bin$/, '') + u; if (u.startsWith('/')) return SERVER.replace(/\/cgi-bin$/, '') + u; return SERVER + '/' + u; } +const T_START = Date.now(); (async () => { fs.mkdirSync(STILLDIR, { recursive: true }); const browser = await chromium.launch({ headless: true, args: ['--force-color-profile=srgb'] }); const ctx = await browser.newContext({ viewport: { width: VW, height: VH }, deviceScaleFactor: 1, - recordVideo: { dir: path.join(HERE, '.vid_' + base), size: { width: VW, height: VH } }, + ...(FAST ? {} : { recordVideo: { dir: path.join(HERE, '.vid_' + base), size: { width: VW, height: VH } } }), }); await ctx.addInitScript(CURSOR_INIT); await ctx.addInitScript(() => { try { localStorage.setItem('hgTracks_hideTutorial', '1'); } catch (e) {} }); const page = await ctx.newPage(); const cur = { x: 120, y: 120 }; const pinnedTips = []; // recorded mouseover tooltips for the next pinShot (per view) async function captureState() { try { const u = new URL(page.url()); const h = u.searchParams.get('hgsid'); if (h) state.hgsid = h; const db = u.searchParams.get('db'); if (db) state.db = db; const p = u.searchParams.get('position'); if (p) state.position = p; } catch (e) {} // An interactive zoom / drag-select reload stores the new window in the CART, not the // URL, so read the live position straight from hgTracks when we're on a tracks page -- // otherwise a later position-based nav (e.g. turning on a track) reverts the zoom. try { const pos = await page.evaluate(() => { try { if (typeof hgTracks !== 'undefined' && hgTracks.chromName) return hgTracks.chromName + ':' + (hgTracks.winStart + 1) + '-' + hgTracks.winEnd; } catch (_) {} return null; }); if (pos) state.position = pos; } catch (e) {} + // Authoring aid: DOCENT_ROWS=1 logs the rows hgTracks actually drew, so "why is that + // subtrack still there / why is my mouseover track not shown" is one run, not a guess. + if (process.env.DOCENT_ROWS) { + const rows = await page.evaluate(() => + [...document.querySelectorAll('[id^="img_data_"]')].map(e => e.id.replace('img_data_', ''))).catch(() => null); + if (rows) console.log(' rows:', rows.join(', ') || '(none)'); + } } async function nav(u) { pinnedTips.length = 0; await page.goto(absurl(u), { waitUntil: 'load' }); await captureState(); await page.mouse.move(cur.x, cur.y); } async function glide(x, y) { + if (FAST) { await page.mouse.move(x, y); cur.x = x; cur.y = y; return; } const steps = Math.max(10, Math.round(Math.hypot(x - cur.x, y - cur.y) / 9)); for (let i = 1; i <= steps; i++) { await page.mouse.move(cur.x + (x - cur.x) * i / steps, cur.y + (y - cur.y) * i / steps); await sleep(15); } cur.x = x; cur.y = y; } async function glideTo(sel) { const b = await page.locator(sel).first().boundingBox({ timeout: 8000 }).catch(() => null); if (b) await glide(b.x + b.width / 2, b.y + b.height / 2); } async function clickGlide(sel) { await glideTo(sel); await sleep(160); await page.click(sel); } async function checkGlide(sel, want) { await glideTo(sel); await sleep(140); if (want) await page.check(sel).catch(() => {}); else await page.uncheck(sel).catch(() => {}); } // visibly open a native <select>, highlight the target, (optionally) commit, collapse. // commit=false leaves the real value untouched (just shows the pick) -- used for the // track-controls gesture, where the actual state is applied by a follow-up nav(). async function openSelectVisible(sel, val, rows = 12, commit = true) { const loc = page.locator(sel).first(); + if (FAST) { // no one is watching: just set it (the open/highlight is video-only) + if (commit) { await loc.selectOption(val).catch(() => {}); await sleep(120); } + return; + } await loc.evaluate(el => { try { el.scrollIntoView({ block: 'center' }); } catch (e) { el.scrollIntoView(); } }); await sleep(350); await glideTo(sel); await sleep(300); await loc.evaluate((el, r) => { const b = el.getBoundingClientRect(); el.dataset._sz = el.size || 1; el.dataset._cs = el.style.cssText; el.style.position = 'fixed'; el.style.left = b.left + 'px'; el.style.top = b.top + 'px'; el.style.width = 'auto'; el.style.minWidth = b.width + 'px'; el.style.zIndex = '2147483646'; el.style.background = '#fff'; el.style.border = '1px solid #888'; el.style.boxShadow = '0 4px 12px rgba(0,0,0,.35)'; el.size = Math.min(r, el.options.length); }, rows); await sleep(650); await loc.evaluate((el, v) => { const i = [...el.options].findIndex(o => o.value === v); if (i >= 0) { el.selectedIndex = i; try { el.options[i].scrollIntoView({ block: 'center' }); } catch (e) {} } }, val); await sleep(1000); if (commit) { await loc.evaluate((el, v) => { el.value = v; el.dispatchEvent(new Event('change', { bubbles: true })); }, val); await sleep(250); } await loc.evaluate(el => { el.size = parseInt(el.dataset._sz) || 1; el.style.cssText = el.dataset._cs || ''; }).catch(() => {}); } + // ---------- position-box suggestions (the gene-name path of goShow) ---------- + // Rows of the open suggest menu, minus the category headings (and the trailing + // "Unable to find a genome?" div, which is not an <li>). + const SUGGEST_ROW = 'ul.ui-autocomplete li:not(.ui-autocomplete-category):visible'; + async function suggestRows() { + return await page.evaluate(() => { + const rows = [...document.querySelectorAll('ul.ui-autocomplete li')] + .filter(li => !li.classList.contains('ui-autocomplete-category') && li.offsetWidth > 0); + return rows.map(li => { + let d = null; + try { if (window.jQuery) d = jQuery(li).data('ui-autocomplete-item') || null; } catch (e) {} + return { + text: (li.innerText || li.textContent || '').trim(), + // The item's own identifier, preferred over its display label: a gene suggestion + // carries geneSymbol, a genome suggestion (the Convert page's target search) + // carries db/genome. `value` comes LAST because jQuery UI copies the label into + // it when the item has none, which would make every row "match" its own text. + sym: d ? String(d.geneSymbol || d.db || d.genome || d.value || '') : '', + recent: !!(d && d.displayCategory === 'Recent'), // a previously-visited position + }; + }); + }); + } + // Choose the suggestion row for `term`, waiting out the hgSuggest ajax. Two traps this + // handles: the "Recent" positions render INSTANTLY, so a plain "menu is up" wait picks a + // recent position instead of the gene; and when the real suggestions arrive the menu is + // re-rendered, so a row chosen too early is gone by the time we click it. So: poll until + // a row actually matches AND the menu has stopped changing. `want` (goShow's `pick:`) + // overrides the match, for a term with several sensible hits. Returns {index, text} into + // the SUGGEST_ROW set, or null if nothing ever matched. + async function pickSuggest(term, want, ms = 8000) { + const t0 = Date.now(); + const low = s => String(s).toLowerCase(); + const demote = (s, r) => (s ? s - (r.recent ? 0.5 : 0) : 0); // a real hit beats a recent one + // A term that appears as a whole token in the row text counts as a real match -- an + // accession sits in parentheses ("human (HG02148.mat 2021) (GCA_018471535.1)"), so + // neither an id compare nor startsWith would catch it. + const tokenRe = new RegExp(`(^|[^A-Za-z0-9_.])${String(term).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}([^A-Za-z0-9_.]|$)`, 'i'); + const scoreTerm = r => demote( + low(r.sym) === low(term) ? 3 : // the item's own id + low(r.text).startsWith(low(term)) ? 2 : // label starts with the term + tokenRe.test(r.text) ? 2 : // term appears as a token + low(r.text).includes(low(term)) ? 1 : 0, r); + const scoreWant = r => demote(want && low(r.text).includes(low(want)) ? 3 : 0, r); + const pickBest = (rows, fn) => { + let bi = -1, bs = 0; + rows.forEach((r, i) => { const s = fn(r); if (s > bs) { bs = s; bi = i; } }); + return bi < 0 ? null : { index: bi, text: rows[bi].text, score: bs }; + }; + const deadline = Date.now() + ms; + let prev = null, byTerm = null, byWant = null; + while (Date.now() < deadline) { + const rows = await suggestRows(); + const sig = JSON.stringify(rows.map(r => r.text)); + // Same authoring aid as the drawn-row dump: shows what the menu offered and what + // identifier each row carries, which is how you tell why a pick went elsewhere. + if (process.env.DOCENT_ROWS && sig !== prev) console.log(' suggest:', JSON.stringify(rows)); + byTerm = pickBest(rows, scoreTerm) || byTerm; + byWant = want ? (pickBest(rows, scoreWant) || byWant) : null; + const b = want ? byWant : byTerm; + // matched, and the menu settled + if (b && b.score >= 2 && sig === prev) return { ...b, ms: Date.now() - t0 }; + prev = sig; + await sleep(400); + } + // `pick:` that matches nothing shouldn't throw away the gene: fall back to the plain + // term match (and say so) rather than skipping the menu and submitting raw text. + if (want && !byWant && byTerm) + console.warn(`goShow ${term}: pick "${want}" matched no suggestion, using "${byTerm.text}"`); + const out = want ? (byWant || byTerm) : byTerm; + return out ? { ...out, ms: Date.now() - t0, timedOut: true } : null; + } + // ---------- what a `track:` step has to send, per trackDb ---------- + // Cart variables that put `name` in `mode`, as [key, value] pairs: the track itself, its + // subtrack checkbox if trackDb leaves it unselected, and every container above it -- a + // composite or view takes the mode, a superTrack takes show. Nothing is pushed DOWNWARD: + // a container's visibility reaches its selected children on its own (`clinvar=pack` draws + // clinvarMain/clinvarCnv/clinvarSubLolly, `dbSnp155Composite=pack` draws dbSnp155Common), + // which is why the author only has to name deviations from trackDb. A name trackDb doesn't + // know (hub, custom, quickLift target) gets a literal `name=mode`. + async function visVars(name, mode) { + const idx = await tdbIndex(state.db); + const n = idx && idx.get(name); + if (!n) return [[name, mode]]; + if (n.superTrack && !['show', 'hide'].includes(mode)) { + console.warn(`track ${name}: superTrack container takes show/hide, not "${mode}" -- using show`); + mode = 'show'; + } + const out = [[name, mode]]; + // A subtrack of a composite is governed by its CHECKBOX, not its visibility: when the + // container's own vis var is in the same request, hgTracks reshapes the composite and a + // bare `clinvarCnv=hide` is dropped (the cart keeps clinvarCnv_sel=1 and the row still + // draws). So state the selection explicitly in both directions. Views are containers, + // not selectable rows, so they are left out of this. + const parent = n.parent ? idx.get(n.parent) : null; + if (parent && !parent.superTrack && !n.children.length) + out.push([`${name}_sel`, mode === 'hide' ? '0' : '1']); + // Hiding a subtrack says NOTHING about its container: propagating `hide` upward would + // turn the whole composite off (`{clinvar: pack, clinvarCnv: hide}` would end in + // clinvar=hide and no ClinVar at all). + if (mode === 'hide') return out; + for (let p = n.parent; p; ) { + const pn = idx.get(p); + out.push([p, pn && pn.superTrack ? 'show' : mode]); + p = pn ? pn.parent : null; + } + return out; + } + // Leaf descendants of a container, in trackDb order (a container draws no pixels of its + // own, so a `mouseover:`/`click:` naming one has to be resolved to a row that does). + async function tdbLeaves(name) { + const idx = await tdbIndex(state.db); + const n = idx && idx.get(name); + if (!n || !n.children.length) return []; + const out = []; + const walk = k => { + const c = idx.get(k); + if (!c || !c.children.length) out.push(k); + else c.children.forEach(walk); + }; + n.children.forEach(walk); + return out; + } + // The track-controls dropdown to open for the visible gesture. Composite children have no + // dropdown of their own (the container carries it), so walk up until the DOM has one. + async function ctrlSelect(name) { + const idx = await tdbIndex(state.db); + for (let k = name; k; ) { + const sel = `select[name="${k}"]`; + if (await page.locator(sel).count()) return sel; + const n = idx && idx.get(k); + k = n ? n.parent : null; + } + return null; + } async function shot(name) { const p = path.join(STILLDIR, name + '.png'); + // Drop any click ripple still fading: it reads as a red blob over whatever was just + // clicked. It belongs to the video, not to a figure. + await page.evaluate(() => document.querySelectorAll('.__ripple').forEach(e => e.remove())).catch(() => {}); // If a mouseover tooltip is currently up, capture the image + tooltip together // (the tooltip is appended to <body>, so an #imgTbl element shot would clip it). const clip = await page.evaluate(() => { const im = document.getElementById('imgTbl'); if (!im) return null; // Floating overlays to capture together with the image: the mouseover tooltip, // and any visible jQuery-UI dialog (e.g. the drag-select "Zoom In / Highlight" box). const overlays = []; const tip = document.getElementById('mouseoverContainer'); if (tip && tip.offsetWidth > 0 && getComputedStyle(tip).display !== 'none' && getComputedStyle(tip).visibility !== 'hidden') overlays.push(tip); for (const d of document.querySelectorAll('.ui-dialog')) if (d.offsetWidth > 0) overlays.push(d); if (!overlays.length) return null; const a = im.getBoundingClientRect(); let x = a.left, y = a.top, x2 = a.right, y2 = a.bottom; for (const o of overlays) { const r = o.getBoundingClientRect(); x = Math.min(x, r.left); y = Math.min(y, r.top); x2 = Math.max(x2, r.right); y2 = Math.max(y2, r.bottom); } return { x: Math.max(0, x - 4), y: Math.max(0, y - 4), width: (x2 - x) + 8, height: (y2 - y) + 8 }; }); if (clip) { await page.screenshot({ path: p, clip }); + } else if (await page.locator('#imgTbl').count()) { + await page.locator('#imgTbl').screenshot({ path: p }); } else { - const el = (await page.locator('#imgTbl').count()) ? page.locator('#imgTbl') : page.locator('main'); - await el.screenshot({ path: p }); + // Not a tracks page (hgc detail page, an external page a link led to, ...). An element + // shot of <main> would run the whole scrolling page -- thousands of pixels tall and + // unusable as a figure. Capture the viewport only, i.e. the top of the page. + await page.screenshot({ path: p }); } console.log('SHOT', p); await sleep(SHOTHOLD); } // Resolve a track name to its DOM key + data-image and row bounding boxes. On a // quickLift/Convert target the tracks come from a hub, so ids gain a dynamic // `hub_<n>_` prefix -- match by suffix so the YAML can just say `track: quickLiftChain`. + // A container draws nothing itself, so if the name given is one, fall through to the + // leaves trackDb lists under it and take the first that is actually on the page. async function trackBox(t) { - const want0 = imgTrack(t); - const key = await page.evaluate(k => { + const cands = [t, ...await tdbLeaves(t)]; + let key = null; + for (const c of cands) { + key = await page.evaluate(k => { if (document.getElementById('img_data_' + k)) return k; const el = [...document.querySelectorAll('[id^="img_data_"]')] .find(e => e.id === 'img_data_' + k || e.id.endsWith('_' + k)); - return el ? el.id.replace('img_data_', '') : k; - }, want0); + return el ? el.id.replace('img_data_', '') : null; + }, c); + if (key) { if (c !== t) console.log(`track ${t}: drawn by "${c}"`); break; } + } + if (!key) throw new Error(`track "${t}" not shown (no #img_data_ for ${cands.join(', ')})`); const img = await page.locator(`#img_data_${key}`).first().boundingBox({ timeout: 8000 }).catch(() => null); const row = await page.locator(`#imgTbl tr#tr_${key}`).first().boundingBox({ timeout: 8000 }).catch(() => null); if (!img || !row) throw new Error(`track "${t}" not shown (need #img_data_${key} + #tr_${key})`); return { key, img, row }; } // Resolve a NAMED item to a point {x,y} + its map-box HREF (the item's hgc link). We // pick the map <area> whose href(&i=<name>)/title carries the name AND whose box sits in // this track's own ROW band (so stacked items on other rows don't win); fall back to the // JSON mouseOver spans (wig/dense tracks, no per-item href). async function itemXY(t, want, titleOnly) { const { key, img, row } = await trackBox(t); const band = { top: row.y, bot: row.y + row.height }; const area = await page.evaluate(({ want, titleOnly, band, imgBox }) => { const areas = [...document.querySelectorAll('map[name^="map_"] area')]; const cands = []; @@ -305,46 +577,69 @@ } return { x, y: row.y + row.height / 2 }; } // Hover an item to raise its mouseover tooltip (real mousemove -> the browser's own // tooltip). Two ways to place the cursor: // IDENTITY `item:` / `title:` / `value:` -> name the item (lands on the right ROW). // POSITION `at:` (genomic coord) / `frac:` (0..1) / `x:` (raw px) -> a point. async function mouseover(o) { if (typeof o === 'string') o = { track: o }; o = o || {}; const t = o.track; if (!t) throw new Error('mouseover: needs a track'); const want = o.item ?? o.title ?? o.value; // identity mode if any is set const { x, y } = (want != null) ? await itemXY(t, want, o.title != null && o.item == null && o.value == null) : await posXY(t, o); - // Dismiss any tooltip a previous mouseover left open (over the grey side-label strip, - // which has no items) so THIS item raises a FRESH tooltip -- otherwise a back-to-back - // pinned mouseover can capture the previous item's stale popup text. - await page.mouse.move(2, y); cur.x = 2; cur.y = y; await sleep(120); + // Raise a FRESH tooltip for THIS item. Two waits, both on the tooltip's actual state + // rather than on a duration: park over the grey side-label strip (no items there) until + // the previous tooltip is GONE, then hover the item until a tooltip is up whose content + // differs from the one we just dismissed. Sleeping instead only looks right -- with the + // dwells trimmed (FAST) a back-to-back pinned mouseover captured the PREVIOUS item's + // text, and even at full pace that was a race waiting to be lost. + const tipHtml = () => page.evaluate(() => { + const c = document.getElementById('mouseoverContainer'); + if (!c || !c.offsetWidth) return null; + const st = getComputedStyle(c); + return (st.display === 'none' || st.visibility === 'hidden') ? null : c.innerHTML; + }); + const prevTip = await tipHtml(); + await page.mouse.move(2, y); cur.x = 2; cur.y = y; + if (prevTip) await page.waitForFunction(() => { + const c = document.getElementById('mouseoverContainer'); + if (!c || !c.offsetWidth) return true; + const st = getComputedStyle(c); + return st.display === 'none' || st.visibility === 'hidden'; + }, null, { timeout: 2000 }).catch(() => {}); + await sleep(60); await glide(x, y); // small jiggle so the mousemove handler definitely fires and positions the tooltip await page.mouse.move(x + 1, y); await sleep(60); await page.mouse.move(x, y); - await page.waitForSelector('#mouseoverContainer', { state: 'visible', timeout: 2500 }).catch(() => {}); + await page.waitForFunction(prev => { + const c = document.getElementById('mouseoverContainer'); + if (!c || !c.offsetWidth) return false; + const st = getComputedStyle(c); + if (st.display === 'none' || st.visibility === 'hidden') return false; + return prev == null || c.innerHTML !== prev; + }, prevTip, { timeout: 3000 }).catch(() => {}); // Optionally RECORD this tooltip so a later `pinShot:` can show several mouseovers // open together in one figure. We only record (position + the tooltip's own HTML) // here -- nothing is injected into the recorded page, so the mp4 still shows just the // transient native tooltip. `pin:` on the step overrides the document-level // `pinMouseovers:` default. Records accumulate within a view and are cleared on nav. const pin = (o.pin != null) ? o.pin : (doc.pinMouseovers === true); if (pin) await recordTip(x, y); - await sleep(o.hold != null ? Number(o.hold) * 1000 : SHOTHOLD); + if (!FAST) await sleep(o.hold != null ? Number(o.hold) * 1000 : SHOTHOLD); if (o.shot) await shot(o.shot); } // Grab the live mouseover tooltip's HTML and anchor it at the ITEM's coordinate (x,y // that mouseover just hovered), expressed RELATIVE TO the track image (#imgTbl). The // browser parks its own tooltip at a near-fixed spot, so two tips would stack; anchoring // to the item keeps each pinned tooltip on its own feature (and robust to the throwaway // page's image sitting at a different offset). async function recordTip(x, y) { const t = await page.evaluate(({ x, y }) => { const c = document.getElementById('mouseoverContainer'); if (!c || !c.offsetWidth) return null; const im = document.getElementById('imgTbl'); const ir = im ? im.getBoundingClientRect() : { left: 0, top: 0 }; return { dx: x - ir.left + 8, dy: y - ir.top + 8, html: c.outerHTML }; }, { x, y }); @@ -419,32 +714,31 @@ const f = (fr != null) ? Number(fr) : (coord != null ? await coordFrac(coord) : null); if (f == null) throw new Error('drag: need endpoints as coord (from:/to:), frac (fromFrac:/toFrac:) or px (fromX:/toX:)'); return dataLeft + f * dataW; }; const x1 = await endX(o.fromX, o.fromFrac, o.from); const x2 = await endX(o.toX, o.toFrac, o.to); // y band for the drawn selection box. Default: span the FULL track image (like a // real shift+drag, which highlights every track top-to-bottom). A named `track:` // narrows the band to that one row instead. The CURSOR, however, sweeps near the // TOP of the image (over the ruler) — a real drag is horizontal and the highlight // fills downward on its own; sending the cursor to the vertical center would make // it dive through the tracks first. let y = tbl.y + Math.min(90, tbl.height / 2); let y1 = 0, y2 = tbl.height; if (o.track) { - const key = imgTrack(o.track); - const row = await page.locator(`#imgTbl tr#tr_${key}`).first().boundingBox().catch(() => null); + const row = await trackBox(o.track).then(b => b.row).catch(() => null); if (row) { y = row.y + row.height / 2; y1 = row.y - tbl.y; y2 = row.y - tbl.y + row.height; } } // The selected genomic range: from coord endpoints we already have it; otherwise // derive it from the view fractions below (inside the browser). const coordNum = v => +String(v).replace(/.*:/, '').replace(/,/g, ''); let posStr = null; if (typeof o.from === 'string' && o.from.includes(':') && o.to != null) { const chrom = o.from.split(':')[0], a = coordNum(o.from), b = coordNum(o.to); posStr = `${chrom}:${Math.min(a, b)}-${Math.max(a, b)}`; } // Visible cursor sweep across the selection (no button-down — a real drag would // just PAN the image). The drag-select band is grown UNDER the cursor as it moves, // so it reads like a genuine shift+drag rather than popping in at the end; the // dialog is then raised via the browser's own dragSelect entry point, the same way // highlight_shot.js does it. @@ -507,46 +801,102 @@ if (/^GC[AF]_/i.test(to)) return to; const matches = await page.$$eval('#hglft_toDbSelect option', (os, q) => { const norm = t => t.toLowerCase().replace(/[^a-z0-9]+/g, ' '); const toks = norm(q).trim().split(' ').filter(Boolean); return os.filter(o => toks.every(tk => norm(o.text).includes(tk))).map(o => ({ v: o.value, t: o.text })); }, String(to)); if (!matches.length) throw new Error('convert target not found in Assembly dropdown: ' + to); if (matches.length > 1) { console.warn(`WARNING: "${to}" matches ${matches.length} assemblies; using the first. Use an exact accession to disambiguate:`); matches.forEach(m => console.warn(` ${m.v} ${m.t}`)); } return matches[0].v; } async function convert(o) { o = o || {}; + // `shot:` here can name up to three moments of the Convert page, none of which any + // other verb can reach (after Submit the tour is already on the results page): + // shot: convert_filled -- just before Submit (the common one) + // shot: {opened: a, filled: b, result: c} + // opened the page as it comes up, nothing chosen yet + // filled target searched for, QuickLift/Hide-defaults set -- ready to Submit + // result the conversion result page (the coordinate link `open: lift` clicks) + const shots = (o.shot == null) ? {} + : (typeof o.shot === 'string' ? { filled: o.shot } : o.shot); + for (const k of Object.keys(shots)) + if (!['opened', 'filled', 'result'].includes(k)) + console.warn(`convert shot: unknown moment "${k}" (use opened, filled or result)`); try { - await glideTo('#view'); await page.hover('#view'); await sleep(900); + await glideTo('#view'); await page.hover('#view'); await dwell(900); await clickGlide('a#convertMenuLink'); await page.waitForSelector('#hglft_toDbSelect', { timeout: 8000 }); } catch (e) { await nav(`/cgi-bin/hgConvert?hgsid=${state.hgsid}&db=${state.db}&position=${enc(state.position)}`); await page.waitForSelector('#hglft_toDbSelect'); } + if (shots.opened) await shot(shots.opened); + // Find the target the way a user does: TYPE it into the Convert page's own "Search for + // target genome" bar and click the suggestion (the species autocomplete takes an + // accession or a name, and is already filtered to assemblies hg38 can lift to). So the + // string the script names is visibly searched for on screen, not silently selected. + // `search:` overrides what gets typed; `pick:` disambiguates the menu. + const term = o.search != null ? o.search : o.to; + let searched = false; + if (term != null && await page.locator('#toGenomeSearch:visible').count()) { + await glideTo('#toGenomeSearch'); await page.click('#toGenomeSearch'); await dwell(160); + await page.fill('#toGenomeSearch', ''); + const label0 = await page.locator('#toGenomeLabel').textContent().catch(() => ''); + await typeIn(page, '#toGenomeSearch', term); + await dwell(300); + const hit = await pickSuggest(String(term), o.pick); + if (hit) { + console.log(`convert: searched "${term}" -> "${hit.text}"` + + (hit.timedOut ? ` (no strong match, waited ${hit.ms}ms)` : '')); + const li = page.locator(SUGGEST_ROW).nth(hit.index); + const b = await li.boundingBox().catch(() => null); + if (b) await glide(b.x + b.width / 2, b.y + b.height / 2); + await dwell(220); + await li.click(); + // Picking a genome repopulates the Assembly dropdown (ajax) and rewrites the hidden + // toDb field. Wait for the "Selected:" label to actually change instead of guessing + // a duration -- right on a fast server, still correct on a slow one. + await page.waitForFunction(b0 => { + const el = document.getElementById('toGenomeLabel'); + return el && (el.textContent || '') !== b0; + }, label0, { timeout: 5000 }).catch(() => {}); + await dwell(300); + searched = true; + } else { + console.warn(`convert: "${term}" matched nothing in the target-genome search`); + } + } + // The Assembly dropdown is what actually gets submitted, so confirm it landed on the + // requested assembly; open it visibly only if the search didn't get us there. const val = await resolveTarget(o.to); + const landed = await page.locator('#hglft_toDbSelect').inputValue().catch(() => null); + if (!searched || landed !== val) { + if (searched) console.log(`convert: Assembly dropdown is on "${landed}", picking ${val}`); await openSelectVisible('#hglft_toDbSelect', val); + } await page.waitForSelector('#doQuickLift', { timeout: 8000 }).catch(() => {}); - await sleep(400); + await dwell(400); if (o.quicklift !== false) await checkGlide('#doQuickLift', true); await checkGlide('#hideTracksOnConvert', o.hideDefaults !== false); // reverts on assembly change -> set explicitly - await sleep(300); + await dwell(300); + if (shots.filled) await shot(shots.filled); await clickGlide('#hglft_doConvert'); await page.waitForLoadState('load'); await captureState(); + if (shots.result) await shot(shots.result); } // On the "Hub Connect Successful" page, click the "Open:" link for `db` so the demo // ends on the browser with the hub loaded. The links look like // hgTracks?hubUrl=...&db=<genome>&position=lastDbPos (or &genome=<genome>). async function openHubAssembly(db) { const sel = await page.evaluate((db) => { const esc = db.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); const re = new RegExp('[?&](?:db|genome)=' + esc + '(?:&|$)'); // The connect page renders the genome list twice (a short + a collapsed long copy), // so require the link to be VISIBLE, not just the first match in DOM order. const a = [...document.querySelectorAll('a[href*="hgTracks"]')] .find(a => re.test(a.getAttribute('href') || '') && a.getClientRects().length > 0); if (!a) return null; if (!a.id) a.id = '__hubOpen'; @@ -558,44 +908,146 @@ await captureState(); return true; } function norm(step) { if (typeof step === 'string') { const [v, ...r] = step.trim().split(/\s+/); return { verb: v, arg: r.length ? r.join(' ') : true }; } const k = Object.keys(step)[0]; return { verb: k, arg: step[k] }; } async function run({ verb, arg }) { switch (verb) { case 'gateway': await nav(`/cgi-bin/hgGateway?db=${state.db}`); break; case 'go': if (arg === true || arg === '' || arg == null) { await clickGlide('.jwGoButtonContainer'); await page.waitForSelector('#imgTbl'); } else { await nav(`/cgi-bin/hgTracks?db=${state.db}&position=${enc(arg)}&pix=${PIX}`); } await captureState(); break; + case 'goShow': { + // DEMONSTRATE the position change through the UI (vs. `go:` which navs straight to + // the new position): glide to the position box, clear it, type on screen, then let + // the page take it from there -- "Search" (#goButton) on hgTracks, the arrow + // (.jwGoButtonContainer) on hgGateway, so one verb covers either page. + // + // Takes a POSITION or a GENE NAME (or any search term the box accepts: HGVS, an + // accession, ...). A gene name goes through the browser's own suggestion menu, the + // way a user does it: wait for the menu, then click the matching item -- hgTracks' + // handler sets the position from that item and submits, so we land on the gene + // instead of the search-results page. `pick:` chooses among the suggestions when + // the term is ambiguous (substring of the menu row); default is an exact symbol + // match, else the first row. + // + // goShow: BRCA1 + // goShow: {gene: SHH, shot: source} + // goShow: {position: "chr7:155,799,529-155,812,871", shot: source} + // goShow: {gene: BRCA1, pick: "NM_007294", shot: source} + const o = (typeof arg === 'string') ? { position: arg } : (arg || {}); + const pos = [o.position, o.pos, o.gene, o.search].find(v => v != null && v !== ''); + if (pos == null) { console.warn('goShow: no position or gene given'); break; } + const term = String(pos).trim(); + // A coordinate has no suggestions to wait on; anything else is a search term. + const isPos = /^[\w.|-]+:[\d,]+(-[\d,]+)?$/.test(term); + if (!await page.locator('#positionInput:visible').count()) + throw new Error('goShow: no position box on this page (need hgTracks or hgGateway) -- ' + page.url()); + await glideTo('#positionInput'); await page.click('#positionInput'); await dwell(160); + await page.fill('#positionInput', ''); // clear the old position + await dwell(200); + await typeIn(page, '#positionInput', term); // visible typing + await dwell(450); + pinnedTips.length = 0; // new view, old tips don't apply + let done = null; + if (!isPos) { + const hit = await pickSuggest(term, o.pick != null ? o.pick : o.match); + if (hit) { + console.log(`goShow ${term}: suggestion "${hit.text}"` + + (hit.timedOut ? ` (no strong match, waited ${hit.ms}ms)` : '')); + // Address the row by index at click time (a late re-render replaces the <li> + // elements, so a handle or a marker attribute taken earlier goes stale). + const li = page.locator(SUGGEST_ROW).nth(hit.index); + // Arm the nav wait BEFORE the click (the page's own handler submits the form for + // us), and wait on 'commit' so we can tell "navigated" from "only filled the box". + const committed = page.waitForNavigation({ waitUntil: 'commit', timeout: 15000 }).catch(() => null); + const b = await li.boundingBox().catch(() => null); + if (b) await glide(b.x + b.width / 2, b.y + b.height / 2); + await sleep(220); + await li.click(); + done = await committed; + if (done) await page.waitForLoadState('load').catch(() => {}); + } else { + console.warn(`goShow ${term}: no suggestion matched, submitting the term as typed`); + } + } + // Coordinate, no suggestions, or hgGateway (where picking a suggestion only fills + // the box): click the page's own go button. Arm the nav wait BEFORE the click -- on + // hgTracks the OLD page already has an #imgTbl, so waiting on the selector alone + // returns instantly and a following shot races the reload ("Cannot find context + // with specified id"). + if (!done) { + const navDone = page.waitForNavigation({ waitUntil: 'load', timeout: 30000 }).catch(() => {}); + await clickGlide((await page.locator('#goButton').count()) ? '#goButton' : '.jwGoButtonContainer'); + await navDone; + } + // A unique hit lands on the track image; a term with no suggestion and several + // matches lands on the search-results page instead, which has no #imgTbl -- that's + // legal, the script can `click` a result from there. + await page.waitForSelector('#imgTbl', { timeout: 15000 }).catch(() => {}); + await captureState(); + await page.mouse.move(cur.x, cur.y); // re-show the cursor overlay + if (o.shot) { await shot(o.shot); return; } + break; + } case 'hide': if (arg === 'all' || arg === true) { await clickGlide('#hgt\\.hideAll'); await page.waitForSelector('#imgTbl'); } break; case 'track': { - const parts = []; - for (const [name, mode] of Object.entries(arg)) { - // Visible gesture: drive the real track-controls dropdown so the mouse is - // seen turning the track on. State is still applied by the nav() below - // (so composite "clean" configs come through), so this open is non-committing. - if (doc.trackAnim !== false) { - const csel = `select[name="${ctrlName(name)}"]`; - if (await page.locator(csel).count()) await openSelectVisible(csel, mode, 6, false); + const entries = Object.entries(arg); + const named = new Set(entries.map(([n]) => n)); // what the author spelled out + const idx = await tdbIndex(state.db); + // Visible gesture first: drive the real track-controls dropdowns so the mouse is + // seen turning the tracks on. State is still applied by the nav()s below (which + // carry the container/checkbox vars too), so these opens are non-committing. + if (doc.trackAnim !== false) + for (const [name, mode] of entries) { + const csel = await ctrlSelect(name); + if (csel) await openSelectVisible(csel, mode, 6, false); + } + // hgTracks RESHAPES a composite when its container visibility changes, and that wipes + // per-subtrack overrides arriving in the same request (`clinvar=pack&clinvarCnv=hide` + // leaves clinvarCnv_sel=1 and the CNV row still drawn). So a step that names both a + // composite and something under it is applied in rounds -- container first, then the + // deviations -- which is exactly what writing them as two steps does. superTracks + // don't reshape, so they don't force a round. + const rounds = new Map(); + for (const e of entries) { + let d = 0; + for (let k = e[0]; ;) { + const n = idx && idx.get(k); + if (!n || !n.parent) break; + const p = idx.get(n.parent); + if (named.has(n.parent) && !(p && p.superTrack)) d++; + k = n.parent; } - const fn = TRACKS[name]; parts.push(...(fn ? fn(mode) : [`${name}=${mode}`])); + if (!rounds.has(d)) rounds.set(d, []); + rounds.get(d).push(e); } + for (const d of [...rounds.keys()].sort((a, b) => a - b)) { + const vars = new Map(); + for (const [name, mode] of rounds.get(d)) + // A derived variable never overrides one the step names itself, whatever the + // order: `{clinvar: pack, clinvarCnv: hide}` keeps clinvar=pack. + for (const [k, v] of await visVars(name, mode)) + if (k === name || !named.has(k)) vars.set(k, v); + const parts = [...vars].map(([k, v]) => `${k}=${v}`); + console.log('track:', parts.join(' ')); // what trackDb turned the step into await nav(`/cgi-bin/hgTracks?db=${state.db}&position=${enc(state.position)}&${parts.join('&')}&pix=${PIX}`); + } break; } case 'convert': await convert(arg); break; case 'hub': { // Attach a track hub by URL: hgTracks?hubUrl=... connects the hub and makes its // tracks available at their hub-declared visibility. Follow with `track:` to turn // specific ones on. Accepts a bare URL or {url:, db:, position:}. const o = (typeof arg === 'string') ? { url: arg } : (arg || {}); if (!o.url) { console.warn('hub: no url given'); break; } const db = o.db || state.db; const pos = o.position != null ? o.position : state.position; const parts = [`db=${db}`, `hubUrl=${enc(o.url)}`]; if (pos) parts.push(`position=${enc(pos)}`); parts.push(`pix=${PIX}`); await nav(`/cgi-bin/hgTracks?${parts.join('&')}`); @@ -726,33 +1178,52 @@ await clickGlide(arg); await page.waitForLoadState('load').catch(() => {}); await captureState(); } break; case 'hover': await glideTo(arg); await page.hover(arg); break; case 'wait': await page.waitForSelector(arg, { timeout: 15000 }); break; case 'sleep': await sleep(Number(arg)); return; default: console.warn('unknown verb:', verb); } await sleep(PACE); } if (doc.reset) await page.goto(absurl('/cgi-bin/cartReset?skipLs=1'), { waitUntil: 'domcontentloaded' }); const steps = doc.steps || []; + const timing = []; for (let i = 0; i < steps.length; i++) { const s = norm(steps[i]); + const t0 = Date.now(); try { await run(s); } catch (e) { console.error(`step ${i + 1} (${s.verb}) failed:`, e.message); await ctx.close(); await browser.close(); process.exit(1); } + timing.push({ n: i + 1, verb: s.verb, ms: Date.now() - t0 }); } await page.waitForTimeout(300); await ctx.close(); await browser.close(); + // Where did the wall clock go? DOCENT_TIME=1 prints the per-step table -- the dwells + // (pace/shotHold) and the page loads dominate, which is what FAST=1 trims. + if (process.env.DOCENT_TIME) { + const tot = timing.reduce((a, t) => a + t.ms, 0); + console.log(`--- steps: ${(tot / 1000).toFixed(1)}s total`); + for (const t of [...timing].sort((a, b) => b.ms - a.ms)) + console.log(` ${(t.ms / 1000).toFixed(1)}s step ${t.n} ${t.verb}`); + } + if (FAST) { + console.log('DONE (fast: stills only, no mp4) -> stills in', STILLDIR, + `| ${((Date.now() - T_START) / 1000).toFixed(0)}s`); + return; + } + const tVid = Date.now(); // transcode webm -> silent mp4 const vdir = path.join(HERE, '.vid_' + base); const webm = fs.readdirSync(vdir).filter(f => f.endsWith('.webm')).map(f => path.join(vdir, f)).sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs)[0]; const FF = execFileSync('python3', ['-c', 'import imageio_ffmpeg,sys;sys.stdout.write(imageio_ffmpeg.get_ffmpeg_exe())']).toString().trim(); execFileSync(FF, ['-y', '-loglevel', 'error', '-i', webm, '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-crf', '22', '-preset', 'veryfast', '-movflags', '+faststart', OUTMP4]); fs.rmSync(vdir, { recursive: true, force: true }); - console.log('DONE ->', OUTMP4, '| stills in', STILLDIR); + if (process.env.DOCENT_TIME) console.log(`--- mp4 transcode: ${((Date.now() - tVid) / 1000).toFixed(1)}s`); + console.log('DONE ->', OUTMP4, '| stills in', STILLDIR, + `| ${((Date.now() - T_START) / 1000).toFixed(0)}s`); })().catch(e => { console.error(e); process.exit(1); });