7e89607805cac22b52143b23a45464759e66a957 braney Wed Sep 16 12:36:15 2026 -0700 docent: a login: step, with the account kept per hgcentral, refs #37892 hgCollection refuses a visitor who is not signed in -- hgCollection.c doMiddle, "You must be logged in to edit collections" -- and so does the saving half of hgSession. The suite has never had a logged-in page, and until now could not: the login cookie is validated against a salted hash (login.cookieSalt, hg/lib/wikiLink.c), so there is no cookie to hand the browser. A script that needs one of those pages has to sign in the way a person does. `login:` does that, through hgLogin's own form, and it takes no credentials and cannot be given any. They come from ~/.docentLogin (DOCENT_LOGIN_FILE overrides, DOCENT_LOGIN_USER + DOCENT_LOGIN_PASSWORD override both for one run), which is refused unless it is mode 0600 -- the rule hg/lib/hgConfig.c applies to hg.conf, for the same reason. Nothing prints a password. THE FILE IS KEYED BY HGCENTRAL DATABASE, not by server. An account is a row in gbMembers in one central, the way a named session is: genome-test, hgwdev, every hgwdev- sandbox and every ticket park read hgcentraltest and share one account, while hgwbeta reads hgcentralbeta and the RR reads hgcentral. So the file is one [hgcentraltest] section rather than a section per sandbox. Which central a server reads is READ from its hg.conf, following its includes the way hgConfig.c does, and not guessed from the host -- because a sandbox can point itself somewhere else and two on hgwdev do today: of the personal confs there, 45 set central.db=hgcentraltest, one sets hgcentralgsid and one hgcentralbeta. A server whose conf is on another machine falls back to a small table (the RR, the two mirrors, hgwbeta), and [default] catches the rest. A run redirected with DOCENT_TARGET looks up the server it is really driving. Two failures the step has to tell apart, and both cost a run to find: A WRONG PASSWORD IS A PERFECTLY GOOD PAGE. hgLogin answers one by drawing the same form again with a red message, so a step that just navigated on would leave every later step running logged out and the failure would surface somewhere else entirely. The step fails on #accountLoginForm still being there, and quotes what the page said. A RIGHT PASSWORD ARRIVES MID-REDIRECT. hgLogin answers one with a page that navigates ITSELF a moment later: returnToURL(150) at hgLogin.c:1160 writes setTimeout(function(){location=...}, 150). Returning while that timer is pending means the next step's goto: races it and the browser aborts one of the two, which arrives as a bare net::ERR_ABORTED on a URL that is completely fine. The step now waits for that redirect to land. The failure path is checked first, since that page never leaves hgLogin and there is no redirect to wait for. preflight reports the account as a fixture -- which central it resolved, which section it came from, and why it is unusable if it is -- so a missing password is caught before the browser starts. It attempts no login: a wrong password fails loudly at the step itself, which is the one thing preflight cannot do for it. The hg.conf reader is now in both docent.js and tests/preflight.js, beside the resolveTarget each of them already carries. Both say to keep the other in step. If that second copy is a copy too many, the two of them want a shared module. diff --git src/hg/utils/docent/docent.js src/hg/utils/docent/docent.js index 15e8170e720..6efe9200c2e 100755 --- src/hg/utils/docent/docent.js +++ src/hg/utils/docent/docent.js @@ -1,2266 +1,2445 @@ #!/usr/bin/env node /* docent.js SCRIPT.docent.yaml [OUT.mp4] * * Docent -- a language for authoring guided tours of the UCSC Genome Browser. * * Render a hand-authored Docent script into a silent mp4 PLUS a named still PNG at * every `shot:` marker (the figures). One source script -> both outputs, so a figure * is literally a frame of the tour. Reuses the shared Playwright/Chromium install in * /hive/groups/browser/uiTest/pw. * * PW=/hive/groups/browser/uiTest/pw * PLAYWRIGHT_BROWSERS_PATH=$PW/browsers NODE_PATH=$PW/node_modules \ * node docent.js AP1.docent.yaml * * The high-level verbs bake in the quickLift/Convert mechanics (dbSNP composite * params, the hideDefaults-reverts-on-assembly-change bug, target lookup) so the * author writes intent, not selectors. See README.md for the language. * * The surface syntax is YAML so ordinary editors highlight it; the language is the * verb vocabulary layered on top, not the serialization. Scripts are named * .docent.yaml (a bare .docent also works). */ const { chromium } = require('playwright'); const yaml = require('js-yaml'); const fs = require('fs'); const os = require('os'); const path = require('path'); const { execFileSync } = require('child_process'); // ---------- parse script + config ---------- const SCRIPT = process.argv[2]; if (!SCRIPT) { console.error('usage: node docent.js SCRIPT.docent.yaml [OUT.mp4]'); process.exit(2); } const doc = yaml.load(fs.readFileSync(SCRIPT, 'utf8')) || {}; // Lint: in a YAML flow map a colon needs a trailing space, so `{item:name5568747}` // parses as ONE key "item:name5568747" (value null) and the intended `item:` arg is // silently dropped -- the verb then quietly falls back to a default. Catch that here // (before the long browser run) by flagging any arg key that contains a ':'. (function lintSteps(steps) { let n = 0; const scan = (obj, where) => { if (!obj || typeof obj !== 'object') return; for (const k of Object.keys(obj)) { if (typeof k === 'string' && k.includes(':')) { n++; console.warn(`WARNING ${where}: key "${k}" contains ':' -- a YAML flow map needs a ` + `space after the colon. Did you mean "${k.replace(/:(?=\S)/, ': ')}"? ` + `(this key is being IGNORED, so the verb may fall back to a default)`); } scan(obj[k], where); } }; (steps || []).forEach((s, i) => { if (s && typeof s === 'object') { const v = Object.keys(s)[0]; scan(s[v], `step ${i + 1} (${v})`); } }); if (n) console.warn(`(${n} suspicious key${n > 1 ? 's' : ''} above -- fix the missing space, or the arg is dropped)`); })(doc.steps); const HERE = path.dirname(path.resolve(SCRIPT)); const base = path.basename(SCRIPT).replace(/\.(docent\.)?ya?ml$/i, '').replace(/\.docent$/i, ''); const FIGDIR = path.resolve(HERE, '..'); // figures dir beside the scripts const OUTMP4 = process.argv[3] || doc.mp4 || path.join(FIGDIR, base + '.mp4'); // Stills go to stills//. DOCENT_STILLS names a different PARENT ("stills.hires"), // which is how a high-resolution run keeps its figures beside the screen-resolution ones // instead of overwriting them. const STILLPARENT = process.env.DOCENT_STILLS; const STILLDIR = STILLPARENT ? path.resolve(HERE, STILLPARENT, base) : doc.stills ? path.resolve(HERE, doc.stills) : path.join(HERE, 'stills', base); // Saved sessions go to sessions//, beside stills/. `sessions:` and DOCENT_SESSIONS // name the PARENT (not the per-scenario directory), so `sessionUrlBase:` below always maps // onto it as /.txt, and a print run keeps its files out of the screen run's way. const SESSDIR = path.join( path.resolve(HERE, process.env.DOCENT_SESSIONS || doc.sessions || 'sessions'), base); // `target:` takes a shorthand from this table, a bare `hgwdev-` sandbox name // (expanded below), or a full https://.../cgi-bin URL. Default is genome-test, so a // script that forgets to say where it runs does not silently hit someone's sandbox. const SERVERS = { 'rr': 'https://genome.ucsc.edu/cgi-bin', 'genome-test': 'https://genome-test.gi.ucsc.edu/cgi-bin', 'hgwdev': 'https://hgwdev.gi.ucsc.edu/cgi-bin', 'hgwbeta': 'https://hgwbeta.soe.ucsc.edu/cgi-bin', }; const resolveTarget = t => { if (!t) return SERVERS['genome-test']; if (SERVERS[t]) return SERVERS[t]; if (/^hgwdev-[a-z0-9._-]+$/i.test(t)) return `https://${t}.gi.ucsc.edu/cgi-bin`; // personal sandbox return t; // full URL }; // DOCENT_TARGET overrides `target:` for the whole run, so a suite written against one // server can be pointed at another -- a sandbox, a ticket park, a demo browser -- without // editing the scripts it is written from. The trackDb cache below keys on SERVER, so a // redirected run cannot read back a listing fetched from the server the script names. const SERVER = resolveTarget(process.env.DOCENT_TARGET || doc.target).replace(/\/$/, ''); // SCALE: the same tour rendered at k times the resolution, for figures that have to print. // Nothing is upscaled -- a still only ever has the pixels it was drawn with -- so each layer // is asked to draw k times as many while the layout is left alone: // // * deviceScaleFactor: k. The viewport keeps its 1x CSS size, so the page lays out exactly // as at 1x -- same line breaks, same jQuery-dialog width, same tooltip placement -- and // every bit of it is rasterized with k times the pixels. The retina case, natively. // * `pix` x k, so the server draws the browser image k times as wide, with `textSize` // stepped up to match so hgTracks makes the SAME layout decisions in that bigger image: // same tick spacing, same room for labels, same packing of features into rows. Without // the font, a wider image is a different picture rather than a bigger one. // * `zoom: 1/k` on the image table (SCALE_INIT below), handing that k-times-wider image // back the 1x amount of layout space. One image pixel then falls on exactly one device // pixel: native resolution, no resampling anywhere in the path. // // So the still comes out k times the 1x still in each dimension, showing the same figure -- // not a variation of it rendered in a bigger window. const SCALE = Math.max(1, Number(process.env.DOCENT_SCALE || doc.scale || 1)); const [VW, VH] = doc.size || [1000, 760]; const PIX = Math.round((doc.pix || 850) * SCALE); // hgTracks offers a fixed ladder of track font sizes (hgTracks/config.c); step to the one // closest to scaling its 8px default, so rows and labels grow with the image instead of // staying 8px tall in a 3x-wide picture. const TEXTSIZE = [6, 8, 10, 12, 14, 18, 24, 34].reduce((a, b) => Math.abs(b - 8 * SCALE) < Math.abs(a - 8 * SCALE) ? b : a); // What every hgTracks nav carries: the image width, plus the font to draw it with at scale. const IMGVARS = `pix=${PIX}` + (SCALE > 1 ? `&textSize=${TEXTSIZE}` : ''); // The tooltip's font-size, forced back to what a 1x run gives it (see SCALE_INIT). hgTracks // takes it from the browser text size, which is TEXTSIZE on a scaled run, and then the device // pixel ratio scales it a second time. const SCALE_ARGS = { k: SCALE, tipPx: Math.round(TEXTSIZE / SCALE) }; // `pix` makes the image k times WIDER; nothing makes a fixed-height track taller. A bigLolly // or wiggle row is a pixel count (`DEFAULT_HEIGHT_PER` = 128 in hg/inc/wiggle.h), read from // trackDb/the cart and untouched by `pix` or `textSize` -- so a 128px row that was 15% of an // 850px image is 5% of a 2550px one, which is how the ClinVar lollipop row came out a sliver // with unreadable y-axis labels. A print run therefore asks for k times the height of every // track a `track:` step turns on. It is harmless where it means nothing (a bigBed never reads // heightPer) and each track's own `maxHeightPixels` still clamps it, so a track that should // stay short does -- raise that ceiling in trackDb for one that should not. // The k*128 is the DEFAULT height scaled, not each track's own: a row configured at 50px or // 300px gets k*128 too, which is proportional only for the tracks that took the default. That // covers every track a tour has used so far, and going further would mean reading each row's // heightPer out of the cart before asking for k times it. If a tour ever wants a figure of a // deliberately short or tall row, that is the fix -- the symptom is a row that comes back the // wrong size in a scaled still and the right size at 1x. const HEIGHTPER = SCALE > 1 ? Math.round(128 * SCALE) : 0; // FAST: iterate on the FIGURES. Everything that exists only for the video is dropped -- // the dwells, the cursor animation, the dropdown theatrics, the screen recording and the // mp4 transcode. The stills are byte-for-byte what a full run produces, and a run costs // roughly a third as long. `fast: true` in the script, DOCENT_FAST=1, or `make FAST=1 BP1`. // A scaled run is a figure run: at 3x the video would be a 3000px-wide recording of a tour // nobody watches at that size, so the mp4 is skipped and only the stills are produced. Build // the video from an unscaled run of the same script. const FAST = !!(doc.fast || process.env.DOCENT_FAST || SCALE > 1); const PACE = FAST ? 0 : Math.round((doc.pace ?? 1.2) * 1000); // dwell after each step const SHOTHOLD = FAST ? 0 : Math.round((doc.shotHold ?? 2.2) * 1000); // extra pause at a shot // ---------- trackDb ---------- // Docent carries NO table of per-track cart variables. Such a table encodes one snapshot // of trackDb and then quietly lies when trackDb changes (this file used to pin // `clinvarMain=dense` for every `clinvar:` step, for instance). Instead ask the server for // the trackDb it is driving -- hubApi /list/tracks -- and derive what a step needs: // // * the containers above a subtrack (composite, view, superTrack) that have to be // turned on with it, and what each of those takes (a superTrack wants show/hide), // * whether trackDb leaves that subtrack UNSELECTED (`parent off`), in which case // the subtrack checkbox `_sel` has to come along, // * which leaf actually draws the pixels for a container name. // // Everything else -- which dropdown to open, which row to hover -- is read off the live // page. Tracks the listing doesn't know (attached hubs, custom tracks, quickLift's own // tracks on the target) fall back to a literal `name=mode`, which is all Docent could // honestly do for them anyway. const TDB_TTL = 24 * 3600 * 1000; // re-fetch a cached listing daily const TDB_CACHE = path.join(os.tmpdir(), `docent-tdb-${SERVER.replace(/[^\w.-]+/g, '_')}`); let tdbPending = null; // db -> Promise, once each function tdbParse(genome) { const idx = new Map(); const add = (name, o) => { const p = String(o.parent || '').trim().split(/\s+/); idx.set(name, { name, parent: p[0] || null, parentState: (p[1] || '').toLowerCase(), // on | off | a visibility | '' vis: o.visibility || null, view: o.view || null, superChild: !!o.superTrack, // member of a superTrack superTrack: false, // set below for containers children: [], }); for (const [k, v] of Object.entries(o)) if (v && typeof v === 'object' && !Array.isArray(v)) add(k, v); }; for (const [k, v] of Object.entries(genome || {})) if (v && typeof v === 'object' && !Array.isArray(v)) add(k, v); // hubApi flattens superTrack members to the top level and never lists the superTrack // itself, so synthesize the container from the `parent` field it points at. for (const n of [...idx.values()]) { if (!n.parent) continue; if (!idx.has(n.parent)) idx.set(n.parent, { name: n.parent, parent: null, parentState: '', vis: null, view: null, superChild: false, superTrack: true, children: [] }); idx.get(n.parent).children.push(n.name); } return idx; } // The cache holds the DERIVED index (a few hundred kB), not hubApi's reply (~30 MB for // hg38): same information for our purposes, ~20x less to read and parse on every run. function tdbFlatten(idx) { return { docentIndex: 1, rows: [...idx.values()].map(n => [n.name, n.parent, n.parentState, n.vis, n.view, n.superChild ? 1 : 0, n.superTrack ? 1 : 0]) }; } function tdbInflate(o) { const idx = new Map(); for (const [name, parent, parentState, vis, view, superChild, superTrack] of o.rows) idx.set(name, { name, parent, parentState, vis, view, superChild: !!superChild, superTrack: !!superTrack, children: [] }); for (const n of idx.values()) if (n.parent && idx.has(n.parent)) idx.get(n.parent).children.push(n.name); return idx; } async function tdbIndex(db) { if (!tdbPending) tdbPending = new Map(); if (tdbPending.has(db)) return tdbPending.get(db); const p = (async () => { const cache = `${TDB_CACHE}-${db}.json`; try { const st = fs.statSync(cache); if (Date.now() - st.mtimeMs < TDB_TTL) { const o = JSON.parse(fs.readFileSync(cache, 'utf8')); return o && o.docentIndex ? tdbInflate(o) : tdbParse(o); } } catch (e) {} const url = `${SERVER}/hubApi/list/tracks?genome=${enc(db)}&trackLeavesOnly=0`; try { const r = await fetch(url); if (!r.ok) throw new Error(`HTTP ${r.status}`); const j = await r.json(); const genome = j[db]; // A hub-supplied genome (an assembly hub, or quickLift's own generated target hub) // isn't in the server's trackDb listing at all -- expected, not a failure. if (!genome) { console.log(`trackDb: ${db} is not a server assembly (hub genome), ` + `so track steps there are sent as literal name=mode`); return null; } const idx = tdbParse(genome); // Write via a unique temp file + rename so parallel builds (make -j) can't read a // half-written cache. try { const tmp = `${cache}.${process.pid}.tmp`; fs.writeFileSync(tmp, JSON.stringify(tdbFlatten(idx))); fs.renameSync(tmp, cache); } catch (e) {} console.log(`trackDb: ${idx.size} tracks for ${db} from ${SERVER}/hubApi`); return idx; } catch (e) { console.warn(`trackDb: could not read ${url} (${e.message}) -- ` + `falling back to literal name=mode for every track step`); return null; } })(); tdbPending.set(db, p); return p; } const sleep = ms => new Promise(r => setTimeout(r, ms)); // A pause that exists only so a viewer can follow the video: skipped entirely in FAST. const dwell = ms => (FAST ? Promise.resolve() : sleep(ms)); // Typing is shown on screen for the video; in FAST just put the text in the box (fill() // still fires the input events the autocompletes listen for). async function typeIn(pg, sel, text) { if (FAST) await pg.fill(sel, String(text)); else await pg.type(sel, String(text), { delay: 45 }); } const enc = s => encodeURIComponent(String(s)); const state = { db: doc.db || 'hg38', position: doc.position || '', hgsid: '' }; // ---------- SCALE: give the k-times-wider browser image the 1x amount of layout space ---------- // hgTracks sizes the image table in the HTML it writes, so shrinking the elements alone // would leave the table 3x wider than its own contents. Zoom the table: the zoom reaches the // images inside it, and its box goes back to the width the 1x page gives it -- 854 CSS px for // a pix=850 run, whatever k is -- so the rest of the page is laid out exactly as at 1x while // the image keeps all k times its pixels (one per device pixel at deviceScaleFactor: k). // // Installed for the whole run rather than at the shutter: the tour's own geometry -- a drag // across the image, a mouseover on a feature -- then works in the same coordinates a 1x run // works in, and needs no scale arithmetic of its own. // // The tooltip needs the opposite correction. It is a DOM element, so deviceScaleFactor // already draws it k times bigger -- and hgTracks sets its font-size from the BROWSER TEXT // SIZE (`window.browserTextSize` -> hg/js/utils.js addMouseover, hgTracks.js #mouseOverText), // which a print run has just multiplied by k for the image. Both scalings land on the same // text, so a 3x still gets a tooltip 3x too big -- the popups swamp the figure and the last // one pinned falls off the crop. Pin the font-size back to the 1x value (TEXTSIZE / k); it // is written as an inline style, so the rule has to be !important to win. `.__pinnedTip` is // a recorded tooltip re-injected by pinShot() (which strips the id, keeps the class). const SCALE_INIT = ({ k, tipPx }) => { const add = () => { if (document.getElementById('__scale')) return; const s = document.createElement('style'); s.id = '__scale'; s.textContent = `#imgTbl, #chromIdeoImg, img[src*="hgtIdeo"] { zoom: ${1 / k} !important; }` + `\n#mouseoverContainer, #mouseOverText, .tooltip, .__pinnedTip { font-size: ${tipPx}px !important; }`; (document.head || document.documentElement).appendChild(s); }; if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', add); else add(); }; // ---------- animated cursor (same technique as the walkthrough-video skill's record.js) ---------- // The glyph and its box are shared with pinShot(), which draws a STATIC copy at every // pinned mouseover so a combined figure shows where each tooltip was raised from. Keep // them one definition: a pinned cursor that did not match the animated one would read as // a different pointer rather than as the same tour paused. const CURSOR_BOX = 'position:fixed;left:0;top:0;z-index:2147483647;pointer-events:none;' + 'width:24px;height:24px;margin-left:-3px;margin-top:-2px;filter:drop-shadow(0 1px 1px rgba(0,0,0,.4));'; const CURSOR_SVG = ''; const CURSOR_INIT = ({ box, svg }) => { const add = () => { if (document.getElementById('__cur')) return; const c = document.createElement('div'); c.id = '__cur'; c.style.cssText = box; c.innerHTML = svg; document.documentElement.appendChild(c); const place = (x, y) => { c.style.transform = `translate(${x}px,${y}px)`; }; place(120, 120); document.addEventListener('mousemove', e => place(e.clientX, e.clientY), true); document.addEventListener('mousedown', e => { const r = document.createElement('div'); r.className = '__ripple'; r.style.cssText = `position:fixed;left:${e.clientX}px;top:${e.clientY}px;z-index:2147483645;pointer-events:none;width:6px;height:6px;margin:-3px 0 0 -3px;border:3px solid rgba(225,30,30,.95);border-radius:50%;`; document.documentElement.appendChild(r); r.animate([{ transform: 'scale(1)', opacity: 1 }, { transform: 'scale(6)', opacity: 0 }], { duration: 520, easing: 'ease-out' }); setTimeout(() => r.remove(), 540); }, true); }; if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', add); else add(); }; +// ---------- LOGIN: the one fixture that cannot live in the repository ---------- +// hgCollection, and the saving half of hgSession, refuse to run for a visitor who is not +// logged in (hgCollection.c doMiddle, "You must be logged in to edit collections"). The +// login cookie is validated against a salted hash (login.cookieSalt, hg/lib/wikiLink.c), +// so there is no way to hand the browser a cookie: a script that needs one of those pages +// has to sign in the way a person does. +// +// An account is a row in gbMembers in ONE hgcentral, so that database is what the +// credentials are keyed by -- not the server, and not the sandbox. genome-test, hgwdev, +// every hgwdev- sandbox and every ticket park read hgcentraltest, so one account +// covers all of them; hgwbeta reads hgcentralbeta and the RR reads hgcentral, which are +// different sets of accounts entirely. +// +// Which central a server reads is READ rather than assumed, because a sandbox may say so +// for itself: of the personal hg.conf files on hgwdev today, 45 set central.db to +// hgcentraltest and two do not (hgcentralgsid, hgcentralbeta). tests/preflight.js has a +// fuller copy of this hg.conf reader and prints what it found; keep the two in step. +const CENTRAL_BY_HOST = { // servers whose hg.conf is on another machine + 'genome.ucsc.edu': 'hgcentral', + 'genome-euro.ucsc.edu': 'hgcentral', // its own database of the same name + 'genome-asia.ucsc.edu': 'hgcentral', // ... and so is this one + 'hgwbeta.soe.ucsc.edu': 'hgcentralbeta', +}; + +function hgConfFor(server) { + let u; + try { u = new URL(server); } catch (e) { return null; } + const host = u.hostname, name = host.split('.')[0]; + if (name === 'hgwdev' || name === 'genome-test') return '/usr/local/apache/cgi-bin/hg.conf'; + if (name.startsWith('hgwdev-')) return `/usr/local/apache/cgi-bin-${name.slice(7)}/hg.conf`; + if ((host === '127.0.0.1' || host === 'localhost') && u.port) { + const root = process.env.TS_ROOT || path.join(os.homedir(), 'ticketSandboxes'); + let reg; + try { reg = fs.readFileSync(path.join(root, 'ports.tsv'), 'utf8'); } catch (e) { return null; } + for (const line of reg.split('\n')) { + const f = line.split('\t'); + if (f[1] === u.port) return path.join(root, f[0], 'cgi-bin', 'hg.conf'); + } + } + return null; +} + +// hg.conf as hg/lib/hgConfig.c reads it: `include` is relative to the including file, +// `delete` drops a name, and a later assignment wins over an earlier one. +function readHgConf(file, out = new Map(), seen = new Set(), depth = 0) { + if (depth > 10 || seen.has(file)) return out; + seen.add(file); + let text; + try { text = fs.readFileSync(file, 'utf8'); } catch (e) { return out; } + for (const raw of text.split('\n')) { + const line = raw.trim(); + if (!line || line.startsWith('#')) continue; + if (/^include\s/.test(line)) + readHgConf(path.resolve(path.dirname(file), line.slice(7).trim()), out, seen, depth + 1); + else if (/^delete\s/.test(line)) + for (const name of line.slice(6).trim().split(/\s+/)) out.delete(name); + else { + const eq = line.indexOf('='); + if (eq > 0) out.set(line.slice(0, eq).trim(), line.slice(eq + 1).trim()); + } + } + return out; +} + +function centralDbFor(server) { + const file = hgConfFor(server); + if (file) { + const db = readHgConf(file).get('central.db'); + if (db) return db; + } + try { return CENTRAL_BY_HOST[new URL(server).hostname] || null; } catch (e) { return null; } +} + +// A password cannot go in a script. It is read from a file outside the tree that only its +// owner can read -- the arrangement hg.conf uses for hg.conf.private, and for the same +// reason -- one section per hgcentral: +// +// [hgcentraltest] +// user=docentTest +// password=... +// +// [default] is used when no section matches, and when the central cannot be worked out +// at all (a server on another machine that is not in CENTRAL_BY_HOST). +// +// Nothing here prints a password, and `login:` has no argument that could carry one. +function loginFile() { + return process.env.DOCENT_LOGIN_FILE || path.join(os.homedir(), '.docentLogin'); +} + +// Parse the sectioned file into [{central, user, password}], in file order. Lines before +// any [section] are ignored rather than treated as a default: an unsectioned file is one +// written against the older single-account form, and silently using it everywhere is how +// an hgcentraltest password would reach the RR. +function loginSections(text) { + const out = []; + let cur = null; + for (const raw of text.split('\n')) { + const line = raw.trim(); + if (!line || line.startsWith('#')) continue; + const sec = /^\[(.+)\]$/.exec(line); + if (sec) { cur = { central: sec[1].trim(), user: '', password: '' }; out.push(cur); continue; } + const eq = line.indexOf('='); + if (eq < 0 || !cur) continue; + const k = line.slice(0, eq).trim(), v = line.slice(eq + 1).trim(); + if (k === 'user' || k === 'password') cur[k] = v; + } + return out; +} + +function loginCreds(server) { + const env = process.env; + // A single-run override, for trying an account without writing it down. It applies to + // whatever server this run drives, which is why it wins over the file. + if (env.DOCENT_LOGIN_USER && env.DOCENT_LOGIN_PASSWORD) + return { user: env.DOCENT_LOGIN_USER, password: env.DOCENT_LOGIN_PASSWORD, from: 'the environment' }; + const central = centralDbFor(server); + const where = central ? `${server} (central.db ${central})` : `${server} (central.db unknown)`; + const file = loginFile(); + let text; + try { text = fs.readFileSync(file, 'utf8'); } + catch (e) { + throw new Error(`login: no credentials for ${where}. Write ${file} with a ` + + `[] section holding "user=" and "password=" lines (mode 0600), ` + + `or set DOCENT_LOGIN_USER and DOCENT_LOGIN_PASSWORD for this run.`); + } + // Refuse a file anyone else can read, the way hg/lib/hgConfig.c checkConfigPerms refuses + // a group- or world-readable hg.conf. A test that quietly used a readable password file + // would make one on every machine it ran on. + const perm = fs.statSync(file).mode & 0o777; + if (perm & 0o077) + throw new Error(`login: ${file} is readable by group or other (mode ${perm.toString(8)}); chmod 600 it`); + const secs = loginSections(text); + const match = (central && secs.find(x => x.central === central)) + || secs.find(x => x.central === 'default'); + if (!match) + throw new Error(`login: ${file} has no section for ${where}` + + (secs.length ? ` (it has ${secs.map(x => `[${x.central}]`).join(' ')})` : ' (it has no [section] at all)') + + '; add one, or a [default]'); + if (!match.user || !match.password) + throw new Error(`login: [${match.central}] in ${file} needs a "user=" line and a "password=" line`); + return { user: match.user, password: match.password, from: `[${match.central}] in ${file}` }; +} + function absurl(u) { if (/^https?:/.test(u)) return u; if (u.startsWith('/cgi-bin/')) return SERVER.replace(/\/cgi-bin$/, '') + u; if (u.startsWith('/')) return SERVER.replace(/\/cgi-bin$/, '') + u; return SERVER + '/' + u; } // DOCENT_DERIVE=1: print what each `track:` step turns into and stop, with no browser and // no server drive. Most of Docent's own decisions live in that derivation -- which // containers come along, which `_sel` goes with them, where a hideKids walk stops -- and // until now the only way to see them was the log of a full run against a live view. This // makes them cheap to look at, and cheap to diff when the derivation is changed. const DERIVE = !!process.env.DOCENT_DERIVE; const T_START = Date.now(); (async () => { // Before the browser: nothing in the derivation touches the page, and the point is to // not pay for one. (The helpers below are function declarations, so they are hoisted.) if (DERIVE) { await deriveMain(); return; } fs.mkdirSync(STILLDIR, { recursive: true }); const browser = await chromium.launch({ headless: true, args: ['--force-color-profile=srgb'] }); const ctx = await browser.newContext({ viewport: { width: VW, height: VH }, deviceScaleFactor: SCALE, ...(FAST ? {} : { recordVideo: { dir: path.join(HERE, '.vid_' + base), size: { width: VW, height: VH } } }), }); if (SCALE > 1) await ctx.addInitScript(SCALE_INIT, SCALE_ARGS); await ctx.addInitScript(CURSOR_INIT, { box: CURSOR_BOX, svg: CURSOR_SVG }); await ctx.addInitScript(() => { try { localStorage.setItem('hgTracks_hideTutorial', '1'); } catch (e) {} }); const page = await ctx.newPage(); const cur = { x: 120, y: 120 }; const pinnedTips = []; // recorded mouseover tooltips for the next pinShot (per view) async function captureState() { try { const u = new URL(page.url()); const h = u.searchParams.get('hgsid'); if (h) state.hgsid = h; const db = u.searchParams.get('db'); if (db) state.db = db; const p = u.searchParams.get('position'); if (p) state.position = p; } catch (e) {} // An interactive zoom / drag-select reload stores the new window in the CART, not the // URL, so read the live position straight from hgTracks when we're on a tracks page -- // otherwise a later position-based nav (e.g. turning on a track) reverts the zoom. try { const pos = await page.evaluate(() => { try { if (typeof hgTracks !== 'undefined' && hgTracks.chromName) return hgTracks.chromName + ':' + (hgTracks.winStart + 1) + '-' + hgTracks.winEnd; } catch (_) {} return null; }); if (pos) state.position = pos; } catch (e) {} // Authoring aid: DOCENT_ROWS=1 logs the rows hgTracks actually drew, so "why is that // subtrack still there / why is my mouseover track not shown" is one run, not a guess. if (process.env.DOCENT_ROWS) { const rows = await page.evaluate(() => [...document.querySelectorAll('[id^="img_data_"]')].map(e => e.id.replace('img_data_', ''))).catch(() => null); if (rows) console.log(' rows:', rows.join(', ') || '(none)'); } await scaleHeights(); // print runs only; see below } // A print run makes the image k times wider, and a track with a FIXED PIXEL height does not // follow: a bigLolly or wiggle row is a pixel count read from trackDb/the cart, untouched by // `pix` and `textSize`, so a 128px row that was 15% of an 850px image is 5% of a 2550px one. // That is how the ClinVar lollipop row came out a sliver with unreadable y-axis labels next to // a bed track that DID grow with the font. So after each view change, ask for k times the // height of every row hgTracks just drew. It is asked of the rows the page actually has -- // which is the only way to reach the LIFTED view, whose tracks are hub tracks under names // trackDb never saw. Harmless where it means nothing (a bigBed never reads heightPer), and // each track's own `maxHeightPixels` still clamps it, so a track that should stay short does; // raise that ceiling in trackDb for one that should not (clinvarSubLolly does this). const heightsSent = new Set(); let inHeightNav = false; async function scaleHeights() { if (!HEIGHTPER || inHeightNav) return; const drawn = await page.evaluate(() => [...document.querySelectorAll('#imgTbl [id^="img_data_"]')].map(e => e.id.replace('img_data_', '')) ).catch(() => []); const fresh = (drawn || []).filter(n => n && !heightsSent.has(n)); if (!fresh.length) return; // same rows as last time: no second load fresh.forEach(n => heightsSent.add(n)); inHeightNav = true; // the nav below must not recurse try { await nav(`/cgi-bin/hgTracks?${fresh.map(n => `${n}.heightPer=${HEIGHTPER}`).join('&')}&${IMGVARS}`); } finally { inHeightNav = false; } } // Apache's LimitRequestLine defaults to 8190 bytes for the whole request line, and a // step that derives a lot of cart variables can sail past it. The server then answers 414 // and the page LOADS -- so nothing throws, captureState finds no image, and the next // shot: quietly photographs "Request-URI Too Long". Say so, since only an eyeball on the // still would otherwise catch it. const URL_WARN = 7800; async function nav(u) { const full = absurl(u); if (full.length > URL_WARN) console.warn(`nav: URL is ${full.length} chars, over Apache's usual ${8190} limit ` + `-- expect a 414 "Request-URI Too Long" page instead of the view`); pinnedTips.length = 0; await page.goto(full, { waitUntil: 'load' }); await captureState(); await page.mouse.move(cur.x, cur.y); } async function glide(x, y) { if (FAST) { await page.mouse.move(x, y); cur.x = x; cur.y = y; return; } const steps = Math.max(10, Math.round(Math.hypot(x - cur.x, y - cur.y) / 9)); for (let i = 1; i <= steps; i++) { await page.mouse.move(cur.x + (x - cur.x) * i / steps, cur.y + (y - cur.y) * i / steps); await sleep(15); } cur.x = x; cur.y = y; } async function glideTo(sel) { const b = await page.locator(sel).first().boundingBox({ timeout: 8000 }).catch(() => null); if (b) await glide(b.x + b.width / 2, b.y + b.height / 2); } async function clickGlide(sel) { await glideTo(sel); await sleep(160); await page.click(sel); } async function checkGlide(sel, want) { await glideTo(sel); await sleep(140); if (want) await page.check(sel).catch(() => {}); else await page.uncheck(sel).catch(() => {}); } // visibly open a native