9b762c146f7a44a3bdc783cdd202356fc163b8fb braney Wed Sep 16 12:43:32 2026 -0700 docent: one targetConf.js for the target, the hg.conf, the central and the account, refs #37892 docent.js and tests/preflight.js had grown a second and then a third copy of the same lookups. That is exactly the pair that must not drift: preflight checks the fixtures for the server the RUN will drive, so a run that resolved its target, its hg.conf, its hgcentral or its account even slightly differently would be checked against the wrong machine, and the mismatch would show up as a green preflight in front of a red suite. targetConf.js answers the four questions in order, each from the one before: resolveTarget a `target:` (or DOCENT_TARGET) -> the .../cgi-bin URL to drive hgConfFor that URL -> the hg.conf it reads, if the server is on this box centralDbFor that conf -> which hgcentral it uses loginLookup that central -> the account to sign in with Nothing in it opens a browser or the network, which is why preflight can ask all four in the seconds before a run. loginLookup returns the facts plus, when the account cannot be used, ONE sentence saying why. That sentence is the substantive half and is now written once: the step throws it with a `login:` prefix, preflight prints it on the MISSING line. Each caller still phrases its own success line, since one logs and the other prints a fixture row. No password crosses that boundary to anything that prints. 248 lines net out of the two programs, 197 into the module. docent.js is no longer a single file, and three places now say so: its own require, the Run section of README.md, and docent.mk, whose mp4 rule gains targetConf.js as a prerequisite -- a change there changes what a tour renders, so it has to rebuild one. Nothing in the tree or in ~/docentTours copies docent.js; they all reference it where it sits, with targetConf.js beside it. Measured before and after, with no other change: preflight resolves the same account, central and conf for genome-test, hgwdev-braney, a ts park on 48087 and hgwbeta (which correctly has no section); tests/ is 18 of 18 with the derive baselines matching; tests/regress preflights 14 fixtures for 67 scripts. diff --git src/hg/utils/docent/docent.js src/hg/utils/docent/docent.js index 6efe9200c2e..626bbc0c08f 100755 --- src/hg/utils/docent/docent.js +++ src/hg/utils/docent/docent.js @@ -14,30 +14,34 @@ * * The high-level verbs bake in the quickLift/Convert mechanics (dbSNP composite * params, the hideDefaults-reverts-on-assembly-change bug, target lookup) so the * author writes intent, not selectors. See README.md for the language. * * The surface syntax is YAML so ordinary editors highlight it; the language is the * verb vocabulary layered on top, not the serialization. Scripts are named * <base>.docent.yaml (a bare <base>.docent also works). */ const { chromium } = require('playwright'); const yaml = require('js-yaml'); const fs = require('fs'); const os = require('os'); const path = require('path'); const { execFileSync } = require('child_process'); +// Shared with tests/preflight.js so a run and its fixture check cannot disagree about +// which server is being driven, which hg.conf it reads, or which account signs in. +// docent.js is no longer a single file: targetConf.js has to travel with it. +const { resolveTarget, serverFor, loginLookup } = require('./targetConf.js'); // ---------- parse script + config ---------- const SCRIPT = process.argv[2]; if (!SCRIPT) { console.error('usage: node docent.js SCRIPT.docent.yaml [OUT.mp4]'); process.exit(2); } const doc = yaml.load(fs.readFileSync(SCRIPT, 'utf8')) || {}; // Lint: in a YAML flow map a colon needs a trailing space, so `{item:name5568747}` // parses as ONE key "item:name5568747" (value null) and the intended `item:` arg is // silently dropped -- the verb then quietly falls back to a default. Catch that here // (before the long browser run) by flagging any arg key that contains a ':'. (function lintSteps(steps) { let n = 0; const scan = (obj, where) => { if (!obj || typeof obj !== 'object') return; for (const k of Object.keys(obj)) { @@ -59,50 +63,38 @@ const base = path.basename(SCRIPT).replace(/\.(docent\.)?ya?ml$/i, '').replace(/\.docent$/i, ''); const FIGDIR = path.resolve(HERE, '..'); // figures dir beside the scripts const OUTMP4 = process.argv[3] || doc.mp4 || path.join(FIGDIR, base + '.mp4'); // Stills go to stills/<base>/. DOCENT_STILLS names a different PARENT ("stills.hires"), // which is how a high-resolution run keeps its figures beside the screen-resolution ones // instead of overwriting them. const STILLPARENT = process.env.DOCENT_STILLS; const STILLDIR = STILLPARENT ? path.resolve(HERE, STILLPARENT, base) : doc.stills ? path.resolve(HERE, doc.stills) : path.join(HERE, 'stills', base); // Saved sessions go to sessions/<base>/, beside stills/. `sessions:` and DOCENT_SESSIONS // name the PARENT (not the per-scenario directory), so `sessionUrlBase:` below always maps // onto it as <base>/<name>.txt, and a print run keeps its files out of the screen run's way. const SESSDIR = path.join( path.resolve(HERE, process.env.DOCENT_SESSIONS || doc.sessions || 'sessions'), base); -// `target:` takes a shorthand from this table, a bare `hgwdev-<user>` sandbox name -// (expanded below), or a full https://.../cgi-bin URL. Default is genome-test, so a -// script that forgets to say where it runs does not silently hit someone's sandbox. -const SERVERS = { - 'rr': 'https://genome.ucsc.edu/cgi-bin', - 'genome-test': 'https://genome-test.gi.ucsc.edu/cgi-bin', - 'hgwdev': 'https://hgwdev.gi.ucsc.edu/cgi-bin', - 'hgwbeta': 'https://hgwbeta.soe.ucsc.edu/cgi-bin', -}; -const resolveTarget = t => { - if (!t) return SERVERS['genome-test']; - if (SERVERS[t]) return SERVERS[t]; - if (/^hgwdev-[a-z0-9._-]+$/i.test(t)) return `https://${t}.gi.ucsc.edu/cgi-bin`; // personal sandbox - return t; // full URL -}; -// DOCENT_TARGET overrides `target:` for the whole run, so a suite written against one -// server can be pointed at another -- a sandbox, a ticket park, a demo browser -- without -// editing the scripts it is written from. The trackDb cache below keys on SERVER, so a -// redirected run cannot read back a listing fetched from the server the script names. -const SERVER = resolveTarget(process.env.DOCENT_TARGET || doc.target).replace(/\/$/, ''); +// `target:` takes a shorthand (rr, genome-test, hgwdev, hgwbeta), a bare `hgwdev-<user>` +// sandbox name, or a full https://.../cgi-bin URL; DOCENT_TARGET overrides it for the +// whole run, so a suite written against one server can be pointed at another -- a sandbox, +// a ticket park, a demo browser -- without editing the scripts. Both rules live in +// targetConf.js, beside the fixture check that has to agree with them. The trackDb cache +// below keys on SERVER, so a redirected run cannot read back a listing fetched from the +// server the script names. +const SERVER = serverFor(doc.target); // SCALE: the same tour rendered at k times the resolution, for figures that have to print. // Nothing is upscaled -- a still only ever has the pixels it was drawn with -- so each layer // is asked to draw k times as many while the layout is left alone: // // * deviceScaleFactor: k. The viewport keeps its 1x CSS size, so the page lays out exactly // as at 1x -- same line breaks, same jQuery-dialog width, same tooltip placement -- and // every bit of it is rasterized with k times the pixels. The retina case, natively. // * `pix` x k, so the server draws the browser image k times as wide, with `textSize` // stepped up to match so hgTracks makes the SAME layout decisions in that bigger image: // same tick spacing, same room for labels, same packing of features into rows. Without // the font, a wider image is a different picture rather than a bigger one. // * `zoom: 1/k` on the image table (SCALE_INIT below), handing that k-times-wider image // back the 1x amount of layout space. One image pixel then falls on exactly one device // pixel: native resolution, no resampling anywhere in the path. // @@ -328,164 +320,39 @@ r.animate([{ transform: 'scale(1)', opacity: 1 }, { transform: 'scale(6)', opacity: 0 }], { duration: 520, easing: 'ease-out' }); setTimeout(() => r.remove(), 540); }, true); }; if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', add); else add(); }; // ---------- LOGIN: the one fixture that cannot live in the repository ---------- // hgCollection, and the saving half of hgSession, refuse to run for a visitor who is not // logged in (hgCollection.c doMiddle, "You must be logged in to edit collections"). The // login cookie is validated against a salted hash (login.cookieSalt, hg/lib/wikiLink.c), // so there is no way to hand the browser a cookie: a script that needs one of those pages // has to sign in the way a person does. // -// An account is a row in gbMembers in ONE hgcentral, so that database is what the -// credentials are keyed by -- not the server, and not the sandbox. genome-test, hgwdev, -// every hgwdev-<name> sandbox and every ticket park read hgcentraltest, so one account -// covers all of them; hgwbeta reads hgcentralbeta and the RR reads hgcentral, which are -// different sets of accounts entirely. -// -// Which central a server reads is READ rather than assumed, because a sandbox may say so -// for itself: of the personal hg.conf files on hgwdev today, 45 set central.db to -// hgcentraltest and two do not (hgcentralgsid, hgcentralbeta). tests/preflight.js has a -// fuller copy of this hg.conf reader and prints what it found; keep the two in step. -const CENTRAL_BY_HOST = { // servers whose hg.conf is on another machine - 'genome.ucsc.edu': 'hgcentral', - 'genome-euro.ucsc.edu': 'hgcentral', // its own database of the same name - 'genome-asia.ucsc.edu': 'hgcentral', // ... and so is this one - 'hgwbeta.soe.ucsc.edu': 'hgcentralbeta', -}; - -function hgConfFor(server) { - let u; - try { u = new URL(server); } catch (e) { return null; } - const host = u.hostname, name = host.split('.')[0]; - if (name === 'hgwdev' || name === 'genome-test') return '/usr/local/apache/cgi-bin/hg.conf'; - if (name.startsWith('hgwdev-')) return `/usr/local/apache/cgi-bin-${name.slice(7)}/hg.conf`; - if ((host === '127.0.0.1' || host === 'localhost') && u.port) { - const root = process.env.TS_ROOT || path.join(os.homedir(), 'ticketSandboxes'); - let reg; - try { reg = fs.readFileSync(path.join(root, 'ports.tsv'), 'utf8'); } catch (e) { return null; } - for (const line of reg.split('\n')) { - const f = line.split('\t'); - if (f[1] === u.port) return path.join(root, f[0], 'cgi-bin', 'hg.conf'); - } - } - return null; -} - -// hg.conf as hg/lib/hgConfig.c reads it: `include` is relative to the including file, -// `delete` drops a name, and a later assignment wins over an earlier one. -function readHgConf(file, out = new Map(), seen = new Set(), depth = 0) { - if (depth > 10 || seen.has(file)) return out; - seen.add(file); - let text; - try { text = fs.readFileSync(file, 'utf8'); } catch (e) { return out; } - for (const raw of text.split('\n')) { - const line = raw.trim(); - if (!line || line.startsWith('#')) continue; - if (/^include\s/.test(line)) - readHgConf(path.resolve(path.dirname(file), line.slice(7).trim()), out, seen, depth + 1); - else if (/^delete\s/.test(line)) - for (const name of line.slice(6).trim().split(/\s+/)) out.delete(name); - else { - const eq = line.indexOf('='); - if (eq > 0) out.set(line.slice(0, eq).trim(), line.slice(eq + 1).trim()); - } - } - return out; -} - -function centralDbFor(server) { - const file = hgConfFor(server); - if (file) { - const db = readHgConf(file).get('central.db'); - if (db) return db; - } - try { return CENTRAL_BY_HOST[new URL(server).hostname] || null; } catch (e) { return null; } -} - -// A password cannot go in a script. It is read from a file outside the tree that only its -// owner can read -- the arrangement hg.conf uses for hg.conf.private, and for the same -// reason -- one section per hgcentral: -// -// [hgcentraltest] -// user=docentTest -// password=... -// -// [default] is used when no section matches, and when the central cannot be worked out -// at all (a server on another machine that is not in CENTRAL_BY_HOST). -// -// Nothing here prints a password, and `login:` has no argument that could carry one. -function loginFile() { - return process.env.DOCENT_LOGIN_FILE || path.join(os.homedir(), '.docentLogin'); -} - -// Parse the sectioned file into [{central, user, password}], in file order. Lines before -// any [section] are ignored rather than treated as a default: an unsectioned file is one -// written against the older single-account form, and silently using it everywhere is how -// an hgcentraltest password would reach the RR. -function loginSections(text) { - const out = []; - let cur = null; - for (const raw of text.split('\n')) { - const line = raw.trim(); - if (!line || line.startsWith('#')) continue; - const sec = /^\[(.+)\]$/.exec(line); - if (sec) { cur = { central: sec[1].trim(), user: '', password: '' }; out.push(cur); continue; } - const eq = line.indexOf('='); - if (eq < 0 || !cur) continue; - const k = line.slice(0, eq).trim(), v = line.slice(eq + 1).trim(); - if (k === 'user' || k === 'password') cur[k] = v; - } - return out; -} - +// Which account that is, and why it is keyed by hgcentral rather than by server, is in +// targetConf.js. All this adds is the failure: `login:` is a step, so it throws, while +// preflight reports the same sentence as a missing fixture. A password cannot go in a +// script, `login:` has no argument that could carry one, and nothing here prints one. function loginCreds(server) { - const env = process.env; - // A single-run override, for trying an account without writing it down. It applies to - // whatever server this run drives, which is why it wins over the file. - if (env.DOCENT_LOGIN_USER && env.DOCENT_LOGIN_PASSWORD) - return { user: env.DOCENT_LOGIN_USER, password: env.DOCENT_LOGIN_PASSWORD, from: 'the environment' }; - const central = centralDbFor(server); - const where = central ? `${server} (central.db ${central})` : `${server} (central.db unknown)`; - const file = loginFile(); - let text; - try { text = fs.readFileSync(file, 'utf8'); } - catch (e) { - throw new Error(`login: no credentials for ${where}. Write ${file} with a ` - + `[<hgcentral database>] section holding "user=" and "password=" lines (mode 0600), ` - + `or set DOCENT_LOGIN_USER and DOCENT_LOGIN_PASSWORD for this run.`); - } - // Refuse a file anyone else can read, the way hg/lib/hgConfig.c checkConfigPerms refuses - // a group- or world-readable hg.conf. A test that quietly used a readable password file - // would make one on every machine it ran on. - const perm = fs.statSync(file).mode & 0o777; - if (perm & 0o077) - throw new Error(`login: ${file} is readable by group or other (mode ${perm.toString(8)}); chmod 600 it`); - const secs = loginSections(text); - const match = (central && secs.find(x => x.central === central)) - || secs.find(x => x.central === 'default'); - if (!match) - throw new Error(`login: ${file} has no section for ${where}` - + (secs.length ? ` (it has ${secs.map(x => `[${x.central}]`).join(' ')})` : ' (it has no [section] at all)') - + '; add one, or a [default]'); - if (!match.user || !match.password) - throw new Error(`login: [${match.central}] in ${file} needs a "user=" line and a "password=" line`); - return { user: match.user, password: match.password, from: `[${match.central}] in ${file}` }; + const c = loginLookup(server); + if (c.why) throw new Error(`login: ${c.why}`); + return { user: c.user, password: c.password, + from: c.source === 'the environment' ? c.source : `[${c.section}] in ${c.source}` }; } function absurl(u) { if (/^https?:/.test(u)) return u; if (u.startsWith('/cgi-bin/')) return SERVER.replace(/\/cgi-bin$/, '') + u; if (u.startsWith('/')) return SERVER.replace(/\/cgi-bin$/, '') + u; return SERVER + '/' + u; } // DOCENT_DERIVE=1: print what each `track:` step turns into and stop, with no browser and // no server drive. Most of Docent's own decisions live in that derivation -- which // containers come along, which `_sel` goes with them, where a hideKids walk stops -- and // until now the only way to see them was the log of a full run against a live view. This // makes them cheap to look at, and cheap to diff when the derivation is changed. const DERIVE = !!process.env.DOCENT_DERIVE;