497f2788d49fda531c0836618688f954486eb87c mspeir Fri Sep 4 09:44:13 2026 -0700 trackLists: do not let the public page claim the download cross-check is clean, refs #37781 The static-page rewrite collapsed mkPage.py's three-way branch on the hgdownload cross-check into two, so the public run fell into the all-clear wording whether or not collect.py had found a restricted file that is still reachable. collect.py's current output has three such files, so the page was asserting the opposite of what the check found. Restored as three cases: the public page says only that the check runs and that anything found is reported privately, the --internal page keeps the table of paths, and the all-clear wording is used only when the list really is empty. The comment explaining why the public branch must not be merged back into the all-clear one is back too. collect.py counted any HTTP code that was not literally 404 as reachable, which made a curl timeout or a failed connection (empty output, or 000) look like an exposed file. That mails false alarms and, now that the branch above depends on it, would drop the all-clear line from the public page on a network blip. Only 2xx and 3xx count as served, 4xx as blocked, and anything else is reported separately as not checked, in collected.json and on stderr. Also: the remaining unquoted interpolations into shell=True commands go through the existing q() helper, rows_by_track() no longer builds the "why" set that the new table does not use, and US spelling throughout. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> diff --git src/hg/utils/otto/trackLists/mkPage.py src/hg/utils/otto/trackLists/mkPage.py index 033a969ed86..e12f362fec8 100755 --- src/hg/utils/otto/trackLists/mkPage.py +++ src/hg/utils/otto/trackLists/mkPage.py @@ -1,232 +1,245 @@ #!/usr/bin/env python3 """Render collected.json as a UCSC Genome Browser static page (RM #37781). Emits the house static-page shape: SSI includes for the menubar and footer, no stylesheet of its own (gbStatic.css already styles .gbsPage tables), h2 for sections, h6 for the contents list, and ASCII-only source. """ import json, html, argparse, collections, datetime, textwrap def esc(s): """HTML-escape and force ASCII, since the house style forbids raw UTF-8.""" s = html.escape(str(s or ""), quote=True) return s.encode("ascii", "xmlcharrefreplace").decode("ascii") def cell(w, text, indent=" "): """Write a <td>, wrapping so no source line runs past 100 characters.""" text = esc(text) if len(indent) + len(text) + 9 <= 100: w("%s<td>%s</td>" % (indent, text)) return w("%s<td>" % indent) for line in textwrap.wrap(text, width=94 - len(indent), break_long_words=False, break_on_hyphens=False): w("%s %s" % (indent, line)) w("%s</td>" % indent) def rows_by_track(restricted): - by = collections.defaultdict(lambda: dict(dbs=set(), label="", why=set())) + """Group the restricted rows by track. The per-row "why" is deliberately not + carried through: which tests fired is explained once in prose below the table.""" + by = collections.defaultdict(lambda: dict(dbs=set(), label="")) for r in restricted: e = by[r["track"]] e["dbs"].add(r["db"]) e["label"] = e["label"] or r.get("shortLabel", "") - e["why"].update(r.get("why", [])) return by def main(): ap = argparse.ArgumentParser() ap.add_argument("-i", "--inp", default="collected.json") ap.add_argument("-o", "--out", default="trackLists.html") ap.add_argument("--date", default=None) ap.add_argument("--internal", action="store_true", help="include the hgdownload cross-check, which names restricted files " "that are currently reachable. Never use for a public page.") a = ap.parse_args() d = json.load(open(a.inp)) today = a.date or d.get("generated") or datetime.date.today().isoformat() o = [] w = o.append w('<!DOCTYPE html>') w('<!-- DO NOT EDIT THIS FILE. It is generated by') w(' kent/src/hg/utils/otto/trackLists/mkPage.py -->') w('<!--#set var="TITLE" value="Track lists: redistribution, updates and contributed data" -->') w('<!--#set var="ROOT" value="." -->') w('') w('<!-- Relative paths to support mirror sites with non-standard GB docs install -->') w('<!--#include virtual="$ROOT/inc/gbPageStart.html" -->') w('') w('<h1>Track lists: redistribution, automatic updates and contributed data</h1>') w('') w('<h2>Contents</h2>') w("<h6><a href='#notRedistributable'>Tracks we cannot redistribute</a></h6>") w("<h6><a href='#autoUpdating'>Tracks that update themselves</a></h6>") w("<h6><a href='#contributed'>Contributed tracks</a></h6>") w('') w('<p>') w('People running their own copy of the Genome Browser ask us three questions often') w('enough that it is worth answering them in one place: which tracks we are not allowed') w('to pass on, which tracks change on their own, and which tracks were built by someone') w('other than UCSC. This page is rebuilt automatically, so it reflects the current state') w('of our servers rather than a hand-kept list.') w('</p>') w('<p>') w('For installation instructions see the') w('<a href="goldenPath/help/mirror.html">mirror documentation</a>. Questions are welcome') w('on the <a href="goldenPath/help/mirror.html#the-genome-mirror-mailing-list">genome-mirror') w('mailing list</a>.') w('</p>') w('') # ---- 1. not redistributable ------------------------------------------- w("<a name='notRedistributable'></a>") w('<h2>Tracks we cannot redistribute</h2>') w('<p>') w('These tracks reach us under terms that let us display the data but not pass it on.') w('You can see them on our site, and in most cases you can obtain the same data yourself') w('directly from the group that produced it, but we cannot include them in a mirror or on') - w('our download server. The reasons vary: some are commercial licences, others are') + w('our download server. The reasons vary: some are commercial licenses, others are') w('consent agreements attached to human cohorts. Check the description page of an') w('individual track for who to approach about access.') w('</p>') w('<table>') w(' <tr>') w(' <th>Track</th>') w(' <th>Table or track name</th>') w(' <th>Assemblies</th>') w(' </tr>') by = rows_by_track(d["restricted"]) for t, e in sorted(by.items(), key=lambda x: (x[1]["label"] or x[0]).lower()): w(' <tr>') cell(w, e["label"] or t) w(' <td><code>%s</code></td>' % esc(t)) cell(w, " ".join(sorted(e["dbs"]))) w(' </tr>') w('</table>') w('') w('<h3>How this list is put together</h3>') w('<p>') w('A track appears above if any of three things is true of it: its configuration says') w('<code>tableBrowser off</code>; its <code>noGenomeReason</code> refers to distribution') w('terms, which is how OMIM is marked and is missed by a search for the first setting') w('alone; or its table exists on our servers but is deliberately absent from the download') w('server. No single one of those catches everything, so all three are checked. Note that') w('some tracks are withheld from whole-genome Table Browser queries only because they are') w('too large to return, not for any licensing reason, and those are not listed above.') w('</p>') exposed = d.get("exposed", []) - if exposed and a.internal: + if exposed and not a.internal: + # Never name reachable restricted files on a page anyone can read: the path + # of a file we should be blocking is a pointer straight at it. Say only that + # the check runs; the internal copy and the cron mail carry the detail. Do + # not fold this branch into the all-clear one below, which would have the + # public page claim the list is clean when the check says otherwise. + w('<p>') + w('Every track named above is cross-checked against the download server each time') + w('this page is rebuilt. Any file that turns out to be reachable there is reported') + w('to us privately rather than named on this page.') + w('</p>') + elif exposed: w('<h3>Reachable on hgdownload</h3>') w('<p>') w('%d file(s) marked as restricted are currently served by the download server and' % len(exposed)) w('need to be added to its exclude list.') w('</p>') w('<table>') w(' <tr>') w(' <th>Track</th>') w(' <th>Assembly</th>') w(' <th>Path</th>') w(' </tr>') for r in exposed: w(' <tr>') cell(w, r["shortLabel"] or r["track"]) cell(w, r["db"]) w(' <td><code>%s</code></td>' % esc(r["path"])) w(' </tr>') w('</table>') else: w('<p>') w('Every track named above is checked against the download server each time this page') - w('is rebuilt, so that a track listed as restricted is genuinely blocked there.') + w('is rebuilt, and every file marked as restricted is correctly blocked there. Checked') + w('on %s.' % esc(today)) w('</p>') w('') # ---- 2. otto ---------------------------------------------------------- w("<a name='autoUpdating'></a>") w('<h2>Tracks that update themselves</h2>') w('<p>') w('These tracks are rebuilt on a schedule without anyone at UCSC touching them. If you') - w('mirror them, your copy will drift from ours until you synchronise again. Times are US') + w('mirror them, your copy will drift from ours until you synchronize again. Times are US') w('Pacific.') w('</p>') w('<table>') w(' <tr>') w(' <th>Source</th>') w(' <th>Updated</th>') w(' <th>Tracks affected</th>') w(' <th>Assemblies</th>') w(' </tr>') jobs = [j for j in d["otto"] if j["kind"] in ("track", "hub", "table")] for j in sorted(jobs, key=lambda x: x["name"].lower()): w(' <tr>') cell(w, j["name"]) cell(w, j["schedule"]) cell(w, j["detail"]) cell(w, j.get("assemblies", "") if j["kind"] == "track" else "") w(' </tr>') w('</table>') notifiers = [j for j in d["otto"] if j["kind"] == "notifier"] if notifiers: w('') w('<h3>Scheduled checks that change no data</h3>') w('<p>') w('These watch for new releases upstream and send us mail. They update nothing on') w('their own, and are listed so that the schedule above is not mistaken for the whole') w('picture.') w('</p>') w('<table>') w(' <tr>') w(' <th>Check</th>') w(' <th>Runs</th>') w(' <th>What it looks at</th>') w(' </tr>') for j in notifiers: w(' <tr>') cell(w, j["name"], indent=" ") cell(w, j["schedule"], indent=" ") cell(w, j["detail"], indent=" ") w(' </tr>') w('</table>') w('') # ---- 3. contributed --------------------------------------------------- contrib = d.get("contrib", []) w("<a name='contributed'></a>") w('<h2>Contributed tracks</h2>') w('<p>') w('Some assemblies in our') w('<a href="https://hgdownload.soe.ucsc.edu/hubs/" target="_blank">GenArk</a> collection') - w('carry annotation built by outside groups rather than by UCSC. The data sit alongside') - w('our own tracks, but the group named below produced them, and questions about the') + w('carry annotation built by outside groups rather than by UCSC. The data sits alongside') + w('our own tracks, but the group named below produced it, and questions about the') w('underlying annotation are best sent to that group.') w('</p>') w('<table>') w(' <tr>') w(' <th>Contributing group</th>') w(' <th>Assemblies</th>') w(' </tr>') for c in contrib: w(' <tr>') w(' <td>%s</td>' % esc(c["name"])) w(' <td>%d</td>' % c["assemblies"]) w(' </tr>') w('</table>') w('<p>') w('%d assemblies carry contributed annotation, from %d groups.' % (sum(c["assemblies"] for c in contrib), len(contrib))) w('</p>') w('') w('<p>') w('This page was generated on %s from the current state of our servers.' % esc(today)) w('</p>') w('') w('<!--#include virtual="$ROOT/inc/gbPageEnd.html" -->') text = "\n".join(o) + "\n" open(a.out, "w").write(text) longest = max(len(x) for x in o) print("wrote %s (%d bytes, longest line %d)" % (a.out, len(text), longest)) if __name__ == "__main__": main()