fbca0ee885a9c343e160f7ab7d584d44fa1cf8a5 mspeir Thu Sep 3 18:36:22 2026 -0700 trackLists: emit the page in the standard static-page style, refs #37781 Drops the hand-rolled stylesheet and uses the house shape instead: the gbPageStart/gbPageEnd SSI includes for the menu bar and footer, h2 per section with an h6 contents list, and bare tables, which gbStatic.css already styles through the .gbsPage wrapper. Source is ASCII-only, lowercase tags, and no line over 100 characters, per the QA conventions; long cells wrap rather than run off. The deploy step now chmods the page to 775. Apache runs the SSI includes on a .html file only when its execute bit is set (XBitHack), so without it the page is served verbatim and the reader gets bare content with no menu bar and no stylesheets. allTips.html is in that state on the RR today, which is how the behaviour came to light. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> diff --git src/hg/utils/otto/trackLists/mkPage.py src/hg/utils/otto/trackLists/mkPage.py index c3cc3e6d8bc..033a969ed86 100755 --- src/hg/utils/otto/trackLists/mkPage.py +++ src/hg/utils/otto/trackLists/mkPage.py @@ -1,178 +1,232 @@ #!/usr/bin/env python3 -"""Render collected.json into the mirror-facing HTML page (RM #37781).""" -import json, html, argparse, collections, datetime +"""Render collected.json as a UCSC Genome Browser static page (RM #37781). -def esc(s): return html.escape(str(s or "")) - -CSS = """ -:root{--ink:#1a1a1a;--mut:#5b6570;--line:#d6dce2;--blue:#1c3f70;--panel:#f5f7f9; ---okc:#1a7a3c;--badc:#b3261e;--warnc:#9a6400} -*{box-sizing:border-box} -body{margin:0;background:#fff;color:var(--ink); -font:16px/1.6 -apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif} -.wrap{max-width:1040px;margin:0 auto;padding:32px 24px 80px} -header{border-bottom:3px solid var(--blue);padding-bottom:16px;margin-bottom:26px} -h1{font-size:26px;margin:0 0 6px;color:var(--blue)} -.sub{color:var(--mut);font-size:14px} -h2{font-size:20px;margin:38px 0 10px;color:var(--blue); -border-bottom:1px solid var(--line);padding-bottom:6px} -h3{font-size:16px;margin:22px 0 8px} -p,li{margin:0 0 12px} -code{font-family:ui-monospace,Menlo,Consolas,monospace;font-size:13.5px; -background:var(--panel);padding:1px 5px;border-radius:3px} -table{border-collapse:collapse;width:100%;font-size:14px;margin:12px 0} -th,td{border:1px solid var(--line);padding:6px 9px;text-align:left;vertical-align:top} -th{background:var(--panel);font-weight:600} -td.n,th.n{text-align:right} -.note{background:var(--panel);border-left:5px solid var(--blue);padding:14px 18px; -border-radius:0 4px 4px 0;margin:0 0 16px} -.warn{border-left-color:var(--warnc)} -.scroll{overflow-x:auto} -.mut{color:var(--mut)} -.tag{display:inline-block;font-size:11px;font-weight:700;letter-spacing:.05em; -text-transform:uppercase;padding:1px 7px;border-radius:9px;background:#eef1f4;color:var(--mut)} -footer{margin-top:44px;padding-top:14px;border-top:1px solid var(--line); -font-size:13px;color:var(--mut)} +Emits the house static-page shape: SSI includes for the menubar and footer, no +stylesheet of its own (gbStatic.css already styles .gbsPage tables), h2 for +sections, h6 for the contents list, and ASCII-only source. """ +import json, html, argparse, collections, datetime, textwrap + +def esc(s): + """HTML-escape and force ASCII, since the house style forbids raw UTF-8.""" + s = html.escape(str(s or ""), quote=True) + return s.encode("ascii", "xmlcharrefreplace").decode("ascii") + +def cell(w, text, indent=" "): + """Write a <td>, wrapping so no source line runs past 100 characters.""" + text = esc(text) + if len(indent) + len(text) + 9 <= 100: + w("%s<td>%s</td>" % (indent, text)) + return + w("%s<td>" % indent) + for line in textwrap.wrap(text, width=94 - len(indent), + break_long_words=False, break_on_hyphens=False): + w("%s %s" % (indent, line)) + w("%s</td>" % indent) + +def rows_by_track(restricted): + by = collections.defaultdict(lambda: dict(dbs=set(), label="", why=set())) + for r in restricted: + e = by[r["track"]] + e["dbs"].add(r["db"]) + e["label"] = e["label"] or r.get("shortLabel", "") + e["why"].update(r.get("why", [])) + return by def main(): ap = argparse.ArgumentParser() ap.add_argument("-i", "--inp", default="collected.json") - ap.add_argument("-o","--out",default="index.html") + ap.add_argument("-o", "--out", default="trackLists.html") ap.add_argument("--date", default=None) ap.add_argument("--internal", action="store_true", help="include the hgdownload cross-check, which names restricted files " "that are currently reachable. Never use for a public page.") a = ap.parse_args() d = json.load(open(a.inp)) - today = a.date or datetime.date.today().isoformat() - - # ---- list 1: not redistributable, grouped by track name - by = collections.defaultdict(lambda: dict(dbs=[], label="", why=set())) - for r in d["restricted"]: - e = by[r["track"]] - e["dbs"].append(r["db"]) - e["label"] = e["label"] or r.get("shortLabel","") - for w in r.get("why",[]): e["why"].add(w) - rows1 = [] + today = a.date or d.get("generated") or datetime.date.today().isoformat() + o = [] + w = o.append + + w('<!DOCTYPE html>') + w('<!-- DO NOT EDIT THIS FILE. It is generated by') + w(' kent/src/hg/utils/otto/trackLists/mkPage.py -->') + w('<!--#set var="TITLE" value="Track lists: redistribution, updates and contributed data" -->') + w('<!--#set var="ROOT" value="." -->') + w('') + w('<!-- Relative paths to support mirror sites with non-standard GB docs install -->') + w('<!--#include virtual="$ROOT/inc/gbPageStart.html" -->') + w('') + w('<h1>Track lists: redistribution, automatic updates and contributed data</h1>') + w('') + w('<h2>Contents</h2>') + w("<h6><a href='#notRedistributable'>Tracks we cannot redistribute</a></h6>") + w("<h6><a href='#autoUpdating'>Tracks that update themselves</a></h6>") + w("<h6><a href='#contributed'>Contributed tracks</a></h6>") + w('') + w('<p>') + w('People running their own copy of the Genome Browser ask us three questions often') + w('enough that it is worth answering them in one place: which tracks we are not allowed') + w('to pass on, which tracks change on their own, and which tracks were built by someone') + w('other than UCSC. This page is rebuilt automatically, so it reflects the current state') + w('of our servers rather than a hand-kept list.') + w('</p>') + w('<p>') + w('For installation instructions see the') + w('<a href="goldenPath/help/mirror.html">mirror documentation</a>. Questions are welcome') + w('on the <a href="goldenPath/help/mirror.html#the-genome-mirror-mailing-list">genome-mirror') + w('mailing list</a>.') + w('</p>') + w('') + + # ---- 1. not redistributable ------------------------------------------- + w("<a name='notRedistributable'></a>") + w('<h2>Tracks we cannot redistribute</h2>') + w('<p>') + w('These tracks reach us under terms that let us display the data but not pass it on.') + w('You can see them on our site, and in most cases you can obtain the same data yourself') + w('directly from the group that produced it, but we cannot include them in a mirror or on') + w('our download server. The reasons vary: some are commercial licences, others are') + w('consent agreements attached to human cohorts. Check the description page of an') + w('individual track for who to approach about access.') + w('</p>') + w('<table>') + w(' <tr>') + w(' <th>Track</th>') + w(' <th>Table or track name</th>') + w(' <th>Assemblies</th>') + w(' </tr>') + by = rows_by_track(d["restricted"]) for t, e in sorted(by.items(), key=lambda x: (x[1]["label"] or x[0]).lower()): - rows1.append("<tr><td>%s</td><td><code>%s</code></td><td>%s</td><td class='mut'>%s</td></tr>" - % (esc(e["label"] or t), esc(t), esc(" ".join(sorted(set(e["dbs"])))), - esc("; ".join(sorted(e["why"]))))) - - # ---- exposed + w(' <tr>') + cell(w, e["label"] or t) + w(' <td><code>%s</code></td>' % esc(t)) + cell(w, " ".join(sorted(e["dbs"]))) + w(' </tr>') + w('</table>') + w('') + w('<h3>How this list is put together</h3>') + w('<p>') + w('A track appears above if any of three things is true of it: its configuration says') + w('<code>tableBrowser off</code>; its <code>noGenomeReason</code> refers to distribution') + w('terms, which is how OMIM is marked and is missed by a search for the first setting') + w('alone; or its table exists on our servers but is deliberately absent from the download') + w('server. No single one of those catches everything, so all three are checked. Note that') + w('some tracks are withheld from whole-genome Table Browser queries only because they are') + w('too large to return, not for any licensing reason, and those are not listed above.') + w('</p>') exposed = d.get("exposed", []) - if exposed and not a.internal: - # Never name reachable restricted files on a page anyone can read. - exposed_html = ("<p>The list above is cross-checked against the download server every " - "time this page is built, so that a track marked as restricted here is " - "actually blocked there. Discrepancies are reported to us privately " - "rather than shown on this page.</p>") - elif exposed: - ex = "".join("<tr><td>%s</td><td><code>%s</code></td><td><code>%s</code></td></tr>" - % (esc(r["shortLabel"] or r["track"]), esc(r["db"]), esc(r["path"])) - for r in exposed) - exposed_html = ("<div class='note warn'><p><strong>%d file(s) marked as restricted are " - "currently reachable on hgdownload.</strong> These need to go on the " - "download server's exclude list.</p></div>" - "<div class='scroll'><table><tr><th>Track</th><th>Assembly</th>" - "<th>Path</th></tr>%s</table></div>" % (len(exposed), ex)) + if exposed and a.internal: + w('<h3>Reachable on hgdownload</h3>') + w('<p>') + w('%d file(s) marked as restricted are currently served by the download server and' + % len(exposed)) + w('need to be added to its exclude list.') + w('</p>') + w('<table>') + w(' <tr>') + w(' <th>Track</th>') + w(' <th>Assembly</th>') + w(' <th>Path</th>') + w(' </tr>') + for r in exposed: + w(' <tr>') + cell(w, r["shortLabel"] or r["track"]) + cell(w, r["db"]) + w(' <td><code>%s</code></td>' % esc(r["path"])) + w(' </tr>') + w('</table>') else: - exposed_html = ("<p>Every file marked as restricted is correctly blocked on " - "hgdownload. Checked on %s.</p>" % esc(today)) - - # ---- list 2: otto - data_jobs = [j for j in d["otto"] if j["kind"] in ("track","hub","table")] + w('<p>') + w('Every track named above is checked against the download server each time this page') + w('is rebuilt, so that a track listed as restricted is genuinely blocked there.') + w('</p>') + w('') + + # ---- 2. otto ---------------------------------------------------------- + w("<a name='autoUpdating'></a>") + w('<h2>Tracks that update themselves</h2>') + w('<p>') + w('These tracks are rebuilt on a schedule without anyone at UCSC touching them. If you') + w('mirror them, your copy will drift from ours until you synchronise again. Times are US') + w('Pacific.') + w('</p>') + w('<table>') + w(' <tr>') + w(' <th>Source</th>') + w(' <th>Updated</th>') + w(' <th>Tracks affected</th>') + w(' <th>Assemblies</th>') + w(' </tr>') + jobs = [j for j in d["otto"] if j["kind"] in ("track", "hub", "table")] + for j in sorted(jobs, key=lambda x: x["name"].lower()): + w(' <tr>') + cell(w, j["name"]) + cell(w, j["schedule"]) + cell(w, j["detail"]) + cell(w, j.get("assemblies", "") if j["kind"] == "track" else "") + w(' </tr>') + w('</table>') notifiers = [j for j in d["otto"] if j["kind"] == "notifier"] - infra = [j for j in d["otto"] if j["kind"] == "infrastructure"] - unc = [j for j in d["otto"] if j["kind"] == "unclassified"] - rows2 = "".join( - "<tr><td>%s</td><td>%s</td><td>%s</td><td class='n'>%s</td></tr>" - % (esc(j["name"]), esc(j["schedule"]), esc(j["detail"]), - esc(j.get("assemblies","")) if j["kind"]=="track" else "") - for j in sorted(data_jobs, key=lambda x: x["name"].lower())) - rows2b = "".join("<tr><td>%s</td><td>%s</td><td>%s</td></tr>" - % (esc(j["name"]), esc(j["schedule"]), esc(j["detail"])) - for j in notifiers) - - # ---- list 3: contributed + if notifiers: + w('') + w('<h3>Scheduled checks that change no data</h3>') + w('<p>') + w('These watch for new releases upstream and send us mail. They update nothing on') + w('their own, and are listed so that the schedule above is not mistaken for the whole') + w('picture.') + w('</p>') + w('<table>') + w(' <tr>') + w(' <th>Check</th>') + w(' <th>Runs</th>') + w(' <th>What it looks at</th>') + w(' </tr>') + for j in notifiers: + w(' <tr>') + cell(w, j["name"], indent=" ") + cell(w, j["schedule"], indent=" ") + cell(w, j["detail"], indent=" ") + w(' </tr>') + w('</table>') + w('') + + # ---- 3. contributed --------------------------------------------------- contrib = d.get("contrib", []) - rows3 = "".join("<tr><td>%s</td><td class='n'>%s</td></tr>" % (esc(c["name"]), c["assemblies"]) - for c in contrib) - contrib_total = sum(c["assemblies"] for c in contrib) - - doc = f"""<!DOCTYPE html> -<html lang="en"><head><meta charset="utf-8"> -<meta name="viewport" content="width=device-width, initial-scale=1"> -<title>Track redistribution, automatic updates, and contributed tracks</title> -<style>{CSS}</style></head><body><div class="wrap"> - -<header> -<h1>Which tracks you can mirror, which update themselves, and which came from outside</h1> -<div class="sub">Generated {esc(today)} from the hgwbeta trackDb, the otto crontab, and the -GenArk build tree · Redmine -<a href="https://redmine.gi.ucsc.edu/issues/37781">#37781</a></div> -</header> - -<p>People running their own copy of the Genome Browser ask us three questions often enough that -it is worth answering them in one place: which tracks we are not allowed to pass on, which tracks -change on their own, and which tracks were built by someone other than UCSC.</p> - -<h2>1. Tracks we cannot redistribute</h2> -<p>These come to us under terms that let us display the data but not hand it on. You can see them -on our site, and in most cases you can get the same data yourself directly from the group that -produced it, but we cannot include them in a mirror or on the download server. The reasons vary: -some are commercial licences, some are consent agreements attached to human cohorts.</p> -<div class="scroll"><table> -<tr><th>Track</th><th>Table or track name</th><th>Assemblies</th><th>How it is marked</th></tr> -{''.join(rows1)} -</table></div> -<p class="mut">A track is listed here if any of these is true: its trackDb entry says -<code>tableBrowser off</code>; its <code>noGenomeReason</code> mentions distribution terms (this -is how OMIM is marked, and it is missed by a query that only looks for <code>off</code>); or its -table exists on our servers but is deliberately absent from hgdownload. No single one of those -catches everything, so the page checks all three.</p> - -<h3>Cross-check against the download server</h3> -{exposed_html} - -<h2>2. Tracks that update themselves</h2> -<p>These are refreshed on a schedule without anyone touching them, by the process we call otto. -If you mirror them, they will drift from our copy unless you re-sync. The times are US Pacific.</p> -<div class="scroll"><table> -<tr><th>Source</th><th>Runs</th><th>Tracks it updates</th><th class="n">Assemblies</th></tr> -{rows2} -</table></div> - -<h3>Scheduled checks that do not change data</h3> -<p>These watch for new releases upstream and email us; they update nothing on their own.</p> -<div class="scroll"><table> -<tr><th>Job</th><th>Runs</th><th>What it checks</th></tr> -{rows2b} -</table></div> -<p class="mut">A further {len(infra)} otto jobs handle pushes, mirrors, logs and other -housekeeping rather than track data, so they are left out here. -{('<strong>' + str(len(unc)) + ' job(s) could not be classified and need a look.</strong>') if unc else ''}</p> - -<h2>3. Contributed tracks</h2> -<p>Some GenArk assemblies carry annotation built by outside groups rather than by us. The data -sits alongside our own tracks but the group named below produced it, and questions about the -underlying annotation are best sent to them.</p> -<div class="scroll"><table> -<tr><th>Contributor</th><th class="n">Assemblies</th></tr> -{rows3} -</table></div> -<p class="mut">{contrib_total} assembly/contributor pairings across -{len(contrib)} contributing groups.</p> - -<footer> -Built by <code>collect.py</code> and <code>mkPage.py</code> · sources: hgwbeta trackDb -({d['counts'].get('databases','?')} databases), <code>~/kent/src/hg/utils/otto/otto.crontab</code>, -<code>/gbdb/genark/*/contrib/</code>, and live checks against hgdownload. -</footer> -</div></body></html>""" - open(a.out,"w").write(doc) - print("wrote %s (%d bytes)" % (a.out, len(doc))) + w("<a name='contributed'></a>") + w('<h2>Contributed tracks</h2>') + w('<p>') + w('Some assemblies in our') + w('<a href="https://hgdownload.soe.ucsc.edu/hubs/" target="_blank">GenArk</a> collection') + w('carry annotation built by outside groups rather than by UCSC. The data sit alongside') + w('our own tracks, but the group named below produced them, and questions about the') + w('underlying annotation are best sent to that group.') + w('</p>') + w('<table>') + w(' <tr>') + w(' <th>Contributing group</th>') + w(' <th>Assemblies</th>') + w(' </tr>') + for c in contrib: + w(' <tr>') + w(' <td>%s</td>' % esc(c["name"])) + w(' <td>%d</td>' % c["assemblies"]) + w(' </tr>') + w('</table>') + w('<p>') + w('%d assemblies carry contributed annotation, from %d groups.' + % (sum(c["assemblies"] for c in contrib), len(contrib))) + w('</p>') + w('') + w('<p>') + w('This page was generated on %s from the current state of our servers.' % esc(today)) + w('</p>') + w('') + w('<!--#include virtual="$ROOT/inc/gbPageEnd.html" -->') + + text = "\n".join(o) + "\n" + open(a.out, "w").write(text) + longest = max(len(x) for x in o) + print("wrote %s (%d bytes, longest line %d)" % (a.out, len(text), longest)) if __name__ == "__main__": main()