497f2788d49fda531c0836618688f954486eb87c
mspeir
  Fri Sep 4 09:44:13 2026 -0700
trackLists: do not let the public page claim the download cross-check is clean, refs #37781

The static-page rewrite collapsed mkPage.py's three-way branch on the
hgdownload cross-check into two, so the public run fell into the all-clear
wording whether or not collect.py had found a restricted file that is still
reachable. collect.py's current output has three such files, so the page was
asserting the opposite of what the check found. Restored as three cases: the
public page says only that the check runs and that anything found is reported
privately, the --internal page keeps the table of paths, and the all-clear
wording is used only when the list really is empty. The comment explaining why
the public branch must not be merged back into the all-clear one is back too.

collect.py counted any HTTP code that was not literally 404 as reachable, which
made a curl timeout or a failed connection (empty output, or 000) look like an
exposed file. That mails false alarms and, now that the branch above depends on
it, would drop the all-clear line from the public page on a network blip. Only
2xx and 3xx count as served, 4xx as blocked, and anything else is reported
separately as not checked, in collected.json and on stderr.

Also: the remaining unquoted interpolations into shell=True commands go through
the existing q() helper, rows_by_track() no longer builds the "why" set that the
new table does not use, and US spelling throughout.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

diff --git src/hg/utils/otto/trackLists/mkPage.py src/hg/utils/otto/trackLists/mkPage.py
index 033a969ed86..e12f362fec8 100755
--- src/hg/utils/otto/trackLists/mkPage.py
+++ src/hg/utils/otto/trackLists/mkPage.py
@@ -13,36 +13,37 @@
     return s.encode("ascii", "xmlcharrefreplace").decode("ascii")
 
 def cell(w, text, indent="    "):
     """Write a <td>, wrapping so no source line runs past 100 characters."""
     text = esc(text)
     if len(indent) + len(text) + 9 <= 100:
         w("%s<td>%s</td>" % (indent, text))
         return
     w("%s<td>" % indent)
     for line in textwrap.wrap(text, width=94 - len(indent),
                               break_long_words=False, break_on_hyphens=False):
         w("%s  %s" % (indent, line))
     w("%s</td>" % indent)
 
 def rows_by_track(restricted):
-    by = collections.defaultdict(lambda: dict(dbs=set(), label="", why=set()))
+    """Group the restricted rows by track. The per-row "why" is deliberately not
+    carried through: which tests fired is explained once in prose below the table."""
+    by = collections.defaultdict(lambda: dict(dbs=set(), label=""))
     for r in restricted:
         e = by[r["track"]]
         e["dbs"].add(r["db"])
         e["label"] = e["label"] or r.get("shortLabel", "")
-        e["why"].update(r.get("why", []))
     return by
 
 def main():
     ap = argparse.ArgumentParser()
     ap.add_argument("-i", "--inp", default="collected.json")
     ap.add_argument("-o", "--out", default="trackLists.html")
     ap.add_argument("--date", default=None)
     ap.add_argument("--internal", action="store_true",
                     help="include the hgdownload cross-check, which names restricted files "
                          "that are currently reachable. Never use for a public page.")
     a = ap.parse_args()
     d = json.load(open(a.inp))
     today = a.date or d.get("generated") or datetime.date.today().isoformat()
     o = []
     w = o.append
@@ -73,93 +74,105 @@
     w('<p>')
     w('For installation instructions see the')
     w('<a href="goldenPath/help/mirror.html">mirror documentation</a>. Questions are welcome')
     w('on the <a href="goldenPath/help/mirror.html#the-genome-mirror-mailing-list">genome-mirror')
     w('mailing list</a>.')
     w('</p>')
     w('')
 
     # ---- 1. not redistributable -------------------------------------------
     w("<a name='notRedistributable'></a>")
     w('<h2>Tracks we cannot redistribute</h2>')
     w('<p>')
     w('These tracks reach us under terms that let us display the data but not pass it on.')
     w('You can see them on our site, and in most cases you can obtain the same data yourself')
     w('directly from the group that produced it, but we cannot include them in a mirror or on')
-    w('our download server. The reasons vary: some are commercial licences, others are')
+    w('our download server. The reasons vary: some are commercial licenses, others are')
     w('consent agreements attached to human cohorts. Check the description page of an')
     w('individual track for who to approach about access.')
     w('</p>')
     w('<table>')
     w('  <tr>')
     w('    <th>Track</th>')
     w('    <th>Table or track name</th>')
     w('    <th>Assemblies</th>')
     w('  </tr>')
     by = rows_by_track(d["restricted"])
     for t, e in sorted(by.items(), key=lambda x: (x[1]["label"] or x[0]).lower()):
         w('  <tr>')
         cell(w, e["label"] or t)
         w('    <td><code>%s</code></td>' % esc(t))
         cell(w, " ".join(sorted(e["dbs"])))
         w('  </tr>')
     w('</table>')
     w('')
     w('<h3>How this list is put together</h3>')
     w('<p>')
     w('A track appears above if any of three things is true of it: its configuration says')
     w('<code>tableBrowser off</code>; its <code>noGenomeReason</code> refers to distribution')
     w('terms, which is how OMIM is marked and is missed by a search for the first setting')
     w('alone; or its table exists on our servers but is deliberately absent from the download')
     w('server. No single one of those catches everything, so all three are checked. Note that')
     w('some tracks are withheld from whole-genome Table Browser queries only because they are')
     w('too large to return, not for any licensing reason, and those are not listed above.')
     w('</p>')
     exposed = d.get("exposed", [])
-    if exposed and a.internal:
+    if exposed and not a.internal:
+        # Never name reachable restricted files on a page anyone can read: the path
+        # of a file we should be blocking is a pointer straight at it. Say only that
+        # the check runs; the internal copy and the cron mail carry the detail. Do
+        # not fold this branch into the all-clear one below, which would have the
+        # public page claim the list is clean when the check says otherwise.
+        w('<p>')
+        w('Every track named above is cross-checked against the download server each time')
+        w('this page is rebuilt. Any file that turns out to be reachable there is reported')
+        w('to us privately rather than named on this page.')
+        w('</p>')
+    elif exposed:
         w('<h3>Reachable on hgdownload</h3>')
         w('<p>')
         w('%d file(s) marked as restricted are currently served by the download server and'
           % len(exposed))
         w('need to be added to its exclude list.')
         w('</p>')
         w('<table>')
         w('  <tr>')
         w('    <th>Track</th>')
         w('    <th>Assembly</th>')
         w('    <th>Path</th>')
         w('  </tr>')
         for r in exposed:
             w('  <tr>')
             cell(w, r["shortLabel"] or r["track"])
             cell(w, r["db"])
             w('    <td><code>%s</code></td>' % esc(r["path"]))
             w('  </tr>')
         w('</table>')
     else:
         w('<p>')
         w('Every track named above is checked against the download server each time this page')
-        w('is rebuilt, so that a track listed as restricted is genuinely blocked there.')
+        w('is rebuilt, and every file marked as restricted is correctly blocked there. Checked')
+        w('on %s.' % esc(today))
         w('</p>')
     w('')
 
     # ---- 2. otto ----------------------------------------------------------
     w("<a name='autoUpdating'></a>")
     w('<h2>Tracks that update themselves</h2>')
     w('<p>')
     w('These tracks are rebuilt on a schedule without anyone at UCSC touching them. If you')
-    w('mirror them, your copy will drift from ours until you synchronise again. Times are US')
+    w('mirror them, your copy will drift from ours until you synchronize again. Times are US')
     w('Pacific.')
     w('</p>')
     w('<table>')
     w('  <tr>')
     w('    <th>Source</th>')
     w('    <th>Updated</th>')
     w('    <th>Tracks affected</th>')
     w('    <th>Assemblies</th>')
     w('  </tr>')
     jobs = [j for j in d["otto"] if j["kind"] in ("track", "hub", "table")]
     for j in sorted(jobs, key=lambda x: x["name"].lower()):
         w('  <tr>')
         cell(w, j["name"])
         cell(w, j["schedule"])
         cell(w, j["detail"])
@@ -185,32 +198,32 @@
             w('  <tr>')
             cell(w, j["name"], indent="    ")
             cell(w, j["schedule"], indent="    ")
             cell(w, j["detail"], indent="    ")
             w('  </tr>')
         w('</table>')
     w('')
 
     # ---- 3. contributed ---------------------------------------------------
     contrib = d.get("contrib", [])
     w("<a name='contributed'></a>")
     w('<h2>Contributed tracks</h2>')
     w('<p>')
     w('Some assemblies in our')
     w('<a href="https://hgdownload.soe.ucsc.edu/hubs/" target="_blank">GenArk</a> collection')
-    w('carry annotation built by outside groups rather than by UCSC. The data sit alongside')
-    w('our own tracks, but the group named below produced them, and questions about the')
+    w('carry annotation built by outside groups rather than by UCSC. The data sits alongside')
+    w('our own tracks, but the group named below produced it, and questions about the')
     w('underlying annotation are best sent to that group.')
     w('</p>')
     w('<table>')
     w('  <tr>')
     w('    <th>Contributing group</th>')
     w('    <th>Assemblies</th>')
     w('  </tr>')
     for c in contrib:
         w('  <tr>')
         w('    <td>%s</td>' % esc(c["name"]))
         w('    <td>%d</td>' % c["assemblies"])
         w('  </tr>')
     w('</table>')
     w('<p>')
     w('%d assemblies carry contributed annotation, from %d groups.'