fed2e9c1d310026a998c2212a2e2508670a16373
braney
  Mon Sep 28 15:16:04 2026 -0700
trackCheckRobot: probe faceted subtracks again, keep hub-backed dbs that have tables, show HGERROR text, refs #37424

Since #37662 a subtrack of a faceted composite can no longer override its
hidden ancestors, so "<subtrack>=full" after hideAll drew nothing. hg38 fell
from 19608 checked tracks to 9927, and mm10 from 8095 to 6182. For those
subtracks the robot now sets the parent composite and superTrack to full and
adds <subtrack>_sel=1. The other default-selected subtracks draw too, so only
hgc links with g=<subtrack> are kept.

The hub-backed skip also dropped hs1, rn8 and mpxvRivers. They have a "hub:"
nibPath but also a MySQL database, and they were checked before. Only
hub-backed assemblies without a database are skipped now.

HGERROR lines now include the message from the page, for example
"No chainAnoCar1 track in database hg38 for chr7".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

diff --git src/utils/qa/weeklybld/trackCheckRobot.py src/utils/qa/weeklybld/trackCheckRobot.py
index 7ae3fa2df11..c0f4492d070 100755
--- src/utils/qa/weeklybld/trackCheckRobot.py
+++ src/utils/qa/weeklybld/trackCheckRobot.py
@@ -1,50 +1,54 @@
 #!/usr/bin/env python3
 """TrackCheck robot - exercise every track on every active assembly.
 
 For each active assembly in hgcentralbeta.dbDb:
   - Look up default position
   - For each track in that assembly's trackDb:
       * Hide all tracks (GET hgTracks with hgt.hideAll=yes) using a session cookie jar
-      * Enable just this track (GET hgTracks with <track>=full)
+      * Enable just this track (GET hgTracks with <track>=full).  A subtrack of a
+        faceted composite can no longer override its hidden ancestors (#37662), so
+        for those the parent chain is set to full and <track>_sel=1 is added.
       * Scrape hgc URLs from the rendered track image map (<AREA HREF=...>)
       * GET up to MAX_LINKS_PER_TRACK of them, check 200 and "HGERROR" in body
 
 Replaces the Java TrackCheck.java which did the same via HttpUnit.
 
 Usage:
   trackCheckRobot.py <propsfile-or-default>
 
 Props file (key space value or key=value per line) keys used:
   httpProto   https (default) or http
   server      web server to test (e.g. hgwbeta.soe.ucsc.edu)
   dbSpec      "all" or a specific assembly
   table       "all" or a specific track
 
 DB credentials come from $HGDB_CONF or ~/.hg.conf.beta (passed to hgsql).
 """
 import argparse
 import http.cookiejar
 import os
 import re
 import subprocess
 import sys
 import time
 from urllib.parse import urlencode, urljoin
 import urllib.request
 import urllib.error
 
+from trackCheckAnalyze import extract_error_text
+
 # Seconds to wait for a response.  The wuhCor1 phylogenetic tree tracks take ~40s in hgTracks,
 # and the hg38 470-way and 241-way alignment details pages take 2-6 minutes in hgc.
 HGTRACKS_TIMEOUT = 120
 HGC_TIMEOUT = 600
 HTTP_PIX = "1200"
 MAX_LINKS_PER_TRACK = 4
 
 
 class Counters:
     checked = 0
     skipped = 0
     errors = 0
 
 
 def log(msg):
@@ -82,126 +86,174 @@
 
 
 def hgsql(hgdb_conf, db, query):
     env = os.environ.copy()
     env["HGDB_CONF"] = hgdb_conf
     r = subprocess.run(
         ["hgsql", "-N", "-B", db, "-e", query],
         capture_output=True, text=True, env=env,
     )
     if r.returncode != 0:
         raise RuntimeError(f"hgsql on {db} failed: {r.stderr.strip()}")
     return [line for line in r.stdout.splitlines() if line]
 
 
 def active_assemblies(hgdb_conf):
-    # Hub-backed assemblies (nibPath "hub:...") have no MySQL database to list tracks from.
-    return hgsql(hgdb_conf, "hgcentralbeta",
-                 "SELECT name FROM dbDb WHERE active = 1 AND nibPath NOT LIKE 'hub:%'")
+    """Return active assemblies that have a MySQL database to list tracks from.
+    Hub-backed assemblies (nibPath "hub:...") usually have none, but some (hs1,
+    rn8, mpxvRivers) do and still get checked."""
+    databases = set(hgsql(hgdb_conf, "hgcentralbeta", "SHOW DATABASES"))
+    rows = hgsql(hgdb_conf, "hgcentralbeta",
+                 "SELECT name, nibPath FROM dbDb WHERE active = 1")
+    out = []
+    for row in rows:
+        name, _, nib_path = row.partition("\t")
+        if nib_path.startswith("hub:") and name not in databases:
+            continue
+        out.append(name)
+    return out
 
 
 def default_position(hgdb_conf, assembly):
     rows = hgsql(hgdb_conf, "hgcentralbeta",
                  f"SELECT defaultPos FROM dbDb WHERE name = '{assembly}'")
     return rows[0] if rows else None
 
 
 def trackdb_tracks(hgdb_conf, assembly):
-    return hgsql(hgdb_conf, assembly, "SELECT tableName FROM trackDb")
+    """Return (tracks, parents, faceted): track names in trackDb order, a map of
+    track -> parent/superTrack name, and the set of faceted composites."""
+    tracks, parents, faceted = [], {}, set()
+    for row in hgsql(hgdb_conf, assembly, "SELECT tableName, settings FROM trackDb"):
+        name, _, settings = row.partition("\t")
+        tracks.append(name)
+        # hgsql -B escapes the newlines between settings as a literal \n
+        for line in settings.split("\\n"):
+            words = line.split()
+            if len(words) < 2:
+                continue
+            if words[0] in ("parent", "superTrack") and words[1] != "on":
+                parents[name] = words[1]
+            elif words[0] == "compositeTrack" and words[1] == "faceted":
+                faceted.add(name)
+    return tracks, parents, faceted
+
+
+def ancestors(track, parents):
+    """Return the parent chain of track, nearest first."""
+    out = []
+    t = parents.get(track)
+    while t and t not in out:
+        out.append(t)
+        t = parents.get(t)
+    return out
 
 
 def make_opener():
     """Per-track opener with its own cookie jar so carts don't cross-contaminate."""
     jar = http.cookiejar.CookieJar()
     opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar))
     opener.addheaders = [("User-Agent", "TrackCheckRobot/1.0"),
                          ("Accept-Encoding", "identity")]
     return opener
 
 
 def http_get(opener, url, timeout=HGTRACKS_TIMEOUT):
     with opener.open(url, timeout=timeout) as resp:
         return resp.status, resp.read().decode("utf-8", errors="replace")
 
 
 AREA_HGC_RE = re.compile(
     r"<AREA\b[^>]*\bHREF=['\"]([^'\"]*\bhgc\b[^'\"]*)['\"]",
     re.IGNORECASE,
 )
 
 
-def extract_hgc_links(body, base_url):
-    """Return unique hgc URLs found in track image AREA tags."""
+def extract_hgc_links(body, base_url, only_track=None):
+    """Return unique hgc URLs found in track image AREA tags, optionally only
+    those whose g= is only_track."""
     seen = set()
     out = []
+    g_re = re.compile(r"[?&]g=" + re.escape(only_track) + r"(&|$)") if only_track else None
     for m in AREA_HGC_RE.finditer(body):
         href = m.group(1).replace("&amp;", "&")
-        if href in seen:
+        if href in seen or (g_re and not g_re.search(href)):
             continue
         seen.add(href)
         out.append(urljoin(base_url, href))
     return out
 
 
 def check_hgc_urls(opener, urls, log_prefix):
     """GET each URL, report HTTP != 200 or 'HGERROR' in response body."""
     for url in urls:
         try:
             status, body = http_get(opener, url, timeout=HGC_TIMEOUT)
         except urllib.error.HTTPError as e:
             err(f"{log_prefix}: HTTP {e.code} for {url}")
             continue
         except Exception as e:
             err(f"{log_prefix}: fetch failed ({e}) for {url}")
             continue
         if status != 200:
             err(f"{log_prefix}: unexpected response code {status} for {url}")
             continue
-        idx = body.find("HGERROR")
-        if idx >= 0:
-            err(f"{log_prefix}: HGERROR at {url}")
+        if "HGERROR" in body:
+            msgs = extract_error_text(body)
+            what = f" ({'; '.join(msgs)})" if msgs else ""
+            err(f"{log_prefix}: HGERROR{what} at {url}")
 
 
-def exercise_track(http_proto, server, assembly, track, default_pos):
+def exercise_track(http_proto, server, assembly, track, default_pos,
+                   parents, faceted):
     """Two-step GET: hide all, then enable track. Scrape hgc links, GET them."""
     opener = make_opener()
     base = f"{http_proto}://{server}/cgi-bin/hgTracks"
     common = {"db": assembly, "position": default_pos, "pix": HTTP_PIX}
 
     # Step 1: hide everything in the session cart.
     url1 = f"{base}?{urlencode({**common, 'hgt.hideAll': 'yes'})}"
     try:
         status, _ = http_get(opener, url1)
     except Exception as e:
         err(f"{assembly}.{track}: hideAll request failed: {e}")
         return
     if status != 200:
         err(f"{assembly}.{track}: hideAll got HTTP {status}")
         return
 
     # Step 2: enable just this track.
-    url2 = f"{base}?{urlencode({**common, track: 'full'})}"
+    enable = {track: "full"}
+    chain = ancestors(track, parents)
+    in_faceted = any(a in faceted for a in chain)
+    if in_faceted:
+        # Faceted children are clamped by their ancestors' visibility, and the
+        # other default-selected subtracks will draw too, so keep only our links.
+        for a in chain:
+            enable[a] = "full"
+        enable[f"{track}_sel"] = "1"
+    url2 = f"{base}?{urlencode({**common, **enable})}"
     try:
         status, body = http_get(opener, url2)
     except Exception as e:
         err(f"{assembly}.{track}: enable request failed: {e}")
         return
     if status != 200:
         err(f"{assembly}.{track}: enable got HTTP {status}")
         return
 
-    links = extract_hgc_links(body, url2)
+    links = extract_hgc_links(body, url2, track if in_faceted else None)
     if not links:
         # No clickable items at default position — not an error, just no coverage.
         Counters.skipped += 1
         return
 
     check_hgc_urls(opener, links[:MAX_LINKS_PER_TRACK],
                    f"{assembly}.{track}")
     Counters.checked += 1
 
 
 def main():
     ap = argparse.ArgumentParser(description="TrackCheck robot (python port)")
     ap.add_argument("propfile", help="Properties file path or 'default'")
     ap.add_argument(
         "--hgdb-conf",
@@ -220,42 +272,42 @@
             assemblies = active_assemblies(args.hgdb_conf)
         except Exception as e:
             err(f"could not list active assemblies: {e}")
             return 1
     else:
         assemblies = [props["dbSpec"]]
 
     start_time = time.time()
     for asm in assemblies:
         log(f"\nAssembly = {asm}")
         pos = default_position(args.hgdb_conf, asm)
         if not pos:
             err(f"{asm}: no default position in dbDb")
             continue
 
-        if props["table"] == "all":
         try:
-                tracks = trackdb_tracks(args.hgdb_conf, asm)
+            tracks, parents, faceted = trackdb_tracks(args.hgdb_conf, asm)
         except Exception as e:
             err(f"{asm}: could not list tracks: {e}")
             continue
-        else:
+        if props["table"] != "all":
             tracks = [props["table"]]
 
         before_checked = Counters.checked
         before_skipped = Counters.skipped
         for t in tracks:
             try:
-                exercise_track(props["httpProto"], props["server"], asm, t, pos)
+                exercise_track(props["httpProto"], props["server"], asm, t, pos,
+                               parents, faceted)
             except Exception as e:
                 err(f"{asm}.{t}: unexpected exception: {e}")
         log(f"checked {Counters.checked - before_checked} of {len(tracks)} "
             f"tracks ({Counters.skipped - before_skipped} had no clickable items)")
 
     elapsed = time.time() - start_time
     log(f"\nTotal: {Counters.checked} tracks checked, "
         f"{Counters.skipped} no-items, {Counters.errors} errors in {elapsed:.1f}s")
     return 0 if Counters.errors == 0 else 2
 
 
 if __name__ == "__main__":
     sys.exit(main())