c2087cb0ed189d946439a090d43d2dba5d71a9eb
lrnassar
  Wed Aug 5 17:28:22 2026 -0700
Fix error containment and a wasted retry sleep in the VCEP version notifier per CR feedback. refs #37795

ClinGen's version field is a free-form string, so an unparseable value raised
ValueError past the per-VCEP except RuntimeError in checkVcep. That aborted the
whole run, skipping the remaining VCEPs and mailing a traceback instead of the
report. normalizeVersion now reports the offending string as a RuntimeError, and
the version comparison moved inside the try so it is caught, keeping the damage
to a single failure line.

fetchUrl also slept 30 seconds after its final failed attempt, which could not
help. It now only sleeps between attempts.

diff --git src/hg/utils/otto/vcepVersions/checkVcepVersions.py src/hg/utils/otto/vcepVersions/checkVcepVersions.py
index be80d370ca2..8af0f3c687e 100755
--- src/hg/utils/otto/vcepVersions/checkVcepVersions.py
+++ src/hg/utils/otto/vcepVersions/checkVcepVersions.py
@@ -1,155 +1,163 @@
 #!/usr/bin/env python3
 """checkVcepVersions.py -- monthly notifier (RM #37795).
 
 Compares the CSpec specification version shown on each of our VCEP hub
 description pages against the current released version in the ClinGen CSpec
 registry. Emails the otto MAILTO ONLY when they disagree, or when the check
 itself fails. Silent (no output, so cron sends no mail) when everything matches.
 
 Our side is read from the live hgdownload description page rather than the hive
 source, so this also catches a hub that was updated on hive but never pushed.
 
 The ClinGen side comes from the CSpec affiliation page, which embeds its
 specification list as an inline "svisData = [...]" JSON assignment in the raw
 HTML. No JavaScript needed, but it is their UI's internal format, so a parse
 failure is reported rather than ignored.
 
 Two things this deliberately does NOT do:
   - Compare against unreleased specs. Pilot/in-prep entries are skipped.
   - Compare against genes our hub does not cover. An affiliation can hold specs
     for more genes than we display (InSiGHT 50099 also carries APC and MUTYH),
     and those move on their own schedule.
 
 To silence after updating a hub: update the version string on that hub's
 description page and push it to hgdownload.
 """
 
 import json
 import re
 import sys
 import time
 import urllib.error
 import urllib.request
 
 # One entry per VCEP hub in the recommended track sets
 # (kent/src/hg/htdocs/data/recTrackSets/recTrackSets.hg*.tab).
 # hubRegex must capture the dotted version from that hub's description page.
 vcepConfig = {
     "ENIGMA BRCA1/BRCA2 VCEP": {
         "hubUrl": "https://hgdownload.soe.ucsc.edu/hubs/enigma/enigma.html",
         "hubRegex": r"Guidelines for BRCA1/BRCA1\s+Version\s+(\d+(?:\.\d+)+)",
         "affiliation": "50087",
         "genes": ["BRCA1", "BRCA2"],
     },
     "InSiGHT Lynch Syndrome VCEP": {
         "hubUrl": "https://hgdownload.soe.ucsc.edu/hubs/insight/insight.html",
         "hubRegex": r"<h1>InSiGHT specs\s+(\d+(?:\.\d+)+)</h1>",
         "affiliation": "50099",
         "genes": ["MLH1", "MSH2", "MSH6", "PMS2"],
     },
 }
 
 cspecUrl = "https://cspec.genome.network/cspec/ui/svi/affiliation/"
 
 
 def fetchUrl(url):
     """Fetch url and return its text. Retries a few times so a transient network
     blip does not turn into a false alarm, then raises."""
+    attempts = 5
     lastErr = None
-    for attempt in range(5):
+    for attempt in range(attempts):
         try:
             with urllib.request.urlopen(url, timeout=120) as resp:
                 return resp.read().decode("utf-8", "replace")
         except (urllib.error.URLError, OSError) as e:
             lastErr = e
+            if attempt < attempts - 1:
                 time.sleep(30)
-    raise RuntimeError("could not fetch " + url + " after 5 attempts: " + str(lastErr))
+    raise RuntimeError("could not fetch " + url + " after " + str(attempts) +
+                       " attempts: " + str(lastErr))
 
 
 def normalizeVersion(version):
     """Turn a dotted version into a tuple for comparison, dropping trailing
-    zeroes so ClinGen's '2.0' matches our page's '2.0.0'."""
+    zeroes so ClinGen's '2.0' matches our page's '2.0.0'. ClinGen's version is a
+    free-form string, so an unparseable one is reported rather than raising past
+    the per-VCEP error handling and killing the rest of the run."""
+    try:
         parts = [int(p) for p in version.split(".")]
+    except ValueError:
+        raise RuntimeError("could not parse version string '" + version + "'")
     while len(parts) > 1 and parts[-1] == 0:
         parts.pop()
     return tuple(parts)
 
 
 def getOurVersion(config):
     """Scrape the version we publish from a hub description page."""
     html = fetchUrl(config["hubUrl"])
     match = re.search(config["hubRegex"], html)
     if match is None:
         raise RuntimeError("no version found on " + config["hubUrl"] +
                            " (page wording changed? regex: " + config["hubRegex"] + ")")
     return match.group(1)
 
 
 def getClinGenVersions(config):
     """Return {gene: version} for the released CSpec specs covering our genes."""
     url = cspecUrl + config["affiliation"]
     html = fetchUrl(url)
     match = re.search(r"svisData\s*=\s*(\[.*?\])\s*;", html, re.S)
     if match is None:
         raise RuntimeError("no svisData block found at " + url + " (registry page format changed?)")
     try:
         svis = json.loads(match.group(1))
     except ValueError as e:
         raise RuntimeError("could not parse svisData JSON at " + url + ": " + str(e))
 
     versions = {}
     for svi in svis:
         if not svi.get("isReleased"):
             continue
         for gene in svi.get("genes", []):
             if gene.get("label") in config["genes"]:
                 versions[gene["label"]] = svi["version"]
 
     missing = [g for g in config["genes"] if g not in versions]
     if missing:
         raise RuntimeError("no released CSpec spec for " + ", ".join(missing) + " at " + url)
     return versions
 
 
 def checkVcep(name, config, report):
     """Compare one VCEP and append any mismatch to report. Returns True on
     success, False if the check itself could not be completed."""
     try:
         ourVersion = getOurVersion(config)
         clinGenVersions = getClinGenVersions(config)
+        stale = {g: v for g, v in clinGenVersions.items()
+                 if normalizeVersion(v) != normalizeVersion(ourVersion)}
     except RuntimeError as e:
         report.append(name + ": check failed: " + str(e))
         return False
 
-    stale = {g: v for g, v in clinGenVersions.items()
-             if normalizeVersion(v) != normalizeVersion(ourVersion)}
     if stale:
         geneList = ", ".join(g + "=" + stale[g] for g in sorted(stale))
         report.append(name + " is out of date.")
         report.append("  our hub page: " + ourVersion + "  (" + config["hubUrl"] + ")")
         report.append("  ClinGen CSpec: " + geneList +
                       "  (" + cspecUrl + config["affiliation"] + ")")
     return True
 
 
 def main():
     report = []
     ok = True
     for name in sorted(vcepConfig):
         if not checkVcep(name, vcepConfig[name], report):
             ok = False
 
     if report:
         print("VCEP specification versions need attention. refs #37795")
         print("")
         for line in report:
             print(line)
         print("")
         print("To silence: update the version on the hub description page and push it")
         print("to hgdownload, or fix the check in")
         print("kent/src/hg/utils/otto/vcepVersions/checkVcepVersions.py")
 
     sys.exit(0 if ok else 1)
 
 
 main()