70ac948e82b41ef316523635c04e5e2c4a89c417 mspeir Fri Sep 4 16:57:54 2026 -0700 trackLists: give the variant frequency projects their own table, move the page to goldenPath/help/mirrorTracks.html, add the otto cron line, refs #37781 Half the restricted list was national sequencing cohorts sitting under varFreqs and phasedVars, and in one alphabetical table they buried the tracks people actually write in about, OMIM, HGMD and DECIPHER. Those cohorts now get a table of their own below the rest. The split is read off the trackDb parent chain, so the next cohort added under varFreqs lands in the right table with no edit here. The page moves off the htdocs root to goldenPath/help/mirrorTracks.html, beside mirror.html, which now links to it. That link goes in src/product/README.txt, the pandoc source mirror.html is generated from. The licensing page link follows the move, and the page title changes with the file name. Also adds the weekly otto line, placed above the HGDB_CONF that would otherwise apply to it, and keeps the job from listing itself as a self-updating track. Co-Authored-By: Claude Opus 5 (1M context) diff --git src/hg/utils/otto/trackLists/collect.py src/hg/utils/otto/trackLists/collect.py index 1f756fd66b5..69ca533b99d 100755 --- src/hg/utils/otto/trackLists/collect.py +++ src/hg/utils/otto/trackLists/collect.py @@ -64,30 +64,48 @@ tdb = collections.defaultdict(dict) def one(db): rows = {} for line in sh("hgsql -h %s -N -e %s %s 2>/dev/null" % (q(BETA), q("select tableName, settings from trackDb"), q(db))).splitlines(): p = line.split("\t") if len(p) >= 2: rows[p[0]] = parse_settings(p[1]) return db, rows with ThreadPoolExecutor(max_workers=8) as ex: for db, rows in ex.map(one, dbs): tdb[db] = rows return tdb +def container_of(tt, track): + """Return the outermost composite or supertrack a track hangs off, "" if none. + + The page splits the restricted list into cohort variant-frequency projects and + everything else. Nothing on an individual track says which it is; the only place + that is written down is the container it belongs to, so record it here and let + the page decide. Walks with a seen set because a parent loop in trackDb should + produce an empty answer, not hang the nightly run.""" + cur, seen = track, set() + while cur not in seen: + seen.add(cur) + p = (tt.get(cur) or {}).get("parent") or (tt.get(cur) or {}).get("subTrack") or "" + p = p.split()[0] if p else "" # value is "varFreqs on", we want the name + if not p or p not in tt: + break + cur = p + return "" if cur == track else cur + LICENSE_RE = re.compile(r"distribut|licen|restrict|permission|agreement", re.I) def restricted_from_trackdb(tdb): out = {} for db, tt in tdb.items(): for t, s in tt.items(): tb, ngr = s.get("tableBrowser", ""), s.get("noGenomeReason", "") why = None if tb.split()[:1] == ["off"]: why = "tableBrowser off" elif ngr and LICENSE_RE.search(ngr): why = "noGenomeReason cites distribution terms" if why: out[(db, t)] = dict(shortLabel=s.get("shortLabel", ""), longLabel=s.get("longLabel", ""), @@ -180,31 +198,34 @@ "malacards":("MalaCards","table", "loads the hg38 malacards table, which no track points at"), "refSeqHistorical":("RefSeq Historical","notifier", "checks whether NCBI has a new release; changes no data"), "vcepVersions":("VCEP spec versions","notifier", "compares our VCEP pages against the ClinGen registry"), } # commands the keyword match gets wrong or too coarse OVERRIDE = { "omimUploadWrapper": ("infrastructure", "pushes the OMIM tables to hgwbeta"), "covidCheck": ("UniProt (wuhCor1)", "UniProt; Mutations on wuhCor1"), "clinGenCspec": ("ClinGen CSpec", "ClinGen VCEP Specifications"), } INFRA = ["readOnlyKentMirror","lastLog","ottoCompareGitVsHiveFiles","liftRequest", "GenArk","buildPublicSessionThumbnails","generateTipOfDay","cellBrowser", - "cbAnnotServer","tabulate_facets","tsv_to_json","updateNewsSec","tusd"] + "cbAnnotServer","tabulate_facets","tsv_to_json","updateNewsSec","tusd", + # this job itself: it writes a page, it does not touch track data, and a + # page that listed its own generator would be its own first entry + "trackLists"] def cron_english(s): if s.startswith("@"): return s m, h, dom, mon, dow = s.split() first = lambda x: int(x.split(",")[0]) if x.split(",")[0].isdigit() else 0 t = "%02d:%02d" % (first(h), first(m)) if dom != "*": return ("monthly on day %s at %s" % (dom, t) if mon == "*" else "day %s of months %s at %s" % (dom, mon, t)) if dow in ("*", "1-7", "0-6"): return "daily at %s" % t if dow == "1-5": return "weekdays at %s" % t days = {"1":"Mon","2":"Tue","3":"Wed","4":"Thu","5":"Fri","6":"Sat","0":"Sun","7":"Sun", @@ -321,30 +342,36 @@ codes = list(ex.map(lambda c: http_code(c[2]), checks)) exposed, unchecked = [], [] for (db, t, p), code in zip(checks, codes): restricted[(db, t)]["hgdownload"] = code # Only a real HTTP response says anything about the file. A curl that timed # out or could not connect comes back empty or as 000, and calling that # "reachable" both mails a false alarm and drops the all-clear line from the # public page on nothing more than a network blip. 4xx means blocked, 2xx and # 3xx mean served, and anything else means the test did not run. if code[:1] in ("2", "3"): exposed.append(dict(db=db, track=t, path=p, code=code, shortLabel=restricted[(db, t)]["shortLabel"])) elif code[:1] != "4": unchecked.append(dict(db=db, track=t, path=p, code=code or "none")) + # which composite or supertrack each one belongs to; the page groups on this + for (db, t), v in restricted.items(): + c = container_of(tdb[db], t) + v["container"] = c + v["containerLabel"] = (tdb[db].get(c) or {}).get("shortLabel", "") if c else "" + contrib = [] if a.no_contrib else contrib_crawl(a.cache, a.refresh_contrib) result = dict( restricted=[dict(db=db, track=t, **v) for (db, t), v in sorted(restricted.items())], exposed=exposed, otto=parse_otto(a.otto_crontab, tdb), contrib=contrib, partialDownloads=partial, uncheckedDownloads=unchecked, counts=dict(databases=len(dbs)), generated=time.strftime("%Y-%m-%d"), ) json.dump(result, open(a.out, "w"), indent=1) note("wrote %s in %.0fs: %d restricted rows, %d exposed, %d otto jobs, %d contributors" % (a.out, time.time() - t0, len(result["restricted"]), len(exposed),