2904215904f1e62317b16bb372a6b176c1f800bd
braney
Sat Jul 25 09:58:54 2026 -0700
add an inventory of track-scoped cart variables refs #37838
Before changing the cart format we need to know what is actually stored in it.
cartTrackVarCatalog.py holds a curated catalog of the 340 track-scoped cart
variables in the tree, arranged as the hierarchy a JSON cart would use: track
name, then the variables common to any track, then a layer per track type, then
the leaf groups shared across types. It emits either JSON or a browsable HTML
page, and --check verifies that every leaf group a type refers to exists.
harvestCartVars.py is the scanner that seeds it. It finds the two ways a
track-scoped name gets built, cart*ClosestToHome() and safef("%s.%s"), resolves
macro identifiers to their string values, and attributes each hit to the
function that reads or writes it. Its output still needs curation by hand, since
it cannot tell a cart variable from a table name or a filename suffix.
No makefile: this is a documentation generator, so it is left out of the utils
DIRS list, same as hg/utils/otto.
diff --git src/hg/utils/cartTrackVarCatalog/harvestCartVars.py src/hg/utils/cartTrackVarCatalog/harvestCartVars.py
new file mode 100755
index 00000000000..d65f47695ca
--- /dev/null
+++ src/hg/utils/cartTrackVarCatalog/harvestCartVars.py
@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+"""harvestCartVars.py - find track-scoped cart variables in the kent tree.
+
+Refs #37838. This is the mechanical half of the cart variable inventory: it
+scans the source for the two ways a track-scoped cart name gets built and
+reports every suffix it finds, attributed to the function that reads or writes
+it. cartTrackVarCatalog.py (next to this file) is the curated half. Run this
+first when the tree has moved, then reconcile what falls out against the
+catalog.
+
+The two signals:
+
+ 1. cart*ClosestToHome(cart, tdb, parentLevel, "suffix")
+ The suffix argument is track-scoped by construction, so the 4th argument
+ of any such call is a cart variable name.
+
+ 2. safef(buf, sizeof buf, "%s.%s", track, SUFFIX)
+ and the "%s.%s.%s" and "%s.<literal>" variants. Note the suffix is the
+ SECOND vararg, not the first: the first one is the track name.
+
+Macro identifiers are resolved against every #define in the scanned trees plus
+inc/, lib/ and hg/inc/, chased up to five levels deep so that things like
+GRAY_LEVEL_SCORE_MIN -> SCORE_MIN -> "scoreMin" come out as strings.
+
+What it cannot resolve it reports rather than drops:
+
+ {ident} an identifier with no #define found, usually a local variable
+ holding a name computed at run time
+ EXPR:... a computed expression, e.g. a ternary
+
+Those are signal, not noise. They mark the places where the name is built at
+run time, which is exactly where the hierarchy nests one level deeper:
+filter.<field>, decorator.<name>.<var>, <track>.<species>.
+
+Output needs curation. The scan cannot tell a cart variable from a table
+name, a filename suffix or an SQL fragment, so expect to throw away things
+like .bai, _gold and .tbi by hand.
+
+Usage:
+ harvestCartVars.py --by-func # grouped by function, for reading
+ harvestCartVars.py --by-var # grouped by variable, with sites
+ harvestCartVars.py --json recs.json # raw records
+ harvestCartVars.py --dirs hg/hgc,hg/hgTables --by-func
+"""
+
+import argparse
+import collections
+import json
+import os
+import re
+import sys
+
+ROOT = os.path.expanduser("~/kent/src")
+
+# Everything that draws or configures a track. hgc and hgTables are in the
+# default list because they read and write per-track vars too, which is easy
+# to forget.
+DEFAULT_DIRS = "hg/lib,hg/hgTracks,hg/hgTrackUi,hg/cgilib,hg/hgc,hg/hgTables"
+
+# Extra trees mined for #define values only, not scanned for call sites.
+MACRO_DIRS = ["inc", "lib", "hg/inc"]
+
+
+# ---------------------------------------------------------------------------
+# macro table
+# ---------------------------------------------------------------------------
+
+def build_macros(dirs):
+ """Map every #define that resolves to a string literal, chasing aliases."""
+ macro = {}
+ chains = []
+ def_re = re.compile(
+ r'^\s*#\s*define\s+([A-Za-z_][A-Za-z0-9_]*)\s+'
+ r'("(?:[^"\\]|\\.)*")\s*(?:/[/*].*)?$')
+ chain_re = re.compile(
+ r'^\s*#\s*define\s+([A-Za-z_][A-Za-z0-9_]*)\s+'
+ r'([A-Za-z_][A-Za-z0-9_]*)\s*(?:/[/*].*)?$')
+ for d in list(dirs) + MACRO_DIRS:
+ p = os.path.join(ROOT, d)
+ if not os.path.isdir(p):
+ continue
+ for fn in os.listdir(p):
+ if not fn.endswith((".h", ".c")):
+ continue
+ for line in open(os.path.join(p, fn), errors="replace"):
+ m = def_re.match(line)
+ if m:
+ macro.setdefault(m.group(1), m.group(2)[1:-1])
+ continue
+ m = chain_re.match(line)
+ if m:
+ chains.append((m.group(1), m.group(2)))
+ for _ in range(5):
+ for a, b in chains:
+ if a not in macro and b in macro:
+ macro[a] = macro[b]
+ return macro
+
+
+# ---------------------------------------------------------------------------
+# C parsing, such as it is
+# ---------------------------------------------------------------------------
+
+CONST_RE = re.compile(
+ r'(?:static\s+)?(?:const\s+)?char\s*\*\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*'
+ r'"((?:[^"\\]|\\.)*)"')
+
+# Kent style puts function bodies at column 0, so a bare "safef(...)" at
+# column 0 looks like a function definition. Requiring a return type plus
+# whitespace (or a star) before the name is what keeps that from matching.
+FUNCDEF_RE = re.compile(
+ r'^(?:static\s+)?(?:INLINE\s+)?(?:const\s+)?'
+ r'(?:struct\s+[A-Za-z_]\w*|unsigned\s+\w+|[A-Za-z_]\w*)'
+ r'(?:\s+\**\s*|\s*\*+\s*)'
+ r'([A-Za-z_]\w*)\s*\([^;]*$')
+
+CTH_RE = re.compile(r'\bcart\w*ClosestToHome\s*\(')
+FMT_RE = re.compile(
+ r'\b(?:safef|dyStringPrintf|sqlDyStringPrintf|printf|jsInlineF)\s*\(')
+
+
+def split_args(s):
+ """Split a C argument list at top-level commas, respecting strings."""
+ out, depth, cur, i, instr = [], 0, "", 0, False
+ while i < len(s):
+ c = s[i]
+ if instr:
+ cur += c
+ if c == "\\":
+ cur += s[i+1:i+2]
+ i += 2
+ continue
+ if c == '"':
+ instr = False
+ i += 1
+ continue
+ if c == '"':
+ instr = True
+ cur += c
+ i += 1
+ continue
+ if c in "([{":
+ depth += 1
+ elif c in ")]}":
+ depth -= 1
+ if c == "," and depth == 0:
+ out.append(cur.strip())
+ cur = ""
+ i += 1
+ continue
+ cur += c
+ i += 1
+ if cur.strip():
+ out.append(cur.strip())
+ return out
+
+
+def resolve(arg, localconst, macro):
+ """Turn an argument into the string it evaluates to, if we can."""
+ arg = arg.strip()
+ if re.fullmatch(r'"((?:[^"\\]|\\.)*)"', arg):
+ return arg[1:-1]
+ toks = re.findall(r'"(?:[^"\\]|\\.)*"|[A-Za-z_][A-Za-z0-9_]*', arg)
+ plain = re.sub(r'"(?:[^"\\]|\\.)*"|[A-Za-z_][A-Za-z0-9_]*|\s+', '', arg)
+ if plain == "" and toks:
+ # nothing but literals and identifiers, i.e. C string concatenation
+ vals = []
+ for t in toks:
+ if t.startswith('"'):
+ vals.append(t[1:-1])
+ elif t in localconst:
+ vals.append(localconst[t])
+ elif t in macro:
+ vals.append(macro[t])
+ else:
+ vals.append("{" + t + "}")
+ return "".join(vals)
+ return "EXPR:" + re.sub(r'\s+', ' ', arg)[:60]
+
+
+def match_close(txt, i):
+ """Index of the paren that closes the one at i."""
+ depth, j, instr = 0, i, False
+ while j < len(txt):
+ c = txt[j]
+ if instr:
+ if c == "\\":
+ j += 2
+ continue
+ if c == '"':
+ instr = False
+ elif c == '"':
+ instr = True
+ elif c == "(":
+ depth += 1
+ elif c == ")":
+ depth -= 1
+ if depth == 0:
+ return j
+ j += 1
+ return len(txt) - 1
+
+
+def enclosing_functions(lines):
+ """Map 1-based line number to the name of the function containing it."""
+ encl = [None] * (len(lines) + 2)
+ cur = None
+ for idx, line in enumerate(lines):
+ if (line and not line[0].isspace()
+ and not line.startswith(("#", "/", "*", "}", "{"))):
+ m = FUNCDEF_RE.match(line)
+ if m:
+ cur = m.group(1)
+ encl[idx + 1] = cur
+ return encl
+
+
+def scan_file(fp, rel, macro):
+ """Return one record per track-scoped cart name found in one file."""
+ txt = open(fp, errors="replace").read()
+ localconst = {m.group(1): m.group(2) for m in CONST_RE.finditer(txt)}
+ encl = enclosing_functions(txt.split("\n"))
+
+ starts = [0]
+ for i, ch in enumerate(txt):
+ if ch == "\n":
+ starts.append(i + 1)
+
+ def lineno(pos):
+ lo, hi = 0, len(starts) - 1
+ while lo < hi:
+ mid = (lo + hi + 1) // 2
+ if starts[mid] <= pos:
+ lo = mid
+ else:
+ hi = mid - 1
+ return lo + 1
+
+ recs = []
+
+ for m in CTH_RE.finditer(txt):
+ i = m.end() - 1
+ args = split_args(txt[i+1:match_close(txt, i)])
+ if len(args) >= 4:
+ ln = lineno(m.start())
+ recs.append(dict(var=resolve(args[3], localconst, macro),
+ file=rel, line=ln, func=encl[ln], how="cth"))
+
+ for m in FMT_RE.finditer(txt):
+ i = m.end() - 1
+ args = split_args(txt[i+1:match_close(txt, i)])
+ ln = lineno(m.start())
+ fi = None
+ for k, a in enumerate(args):
+ if a.strip().startswith('"'):
+ fi = k
+ break
+ if fi is None:
+ continue
+ fmt = resolve(args[fi], localconst, macro)
+ rest = args[fi+1:]
+ mm = re.match(r'^%s([._])(.*)$', fmt or "")
+ if not mm:
+ continue
+ sep, tail = mm.group(1), mm.group(2)
+ if "%" not in tail and tail:
+ recs.append(dict(var=sep+tail, file=rel, line=ln,
+ func=encl[ln], how="fmtlit"))
+ elif tail == "%s" and len(rest) >= 2:
+ # "%s.%s", track, SUFFIX -> the suffix is the SECOND vararg
+ v = resolve(rest[1], localconst, macro)
+ if v and not v.startswith("EXPR:"):
+ recs.append(dict(var=sep+v, file=rel, line=ln,
+ func=encl[ln], how="fmt"))
+ elif tail == "%s.%s" and len(rest) >= 3:
+ v1 = resolve(rest[1], localconst, macro)
+ v2 = resolve(rest[2], localconst, macro)
+ if not v1.startswith("EXPR:") and not v2.startswith("EXPR:"):
+ recs.append(dict(var=sep+v1+"."+v2, file=rel, line=ln,
+ func=encl[ln], how="fmt3"))
+ return recs
+
+
+# ---------------------------------------------------------------------------
+# main
+# ---------------------------------------------------------------------------
+
+def main():
+ ap = argparse.ArgumentParser(
+ description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--dirs", default=DEFAULT_DIRS,
+ help="comma-separated dirs under the kent src root to "
+ "scan (default: %s)" % DEFAULT_DIRS)
+ ap.add_argument("--json", metavar="FILE", help="write raw records as JSON")
+ ap.add_argument("--by-func", action="store_true",
+ help="group by file and function")
+ ap.add_argument("--by-var", action="store_true",
+ help="group by variable, listing where each is used")
+ ap.add_argument("--keep-unresolved", action="store_true",
+ help="include {ident} and EXPR: entries in the groupings")
+ args = ap.parse_args()
+
+ dirs = [d.strip() for d in args.dirs.split(",") if d.strip()]
+ macro = build_macros(dirs)
+
+ files = []
+ for d in dirs:
+ p = os.path.join(ROOT, d)
+ if not os.path.isdir(p):
+ sys.exit("no such directory: %s" % p)
+ for fn in sorted(os.listdir(p)):
+ if fn.endswith(".c"):
+ files.append(os.path.join(p, fn))
+
+ records = []
+ for fp in files:
+ records.extend(scan_file(fp, os.path.relpath(fp, ROOT), macro))
+
+ print("scanned %d files in %d dirs, %d macros, %d records"
+ % (len(files), len(dirs), len(macro), len(records)), file=sys.stderr)
+
+ def wanted(var):
+ if args.keep_unresolved:
+ return True
+ return not var.startswith("EXPR:") and "{" not in var
+
+ if args.json:
+ with open(args.json, "w") as f:
+ json.dump(records, f, indent=1)
+ print("wrote %s" % args.json, file=sys.stderr)
+
+ if args.by_func:
+ byfunc = collections.defaultdict(set)
+ for r in records:
+ if wanted(r["var"]):
+ byfunc[(r["file"], r["func"])].add(r["var"])
+ for k in sorted(byfunc):
+ print("%s %s()" % (k[0], k[1]))
+ print(" " + ", ".join(sorted(byfunc[k])))
+
+ if args.by_var:
+ byvar = collections.defaultdict(set)
+ for r in records:
+ if wanted(r["var"]):
+ byvar[r["var"]].add("%s:%d" % (r["file"], r["line"]))
+ for k in sorted(byvar, key=str.lower):
+ print("%-34s %d %s"
+ % (k, len(byvar[k]), " ".join(sorted(byvar[k])[:4])))
+
+ if not (args.json or args.by_func or args.by_var):
+ print("nothing to do; pass --by-func, --by-var or --json",
+ file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())