6c9c79a4bb5af45d213d8ba5fc373ed0a6181dca braney Sat Aug 1 12:05:30 2026 -0700 cartTrackVarCatalog: add a --reconcile that reads the tree refs #37838 --check only read the catalog itself, verifying that every family a type names exists, so a track-scoped cart variable added tomorrow was invisible to it. It never called the harvester at all. Adds --reconcile, which does: it compares what harvestCartVars finds against the catalog and reports a name in neither the catalog nor cartVarsNotCataloged.txt, silent and exit 0 otherwise, so it can run nightly. --update-baseline accepts new names as a reviewable diff. Matching is on the name with its leading separator stripped, because the harvester cannot always tell which separator a name is used with: the fourth argument of cart*ClosestToHome is a bare suffix while a safef("%s.%s") site carries the dot. A wildcard entry also registers its trailing component, since the harvester sees decorator.<name>.blockMode only as blockMode. Without that nine already-cataloged names read as new. Sixty-seven names are left in the baseline, and about fifteen of them look like real cart variables that were never cataloged, among them FilterLabel, FilterValuesDefault, HighlightType, minAc, fileSortOrder and the tablesTables paging vars. Cataloging those needs a read of the UI code and is not done here. harvestCartVars grows a harvest() entry point so the scan has one definition rather than one in main and a second written out by hand; --by-var output is unchanged. diff --git src/hg/utils/cartTrackVarCatalog/harvestCartVars.py src/hg/utils/cartTrackVarCatalog/harvestCartVars.py index d65f47695ca..fc0b674e40b 100755 --- src/hg/utils/cartTrackVarCatalog/harvestCartVars.py +++ src/hg/utils/cartTrackVarCatalog/harvestCartVars.py @@ -38,31 +38,34 @@ Usage: harvestCartVars.py --by-func # grouped by function, for reading harvestCartVars.py --by-var # grouped by variable, with sites harvestCartVars.py --json recs.json # raw records harvestCartVars.py --dirs hg/hgc,hg/hgTables --by-func """ import argparse import collections import json import os import re import sys -ROOT = os.path.expanduser("~/kent/src") +# The tree to scan. KENT_SRC lets a nightly run point at a pristine +# checkout instead of somebody's working tree, where a stray .c file or a +# half-finished edit would show up as a finding. +ROOT = os.environ.get("KENT_SRC") or os.path.expanduser("~/kent/src") # Everything that draws or configures a track. hgc and hgTables are in the # default list because they read and write per-track vars too, which is easy # to forget. DEFAULT_DIRS = "hg/lib,hg/hgTracks,hg/hgTrackUi,hg/cgilib,hg/hgc,hg/hgTables" # Extra trees mined for #define values only, not scanned for call sites. MACRO_DIRS = ["inc", "lib", "hg/inc"] # --------------------------------------------------------------------------- # macro table # --------------------------------------------------------------------------- def build_macros(dirs): @@ -269,68 +272,101 @@ elif tail == "%s" and len(rest) >= 2: # "%s.%s", track, SUFFIX -> the suffix is the SECOND vararg v = resolve(rest[1], localconst, macro) if v and not v.startswith("EXPR:"): recs.append(dict(var=sep+v, file=rel, line=ln, func=encl[ln], how="fmt")) elif tail == "%s.%s" and len(rest) >= 3: v1 = resolve(rest[1], localconst, macro) v2 = resolve(rest[2], localconst, macro) if not v1.startswith("EXPR:") and not v2.startswith("EXPR:"): recs.append(dict(var=sep+v1+"."+v2, file=rel, line=ln, func=encl[ln], how="fmt3")) return recs +# --------------------------------------------------------------------------- +# entry point for the catalog next door +# --------------------------------------------------------------------------- + +def harvest(dirs=None, quiet=False): + """Scan the tree and return the raw records. + + cartTrackVarCatalog.py --reconcile imports this, so the scan has one + definition rather than one here and a second one written out by hand. + """ + dirs = dirs or [d.strip() for d in DEFAULT_DIRS.split(",") if d.strip()] + macro = build_macros(dirs) + + files = [] + for d in dirs: + p = os.path.join(ROOT, d) + if not os.path.isdir(p): + sys.exit("no such directory: %s" % p) + for fn in sorted(os.listdir(p)): + if fn.endswith(".c"): + files.append(os.path.join(p, fn)) + + records = [] + for fp in files: + records.extend(scan_file(fp, os.path.relpath(fp, ROOT), macro)) + + if not quiet: + print("scanned %d files in %d dirs, %d macros, %d records" + % (len(files), len(dirs), len(macro), len(records)), + file=sys.stderr) + return records + + +def resolved(records): + """name -> first file:line, for the names the scan resolved to a literal. + + An EXPR: or {ident} record marks a name built at run time, which is signal + for a person reading the harvester output but cannot be compared against a + catalog of literal names, so it is dropped here. + """ + out = {} + for r in sorted(records, key=lambda r: (r["file"], r["line"])): + var = r["var"] + if var.startswith("EXPR:") or "{" in var: + continue + out.setdefault(var, "%s:%d" % (r["file"], r["line"])) + return out + + # --------------------------------------------------------------------------- # main # --------------------------------------------------------------------------- def main(): ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--dirs", default=DEFAULT_DIRS, help="comma-separated dirs under the kent src root to " "scan (default: %s)" % DEFAULT_DIRS) ap.add_argument("--json", metavar="FILE", help="write raw records as JSON") ap.add_argument("--by-func", action="store_true", help="group by file and function") ap.add_argument("--by-var", action="store_true", help="group by variable, listing where each is used") ap.add_argument("--keep-unresolved", action="store_true", help="include {ident} and EXPR: entries in the groupings") args = ap.parse_args() dirs = [d.strip() for d in args.dirs.split(",") if d.strip()] - macro = build_macros(dirs) - - files = [] - for d in dirs: - p = os.path.join(ROOT, d) - if not os.path.isdir(p): - sys.exit("no such directory: %s" % p) - for fn in sorted(os.listdir(p)): - if fn.endswith(".c"): - files.append(os.path.join(p, fn)) - - records = [] - for fp in files: - records.extend(scan_file(fp, os.path.relpath(fp, ROOT), macro)) - - print("scanned %d files in %d dirs, %d macros, %d records" - % (len(files), len(dirs), len(macro), len(records)), file=sys.stderr) + records = harvest(dirs) def wanted(var): if args.keep_unresolved: return True return not var.startswith("EXPR:") and "{" not in var if args.json: with open(args.json, "w") as f: json.dump(records, f, indent=1) print("wrote %s" % args.json, file=sys.stderr) if args.by_func: byfunc = collections.defaultdict(set) for r in records: if wanted(r["var"]):