2f60825bdf72a8d0c99b81552266c9537e9ee2dc braney Sat Sep 5 08:03:52 2026 -0700 cartTrackVarCatalog: tell an hg.conf name from a cart name. An hg.conf setting and a track-scoped cart variable are built the same way. jksql.c:1325 does safef(cfgName, sizeof cfgName, "%s.excludeDbs", failoverProf->name) and reads the result with cfgOption on the next line, which the scan cannot distinguish from safef(buf, size, "%s.heightPer", track). So excludeDbs sat in cartVarsNotCataloged.txt: correctly, in that it is not a cart variable, but that left the setting described in no registry at all and suppressed in the wrong one. The accessor that reads the buffer back is what tells them apart, so hgConfRead() looks for a cfg* call taking the same identifier within six lines of the safef. Six because the read is normally the next line and a buffer reused later in the function for something else must not excuse an unrelated name; the destination has to be a plain identifier, or we do not know what was filled in. It claims exactly one name in the tree today, and harvestCartVars.py --hgconf prints it with its call site. --reconcile no longer asks about such a name and --update-baseline no longer writes it back, so the baseline is 55 names to 54. The reverse case is now an error rather than a silence: if this catalog ever describes a name the tree reads with a cfg* accessor, --reconcile says so and exits 1, because that is one of the two registries being wrong about what the name is rather than a matter of taste. Verified by planting excludeDbs in the catalog and confirming the report. hgConfCatalog gained the row in the companion commit. refs #37838 #37925 diff --git src/hg/utils/cartTrackVarCatalog/cartTrackVarCatalog.py src/hg/utils/cartTrackVarCatalog/cartTrackVarCatalog.py index 0e8c0230dc8..2f21bb6652a 100755 --- src/hg/utils/cartTrackVarCatalog/cartTrackVarCatalog.py +++ src/hg/utils/cartTrackVarCatalog/cartTrackVarCatalog.py @@ -1725,40 +1725,48 @@ """The harvester next door, or None if it cannot be imported. Imported by name inside a function rather than at module scope on purpose. registryPages loads this file by path, and in that process the sibling directory is not on sys.path, so a top-level import would break a consumer that only wants the catalog and never asks for a harvest. """ try: import harvestCartVars as h except ImportError: print("harvestCartVars.py not importable from here", file=sys.stderr) return None return h +# The records behind the last harvested() call. Kept so --reconcile can ask +# the harvester about a record's classification without scanning the tree a +# second time, which costs about ten seconds. +_RECORDS = None + + def harvested(): """(name -> file:line) for every literal name the tree yields, or None. Keyed the same way the catalog is, so the two are directly comparable. """ + global _RECORDS h = harvestModule() if h is None: return None + _RECORDS = h.harvest(quiet=True) out = {} - for name, src in h.resolved(h.harvest(quiet=True)).items(): + for name, src in h.resolved(_RECORDS).items(): k = key(name) if k: out.setdefault(k, src) return out def reconcile(cat, out=sys.stdout, verbose=False): """Diff the catalog against the names the tree actually builds. The one thing here that needs a person is a name the tree builds that is in neither the catalog nor the baseline, because that is a track-scoped variable somebody added without saying what it is. A catalog entry with no call site found is not: several are read through a helper or spelled with a macro the scan cannot follow, and printing them on every run is what would make a nightly unreadable. So that half goes out only under --verbose. @@ -1778,67 +1786,96 @@ return 1 cataloged, literals, patterns = cataloged_test() baseline = read_baseline() h = harvestModule() # harvested() above already proved it imports # A filename is not a cart variable, and the scan cannot tell the two # apart: "%s.tmp" built from a filename has the shape of "%s.heightPer" # built from a track name. h.fileNameLike() answers that from the # trailing extension, so those names no longer need a baseline line each. # The test comes after cataloged(), so a name the catalog describes is # never hidden by it, and h.fileNameLike is checked against the catalog's # own literals by --check. files = sorted(n for n in tree if not cataloged(n) and h.fileNameLike(n)) + + # Same story one registry over: jksql.c builds "<profile>.excludeDbs" with + # safef and reads it with cfgOption, so it is an hg.conf setting and lives + # in hgConfCatalog's profile-suffix family. harvestCartVars.py --hgconf + # lists these. + conf = h.hgConfNames(_RECORDS or []) + confSeen = sorted(n for n in tree if n in conf) + new = sorted(n for n in tree if not cataloged(n) and n not in baseline - and not h.fileNameLike(n)) + and not h.fileNameLike(n) and n not in conf) only_cat = sorted(n for n in literals if n not in tree) + # A name this catalog describes that the tree reads with a cfg* accessor + # is not a difference of opinion, it is one of the two registries being + # wrong about what the name is. Rare enough to print rather than count. + claimed = sorted(n for n in tree if n in conf and cataloged(n)) + if verbose: print("catalog literal names %d" % len(literals), file=out) print("catalog patterns %d" % len(patterns), file=out) print("harvested names %d" % len(tree), file=out) print("baseline names %d" % len(baseline), file=out) print("\nin the catalog, no call site found (%d)" % len(only_cat), file=out) print(" (expected for a name read through a helper or built from a " "macro the\n scan cannot follow; anything else is an entry " "whose read has gone away)", file=out) for n in only_cat: print(" %s" % n, file=out) known = sorted(n for n in tree if n in baseline) print("\nharvested, in the baseline rather than the catalog (%d)" % len(known), file=out) for n in known: print(" %-30s %s" % (n, tree[n]), file=out) print("\nharvested, read as a filename rather than a cart variable " "(%d)" % len(files), file=out) print(" (harvestCartVars.py --filenames explains the rule)", file=out) for n in files: print(" %-30s %s" % (n, tree[n]), file=out) + print("\nharvested, an hg.conf setting rather than a cart variable " + "(%d)" % len(confSeen), file=out) + print(" (harvestCartVars.py --hgconf explains the rule; these " + "belong to\n hgConfCatalog, which cannot see them either)", + file=out) + for n in confSeen: + print(" %-30s %s" % (n, tree[n]), file=out) + + if claimed: + print("\nin this catalog, but the tree reads it with a cfg* accessor " + "(%d):" % len(claimed), file=out) + print(" (an hg.conf setting cannot also be a track-scoped cart " + "variable; one of\n the two registries has it wrong)", + file=out) + for n in claimed: + print(" %-30s %s" % (n, tree[n]), file=out) if new: print("\nbuilt by the tree, in neither the catalog nor the baseline " "(%d):" % len(new), file=out) print(" (add an entry to cartTrackVarCatalog.py if it is a " "track-scoped cart\n variable, otherwise accept it with " "--update-baseline)", file=out) for n in new: print(" %-30s %s" % (n, tree[n]), file=out) - return len(new) + return len(new) + len(claimed) # --------------------------------------------------------------------------- # HTML rendering # --------------------------------------------------------------------------- CSS = """ body { font: 14px/1.5 -apple-system, "Segoe UI", Helvetica, Arial, sans-serif; margin: 0; color: #12191f; background: #fff; } header { background: #14385c; color: #fff; padding: 18px 28px; } header h1 { margin: 0 0 4px; font-size: 20px; font-weight: 600; } header p { margin: 0; font-size: 13px; color: #c5d6e8; } main { max-width: 1080px; margin: 0 auto; padding: 22px 28px 60px; } h2 { font-size: 17px; margin: 30px 0 6px; padding-bottom: 5px; border-bottom: 2px solid #14385c; } @@ -2130,31 +2167,32 @@ help="rewrite %s from the current tree; read the diff " "before committing it" % os.path.basename(BASELINE_FILE)) args = ap.parse_args() cat = build() if args.updateBaseline: tree = harvested() if tree is None: return 1 cataloged, _, _ = cataloged_test() h = harvestModule() # harvested() above already proved it imports was = read_baseline() # The same two exclusions --reconcile makes, or accepting the backlog # would write back every filename the harvester reads as a name. now = set(n for n in tree - if not cataloged(n) and not h.fileNameLike(n)) + if not cataloged(n) and not h.fileNameLike(n) + and n not in h.hgConfNames(_RECORDS or [])) write_baseline(now, tree) print("wrote %s: %d names, %d added, %d dropped" % (BASELINE_FILE, len(now), len(now - was), len(was - now))) return 0 if args.reconcile: return 1 if reconcile(cat, verbose=args.verbose) else 0 if args.json: with open(args.json, "w") as f: json.dump(cat, f, indent=1) print("wrote %s" % args.json) if args.html: with open(args.html, "w") as f: f.write(render_html(cat)) print("wrote %s" % args.html) if args.check or not (args.json or args.html):