fd380f3ae9c678f8e92b4728d5f614a71764fecd braney Tue Sep 1 12:56:18 2026 -0700 trackDbConditions: four fixes found by walking the results against the code, refs #37908 Walking the worklist row by row is what found these; none of them was visible from the summary counts. A read wrapped in another call was invisible to the value map, because it stopped at the first call name. cloneString(trackDbSetting(...)) and atoi(cartOrTdbString(...)) are both common. This is why hideEmptySubtracksSourcesUrl came back as merely composite-only when it in fact also needs hideEmptySubtracks and hideEmptySubtracksMultiBedUrl. A macro in a condition was read as an unknown word, so cartVarExistsAnyLevel(cart, tdb, FALSE, MAF_CHAIN_VAR) did not resolve. That hid the fact that irows is consulted only when mafChain is absent from the cart. Same indirection trap as the chained defines, in a different place. The variable-to-setting map kept only the last assignment. wigFetchMinMaxYWithCart assigns defaultViewLimits from defaultViewLimits and then, if that came back NULL, from viewLimits, so the test in between looked like viewLimits testing itself. It is now resolved at the position of the test, which turns an artifact into the real finding: viewLimits is read only when defaultViewLimits is absent. Two classes of noise removed. "Read the trackDb value when the cart has none" is how every setting with a default resolves, and it was a third of the worklist. A guard on the trackDb type line having words is a sanity check that is true of every track, and it was the whole of what the scan had to say about chainNormScoreAvailable, lollyMaxSize and lollyNoStems. diff --git src/hg/utils/trackDbConditions/harvestConditions.py src/hg/utils/trackDbConditions/harvestConditions.py index 02312851ca0..b13b79e89ae 100755 --- src/hg/utils/trackDbConditions/harvestConditions.py +++ src/hg/utils/trackDbConditions/harvestConditions.py @@ -1,778 +1,816 @@ #!/usr/bin/env python3 """harvestConditions.py - find where the browser reads a trackDb setting, and what has to be true first. Refs #37908. A setting's row in trackDbLibrary.shtml says which track types it applies to. It is a flat list, so it cannot say "only in coverage mode", "only when the track is in pack", "only when another setting is on". Those conditions are real and there are a lot of them. This finds them by reading the C. Two questions, kept apart, because they are different questions: render what has to be true for the setting to change the picture. Scanned over hg/hgTracks. This is the one the documentation is about. config what has to be true for the control to appear on the track's configuration page. Scanned over hui.c and the *Ui.c files. How a condition is found ------------------------ Two passes. The first records, for every read of a setting, the if/else/while /for/switch conditions whose block encloses it. Ranges are used rather than brace depth, so `if (x) return;` is handled as exactly as `if (x) { ... }`, and an early return contributes its negation to the rest of the function, which is where a lot of kent's real guards live. That alone finds little, because the read itself is usually plain and the condition sits at the caller. hicUiGetArcLimit() just reads the setting; the test for arc mode is up in hicTrack.c. So the second pass builds a call graph and, for each function, works out the conditions that hold on EVERY path into it. Only conditions true on every path are reported. That keeps the claim sound: a reported condition is necessary, never sufficient. Two things make a function give up and report nothing rather than something false: - its address is taken somewhere, so it can be a track method pointer and be reached from outside anything this scanner can see. - it is recursive, or nothing in scope calls it. What it cannot see ------------------ A name built with safef rather than written at the call site. A condition carried in a variable rather than tested in place: bamColorTag is read plainly and the test for bamColorMode==tag is at the point the value is used, several lines later, which is a def-use question this does not ask. Conditions in JavaScript. So the output is a floor, not a census. """ import collections import glob import json import os import re import sys # The argument that holds the setting name, per reader. A reader not listed # here is not a setting read. # # TDB_READERS take a trackDb, so what they read is a track setting and can carry # a trackDb default. The rest read a plain cart variable belonging to the page: # chromInfoPage, the track search box, which configuration tab is open. Those # are real cart variables and #37923 catalogs them, but they are not trackDb # settings and mixing them in buries the signal. TDB_READERS = { "cartOrTdbBoolean", "cartOrTdbString", "cartOrTdbInt", "cartOrTdbDouble", "trackDbSetting", "trackDbSettingOrDefault", "trackDbSettingClosestToHome", "trackDbSettingClosestToHomeOrDefault", "trackDbFloatSettingOrDefault", "trackDbSettingOn", "trackDbSettingOnOrDefault", "cartUsualStringClosestToHome", "cartUsualIntClosestToHome", "cartUsualBooleanClosestToHome", "cartUsualDoubleClosestToHome", "cartStringClosestToHome", "cartBooleanClosestToHome", } READERS = { "cartOrTdbBoolean": 2, "cartOrTdbString": 2, "cartOrTdbInt": 2, "cartOrTdbDouble": 2, "trackDbSetting": 1, "trackDbSettingOrDefault": 1, "trackDbSettingClosestToHome": 1, "trackDbSettingClosestToHomeOrDefault": 1, "trackDbFloatSettingOrDefault": 1, "trackDbSettingOn": 1, "trackDbSettingOnOrDefault": 1, "cartUsualStringClosestToHome": 3, "cartUsualIntClosestToHome": 3, "cartUsualBooleanClosestToHome": 3, "cartUsualDoubleClosestToHome": 3, "cartStringClosestToHome": 2, "cartBooleanClosestToHome": 2, "cartUsualString": 1, "cartUsualInt": 1, "cartUsualBoolean": 1, "cartUsualDouble": 1, "cartOptionalString": 1, "cartString": 1, "cartInt": 1, "cartBoolean": 1, "cartVarExists": 1, } KEYWORDS = ("if", "while", "for", "switch") NOT_A_CALL = set(KEYWORDS) | {"else", "return", "sizeof", "case", "defined", "do"} # The two scopes. A file can be in both: wiggleCart.c is read from the drawing # code and from the config page, and the answer differs. SCOPES = { "render": ["hg/hgTracks/*.c"], "config": ["hg/hgTrackUi/*.c", "hg/hgTracks/config.c", "hg/hgTracks/searchTracks.c"], } # In hgTracks but not drawing: the configuration page and the track search page # both read track settings, and neither is the picture. NOT_RENDER = ("hg/hgTracks/config.c", "hg/hgTracks/searchTracks.c") # Scanned for both scopes. Naming the libraries one at a time was wrong twice # over: hui.c and the *Ui.c files are not only the config page, they hold the # accessors the drawing code calls, and a hand-kept list silently missed # netCart.c, chainCart.c, pgSnp.c and a dozen more that read track settings on # the drawing path. A read site that is not scanned is worse than one that is # unclassified: the every-path analysis would claim a condition holds at every # read while never having seen one of them. So scan the libraries whole and let # reachability decide which side of the browser each read belongs to. SHARED = ["hg/lib/*.c", "hg/cgilib/*.c"] # A read reached with none of these is unguarded. Anything matching is noise # rather than a condition on the setting: it says the code got far enough to # run, not that the setting only applies sometimes. NOISE = re.compile( r"^NOT \((?:[\w>.\-]+(?:\s*->\s*\w+)* == NULL)" r"(?:\s*\|\|\s*[\w>.\-]+(?:\s*->\s*\w+)* == NULL)*\)$" r"|^errCatchStart\b" r"|^\w+ != NULL$" r"|^NOT \(isEmpty\(" r"|^NOT \(!\w+\)$" r"|^\w+ = \w+; \w+ != NULL" # for (x = y; x != NULL; ...) + # the words of the trackDb type line. A track always has a type, so a guard + # on the count being positive is a sanity check, not a condition: it was + # showing up as a "data" condition on chainNormScoreAvailable, lollyMaxSize + # and lollyNoStems, and it means nothing on any of them. + r"|^NOT \(word(Count|Ct) <= 0\)$|^word(Count|Ct) > 0$" ) def blankOut(src): """Comments and string bodies become spaces, so offsets and lines still line up.""" out = list(src) i, n = 0, len(src) while i < n: c = src[i] if c == "/" and i + 1 < n and src[i+1] == "/": while i < n and src[i] != "\n": out[i] = " " i += 1 elif c == "/" and i + 1 < n and src[i+1] == "*": out[i] = out[i+1] = " " i += 2 while i + 1 < n and not (src[i] == "*" and src[i+1] == "/"): if src[i] != "\n": out[i] = " " i += 1 if i + 1 < n: out[i] = out[i+1] = " " i += 2 elif c in "\"'": quote = c i += 1 while i < n and src[i] != quote: if src[i] == "\\": out[i] = " " i += 1 if i < n and src[i] != "\n": out[i] = " " i += 1 continue if src[i] != "\n": out[i] = " " i += 1 if i < n: i += 1 else: i += 1 return "".join(out) def matchParen(s, i): """i is at '('. Index just past the matching ')'.""" depth = 0 while i < len(s): if s[i] == "(": depth += 1 elif s[i] == ")": depth -= 1 if depth == 0: return i + 1 i += 1 return len(s) def skipSpace(s, i): while i < len(s) and s[i] in " \t\n\r": i += 1 return i def stmtEnd(s, i): """End of the one statement, or block, that starts at i.""" i = skipSpace(s, i) depth = 0 while i < len(s): c = s[i] if c == "{": depth += 1 elif c == "}": depth -= 1 if depth <= 0: return i + 1 elif c == ";" and depth == 0: return i + 1 i += 1 return len(s) def lineIndex(src): idx, line = [0] * (len(src) + 1), 1 for i, c in enumerate(src): idx[i] = line if c == "\n": line += 1 idx[len(src)] = line return idx LEAVES = re.compile(r"^\s*\{?\s*(return\b|errAbort\s*\(|continue\b|break\b)") # How far into a function an early return still reads as a precondition. PRECONDITION_LINES = 25 def loadDefines(paths): """#define NAME "literal", so a macro-spelled setting is not missed. Also follows a macro defined as another macro. GRAY_LEVEL_SCORE_MIN is SCORE_MIN is "scoreMin", and stopping at the first hop loses the read. """ out, alias = {}, {} for path in paths: try: with open(path, errors="replace") as f: for line in f: m = re.match(r'\s*#define\s+([A-Z][A-Z0-9_]*)\s+"([^"]*)"', line) if m: out.setdefault(m.group(1), m.group(2)) continue m = re.match(r"\s*#define\s+([A-Z][A-Z0-9_]*)\s+([A-Z][A-Z0-9_]*)\s*$", line) if m: alias.setdefault(m.group(1), m.group(2)) except OSError: pass for _ in range(8): # a chain longer than this is a loop grew = False for name, target in alias.items(): if name not in out and target in out: out[name] = out[target] grew = True if not grew: break return out def settingNameOf(arg, defines): """The setting an argument names: a literal, or a macro that expands to one.""" arg = arg.strip() lit = re.match(r'^"((?:[^"\\]|\\.)*)"$', arg) if lit: return lit.group(1) if arg in defines: return defines[arg] if re.fullmatch(r"[A-Z][A-Z0-9_]*", arg): return "{%s}" % arg return None def splitArgs(text): out, depth, cur = [], 0, "" for ch in text: if ch in "([": depth += 1 elif ch in ")]": depth -= 1 if ch == "," and depth == 0: out.append(cur) cur = "" else: cur += ch out.append(cur) return out def scanFile(path, kentSrc, defines): """Every function, guard, setting read and call in one file.""" with open(path, errors="replace") as f: src = f.read() s = blankOut(src) line = lineIndex(src) rel = os.path.relpath(path, kentSrc) n = len(s) # functions: name ( ... ) { with the paren at brace depth 0 funcs = [] depth, i = 0, 0 while i < n: c = s[i] if c == "{": depth += 1 i += 1 continue if c == "}": depth -= 1 i += 1 continue m = re.match(r"[A-Za-z_][A-Za-z0-9_]*", s[i:]) if not m: i += 1 continue word = m.group(0) j = skipSpace(s, i + len(word)) if depth == 0 and j < n and s[j] == "(" and word not in NOT_A_CALL: end = matchParen(s, j) p = skipSpace(s, end) if p < n and s[p] == "{": funcs.append((p, stmtEnd(s, p), word)) i += len(word) def enclosing(off): return next((f[2] for f in funcs if f[0] <= off < f[1]), None) # char *scoreMinStr = trackDbSettingClosestToHome(tdb, GRAY_LEVEL_SCORE_MIN); # A later test of scoreMinStr is a test of the setting, not housekeeping, so # remember which local holds which setting. fromSetting = {} fileValue = collections.defaultdict(set) # name -> settings it ever holds assignAt = collections.defaultdict(list) # name -> offsets of its assignments - for m in re.finditer(r"([A-Za-z_][A-Za-z0-9_]*)\s*=\s*([A-Za-z_][A-Za-z0-9_]*)\s*\(", s): - reader = m.group(2) - if reader not in READERS: + for m in re.finditer(r"([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(?=[A-Za-z_])", s): + # The read is often wrapped: cloneString(trackDbSetting(...)) and + # atoi(cartOrTdbString(...)) are both common, and stopping at the first + # call name loses them. hideEmptySubtracksSourcesUrl needing + # hideEmptySubtracksMultiBedUrl was missed for exactly this reason. + pos, reader, open_ = m.end(), None, None + for _ in range(4): + call = re.match(r"([A-Za-z_][A-Za-z0-9_]*)\s*\(", s[pos:]) + if not call: + break + here = pos + call.end() - 1 + if call.group(1) in READERS: + reader, open_ = call.group(1), here + break + pos = here + 1 + if reader is None: continue - open_ = s.index("(", m.end(2)) close = matchParen(s, open_) args = splitArgs(src[open_ + 1:close - 1]) idx = READERS[reader] if idx >= len(args): continue name = settingNameOf(args[idx], defines) if name and not name.startswith("{"): - fromSetting[(enclosing(m.start(1)), m.group(1))] = name + # Keep every assignment with its offset. wigFetchMinMaxYWithCart + # assigns defaultViewLimits from defaultViewLimits and then, if that + # came back NULL, from viewLimits. Remembering only the last one + # made the test in between look like viewLimits testing itself, and + # hid a real relationship between two settings. + fromSetting.setdefault((enclosing(m.start(1)), m.group(1)), []).append( + (m.start(1), name)) fileValue[m.group(1)].add(name) assignAt[m.group(1)].append(m.start(1)) # A name is only taken to carry a setting when it carries exactly one, and # is long enough not to be a word like type, name or vis that means # something else three functions away. carries = {var: sorted(names)[0] for var, names in fileValue.items() if len(names) == 1 and len(var) >= 5} # guards guards, condSpans = [], [] i = 0 while i < n: m = re.match(r"\b(if|while|for|switch)\b", s[i:]) if m and (i == 0 or not (s[i-1].isalnum() or s[i-1] == "_")): word = m.group(1) j = skipSpace(s, i + len(word)) if j < n and s[j] == "(": close = matchParen(s, j) cond = re.sub(r"\s+", " ", src[j+1:close-1]).strip() end = stmtEnd(s, close) guards.append((close, end, word, cond, line[i])) condSpans.append((j, close)) p = skipSpace(s, end) if s[p:p+4] == "else" and not (p+4 < n and (s[p+4].isalnum() or s[p+4] == "_")): q = skipSpace(s, p + 4) guards.append((q, stmtEnd(s, q), "else", "NOT (%s)" % cond, line[p])) elif word == "if" and LEAVES.match(src[close:end]): # The guarded statement leaves, so the rest of the function # is only reached when the condition is false. Only near the # top of the function, though: that is a precondition, and # people read it as one. The same shape four hundred lines # down is just sequencing, and attributing it to every later # read is sound but says nothing about the setting. It was # how the jsonp output check in doTrackForm came to look like # a condition on filterBy. for fs, fe, _ in funcs: if fs <= i < fe: if line[i] - line[fs] <= PRECONDITION_LINES: guards.append((end, fe, "guard", "NOT (%s)" % cond, line[i])) break i = close continue i += 1 def condsAt(off): out = [] for g in guards: if g[0] <= off < g[1]: func = enclosing(g[0]) for part in splitConjuncts(g[3]): idents = re.findall(r"[A-Za-z_][A-Za-z0-9_]*", part) - derived = sorted({fromSetting[(func, ident)] for ident in idents - if (func, ident) in fromSetting} - | {carries[ident] for ident in idents if ident in carries}) + derived = set() + for ident in idents: + # A macro in the test is the setting it expands to. + # cartVarExistsAnyLevel(cart, tdb, FALSE, MAF_CHAIN_VAR) + # is a test of mafChain, and reading it as an unknown + # word hid that irows is only consulted when mafChain is + # absent from the cart. + if re.fullmatch(r"[A-Z][A-Z0-9_]*", ident) and ident in defines: + derived.add(defines[ident]) + continue + writes = fromSetting.get((func, ident)) + if writes: + # the assignment in force where the test is written + before = [w for w in writes if w[0] < g[0]] + derived.add((before[-1] if before else writes[0])[1]) + elif ident in carries: + derived.add(carries[ident]) + derived = sorted(derived) out.append({"kind": g[2], "text": part, "line": g[4], "file": rel, "derivedFrom": derived}) return out # reads and calls, both only inside a function body reads, calls, addrTaken = [], [], set() depth, i = 0, 0 while i < n: c = s[i] if c == "{": depth += 1 i += 1 continue if c == "}": depth -= 1 i += 1 continue m = re.match(r"[A-Za-z_][A-Za-z0-9_]*", s[i:]) if not m: i += 1 continue word = m.group(0) j = skipSpace(s, i + len(word)) isCall = j < n and s[j] == "(" if isCall and word in READERS and depth > 0: close = matchParen(s, j) args = splitArgs(src[j+1:close-1]) idx = READERS[word] name = settingNameOf(args[idx], defines) if idx < len(args) else None if name: reads.append({"file": rel, "line": line[i], "reader": word, "name": name, "tdb": word in TDB_READERS, "func": enclosing(i), "conds": condsAt(i)}) i = close continue if isCall and word not in NOT_A_CALL and depth > 0: # depth > 0 keeps a definition or a prototype from counting as a call calls.append({"callee": word, "caller": enclosing(i), "file": rel, "line": line[i], "conds": condsAt(i)}) i += len(word) continue if not isCall and depth > 0 and re.fullmatch(r"[a-z][A-Za-z0-9_]*", word): # only a real use as a value, which is how a track method is # installed: tg->drawItems = bedDrawItems; or &someFunc before = src[max(0, i-2):i].strip() after = s[skipSpace(s, i + len(word)):][:1] if (before.endswith("=") or before.endswith("&") or before.endswith(",")) \ and after in (";", ",", ")"): addrTaken.add(word) i += len(word) # Where the value goes. A setting is often read plainly and only used # under a test, which is the bamColorTag shape: the value is read at the top # of the loader and used far below, behind a test of bamColorMode. So for # each name that carries a setting, find where the value is used and what # guards those uses. inCond = lambda off: any(a <= off < b for a, b in condSpans) def isTransfer(off): """The value being handed on, not consumed. The test is how deep in parentheses the mention sits, counted from the start of its statement. Inside a call it is an argument, so something is doing work with it: bamGetTagString(bam, btd->userTag, ...) is a use. At depth zero it is one side of an assignment or an element of an initializer list, which only moves the value somewhere else: struct bamTrackData btd = {tg, pairHash, colorMode, userTag, ...} carries the setting into a field without using it. Counting that as a use puts an unguarded site in the set and empties every intersection. """ start = max(s.rfind(";", 0, off), s.rfind("{", 0, off), s.rfind("}", 0, off)) depth = 0 for ch in s[start + 1:off]: if ch in "([": depth += 1 elif ch in ")]": depth -= 1 return depth <= 0 useConds = {} for var, setting in carries.items(): sites = [] for m in re.finditer(r"\b%s\b" % re.escape(var), s): off = m.start() if off in assignAt[var] or inCond(off) or enclosing(off) is None: continue if isTransfer(off): continue sites.append(condsAt(off)) if not sites: continue common = {useKey(c): c for c in sites[0]} for one in sites[1:]: keys = {useKey(c) for c in one} common = {k: v for k, v in common.items() if k in keys} if not common: break # a test of the setting's own value says only that it is set useConds[setting] = [c for c in common.values() if c.get("derivedFrom") != [setting]] for read in reads: read["useConds"] = useConds.get(read["name"], []) if read["tdb"] else [] return {"file": rel, "funcs": [f[2] for f in funcs], "reads": reads, "calls": calls, "addrTaken": sorted(addrTaken)} def splitOn(text, op): """Split on a top-level operator, ignoring anything inside brackets.""" parts, depth, cur, i = [], 0, "", 0 while i < len(text): c = text[i] if c in "([": depth += 1 elif c in ")]": depth -= 1 if depth == 0 and text[i:i+2] == op: parts.append(cur) cur = "" i += 2 continue cur += c i += 1 parts.append(cur) return [p.strip() for p in parts if p.strip()] def unwrap(text): """Drop one layer of parentheses when they wrap the whole expression.""" text = text.strip() while text.startswith("(") and text.endswith(")"): depth = 0 for i, ch in enumerate(text): if ch == "(": depth += 1 elif ch == ")": depth -= 1 if depth == 0 and i != len(text) - 1: return text # the parens are not a wrapper text = text[1:-1].strip() return text def splitConjuncts(text): """Break a condition into the separate things it requires. A && B is two conditions, and splitting it is sound. Splitting A || B is not. But a negated disjunction is a conjunction, so NOT (A || B) splits into NOT A and NOT B, which is where the useful ones hide: the squishyPack guard reads NOT (visibility != tvPack || checkIfWiggling(...)), and as one string it is neither a visibility condition nor a coverage one. """ inner = re.match(r"^NOT\s*\((.*)\)$", text.strip(), re.S) if inner: disjuncts = splitOn(inner.group(1), "||") if len(disjuncts) > 1: return ["NOT (%s)" % unwrap(part) for part in disjuncts] parts = splitOn(text, "&&") return parts if len(parts) > 1 else [text] def condKey(cond): """Two conditions are the same when they say the same thing, spacing aside.""" return (re.sub(r"\s+", "", cond["text"]), cond["kind"]) def useKey(cond): """Looser key, for matching a test of a value against a test of the field holding it. The same test is written sameString(colorMode, ...) in the loader and sameString(btd->colorMode, ...) in the drawer, because the value was carried into a struct on the way. Dropping the field prefix makes those one condition. Only the use-site pass uses this; the every-path analysis keeps the strict key, since there a wrong match would turn a guess into a claim. """ text = re.sub(r"\b\w+\s*->\s*", "", cond["text"]) return (re.sub(r"\s+", "", text), cond["kind"]) UI_NAME = re.compile(r"(CfgUi|Ui|UiSection|Option|Options|Menu|Dropdown|Section|Cfg)$") class Graph: """The call graph of one scope, and the conditions it can prove.""" def __init__(self, units, scopeFiles, dropUiCallers=False): self.defined = set() self.addrTaken = set() for unit in units: self.defined.update(unit["funcs"]) self.addrTaken.update(unit["addrTaken"]) self.external = self.addrTaken & self.defined self.calls = [] for unit in units: if unit["file"] not in scopeFiles: continue for call in unit["calls"]: if dropUiCallers and call["caller"] and UI_NAME.search(call["caller"]): continue # drawing a control is not drawing the track self.calls.append(call) self.callsFrom = collections.defaultdict(set) for call in self.calls: if call["caller"]: self.callsFrom[call["caller"]].add(call["callee"]) self.callsTo = collections.defaultdict(list) self.cache = {} self.reachable = set() def restrictToReachable(self): """Keep only call sites this side of the browser can actually execute. Done after reachFrom, because a call sitting in a shared file is only a real call site for this scope when something in the scope reaches it. """ self.callsTo = collections.defaultdict(list) for call in self.calls: if call["caller"] is None or call["caller"] in self.reachable: self.callsTo[call["callee"]].append(call) self.cache = {} def reachFrom(self, entries): """Functions reachable from a scope's own entry points. A read in a shared file belongs to a scope only when the drawing side, or the config side, can actually get to it. hicUiGetArcLimit lives in hicUi.c and is reached from both; the cfgUi functions beside it are reached only from the config page. """ seen, todo = set(), list(entries) while todo: func = todo.pop() if func in seen: continue seen.add(func) todo.extend(self.callsFrom.get(func, ())) self.reachable = seen return seen def necessary(self, func, stack=None): """Conditions true on every path into func. None when that is unknowable.""" if func in self.cache: return self.cache[func] stack = stack or set() if func in stack or func in self.external or func not in self.defined: return None sites = self.callsTo.get(func) if not sites: self.cache[func] = None return None self.cache[func] = None # break cycles while we recurse stack.add(func) common = None for site in sites: here = {condKey(c): c for c in site["conds"]} up = self.necessary(site["caller"], stack) if up: here.update({condKey(c): c for c in up}) if common is None: common = dict(here) else: common = {k: v for k, v in common.items() if k in here} if not common: break stack.discard(func) self.cache[func] = list(common.values()) if common else [] return self.cache[func] def harvest(kentSrc): """Scan the tree and return one record per setting read, per scope.""" patterns = sorted({p for pats in SCOPES.values() for p in pats} | set(SHARED)) files = [] for pattern in patterns: files += sorted(glob.glob(os.path.join(kentSrc, pattern))) files = sorted(set(files)) if len(files) < 50: raise SystemExit("only %d source files found under %s; is KENT_SRC right?" % (len(files), kentSrc)) defines = loadDefines(glob.glob(os.path.join(kentSrc, "hg/inc/*.h")) + glob.glob(os.path.join(kentSrc, "inc/*.h")) + glob.glob(os.path.join(kentSrc, "hg/hgTracks/*.h"))) units = [scanFile(f, kentSrc, defines) for f in files] byFile = {u["file"]: u for u in units} out = [] for scope, pats in SCOPES.items(): ownFiles = set() for pattern in pats: for f in glob.glob(os.path.join(kentSrc, pattern)): ownFiles.add(os.path.relpath(f, kentSrc)) if scope == "render": ownFiles -= set(NOT_RENDER) scopeFiles = set(ownFiles) for pattern in SHARED: for f in glob.glob(os.path.join(kentSrc, pattern)): scopeFiles.add(os.path.relpath(f, kentSrc)) graph = Graph(units, scopeFiles, dropUiCallers=(scope == "render")) entries = set() for rel in ownFiles: if scope == "render" and rel in NOT_RENDER: continue for func in byFile.get(rel, {}).get("funcs", []): if scope == "render" and UI_NAME.search(func): continue # the config page, reached from hgTracks entries.add(func) if scope == "config": for unit in units: for func in unit["funcs"]: if UI_NAME.search(func): entries.add(func) reachable = graph.reachFrom(entries) graph.restrictToReachable() for rel in sorted(scopeFiles): unit = byFile.get(rel) if not unit: continue for read in unit["reads"]: if scope == "render" and rel in NOT_RENDER: continue if rel not in ownFiles and read["func"] not in reachable: continue # this side of the browser never gets here if scope == "render" and read["func"] and UI_NAME.search(read["func"]): continue up = graph.necessary(read["func"]) record = dict(read) record["scope"] = scope record["callerConds"] = up or [] record["callerUnknown"] = up is None out.append(record) return out def isNoise(cond): """Housekeeping, not a condition on the setting. A loop header says the code is walking a list, not that the setting only applies sometimes. A test of a variable that holds another setting is never noise, however much it looks like a null check: scoreMinStr != NULL is the scoreMin gate. """ if cond["kind"] in ("for", "while"): return True if cond.get("derivedFrom"): return False return bool(NOISE.match(cond["text"])) def realConds(read): """The read's conditions with the housekeeping tests dropped.""" seen, out = set(), [] for cond in read["conds"] + read["callerConds"]: if isNoise(cond): continue key = condKey(cond) if key in seen: continue seen.add(key) out.append(cond) return out def main(): kentSrc = os.environ.get("KENT_SRC") or os.path.abspath( os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..")) reads = harvest(kentSrc) if len(sys.argv) > 1 and sys.argv[1] not in ("-", "--stdout"): with open(sys.argv[1], "w") as f: json.dump(reads, f, indent=1) print("wrote %s" % sys.argv[1]) byScope = collections.Counter(r["scope"] for r in reads) tdbReads = [r for r in reads if r["tdb"]] print("read sites %d (%s)" % (len(reads), dict(byScope))) print(" track settings %d, distinct %d" % (len(tdbReads), len({r["name"] for r in tdbReads}))) print(" page cart vars %d, distinct %d" % (len(reads) - len(tdbReads), len({r["name"] for r in reads if not r["tdb"]}))) for scope in SCOPES: rs = [r for r in reads if r["scope"] == scope and r["tdb"]] print("%-7s guarded at the read %4d guarded at a caller %4d any %4d of %d" % (scope, sum(1 for r in rs if [c for c in r["conds"] if not isNoise(c)]), sum(1 for r in rs if [c for c in r["callerConds"] if not isNoise(c)]), sum(1 for r in rs if realConds(r)), len(rs))) if __name__ == "__main__": main()