a3df9f62a5995302b5a07ce5e3dd0eadda3a7768 braney Sat Sep 12 09:59:01 2026 -0700 cartTrackVarCatalog: describe hgTables' own cart variables, refs #37979 With peel() fixed, 2,331 hgta_ names in the saved sessions were left honestly uncatalogued rather than absorbed by a catch-all. They fall into four groups, each read at its call site before a row was written for it. The two linked-table checkboxes, hgta_fs.linked.. and hgta_fil.linked..
, which offer a joinable table's fields on the Select Fields and filter pages. extraTableList finds the checked tables by scanning the cart for the prefix, so the set is whatever the cart holds. The filter ops. The catalog covered .pat alone; hgTables.h defines six, and all six are in live sessions. pat, dd and cmp belong to one field, while rawLogic, rawQuery and maxOutput apply to the whole table and still carry a field slot in the name, filled with an empty string or a bare _. Fifty-one session-scoped variables: intersection, correlation, subtrack merge, identifiers, user regions, output naming, MAF output, and which table the Select Fields, filter and histogram pages are about. The header itself documents the convention that shapes half of them - the pages with a Cancel button hold their state twice, hgta_ in force and hgta_next proposed, copied one way on open and the other on Submit - so that is in the group's description rather than in every note. Renaming the .pat row to . also removed an accidental cover: a row registers its trailing component, and that "pat" had been standing in for gvfTrack.c's %s_pat, which is an item label rather than a cart variable. Its seven siblings were already in the baseline, so pat joins them there. hgta_ names matched only by a catch-all go from 4,299 to 3. The three left are hgta_identifierFile and hgta_userRegionsFile, described in the #37623 file-variable registry that this audit does not read, and hgta_userRegionsTable, which nothing in the tree reads at all. diff --git src/hg/utils/cartTrackVarCatalog/cartTrackVarCatalog.py src/hg/utils/cartTrackVarCatalog/cartTrackVarCatalog.py index b4f3cb4763d..4071d09d491 100755 --- src/hg/utils/cartTrackVarCatalog/cartTrackVarCatalog.py +++ src/hg/utils/cartTrackVarCatalog/cartTrackVarCatalog.py @@ -1,2374 +1,2482 @@ #!/usr/bin/env python3 """cartTrackVarCatalog.py - the registry of track-scoped cart variables. Refs #37838. This is the hand-curated catalog that backs two things: 1. The JSON cart schema (#37838) - the hierarchy below is the shape the stored cart wants to grow into: track name at the top, then the vars every track has, then a per-type section, then nested leaf groups. 2. The planned cart accessor layer - each entry names one variable, its value type, its separator, and where the tree reads it, so accessors can be generated or at least checked against reality. Harvested mechanically from cart*ClosestToHome() and safef("%s.%s") call sites in hg/lib, hg/hgTracks, hg/hgTrackUi and hg/cgilib, then curated by hand: names that turned out to be table names, file suffixes or non-cart strings were dropped, macro identifiers were resolved to their values, and the type/enum/default columns were read out of the UI code. Usage: cartTrackVarCatalog.py --json out.json cartTrackVarCatalog.py --html out.html cartTrackVarCatalog.py --check # sanity checks, prints counts cartTrackVarCatalog.py --reconcile # diff the catalog against the tree cartTrackVarCatalog.py --reconcile --verbose # ... with the full diff cartTrackVarCatalog.py --update-baseline # accept new tree names --reconcile is the mode meant for a nightly cron: it prints nothing and exits 0 when nothing has changed, and exits 1 with a list when the tree has grown a track-scoped name that is in neither the catalog nor the baseline. --check is a different thing, and not a substitute: it only reads this file, so it cannot see the tree move at all. Matching is on the variable name with its leading separator stripped, because the harvester cannot always tell which separator a name is used with: the fourth argument of cart*ClosestToHome() is a bare suffix, while a safef("%s.%s") site carries the dot. So .foo and _foo reconcile as one name here, even though the catalog records the two spellings separately and the difference between them is a live bug in at least one place (see _pairEndsByName). The scan cannot tell a cart variable from a table name, a filename suffix or an SQL fragment, so a good part of what it finds is not a cart variable at all (.bai, _gold, .tbi). Those live in BASELINE_FILE next to this script rather than being argued with one at a time: reconcile complains only about a name in neither the catalog nor the baseline. Accept new ones with --update-baseline and commit the file, which puts the decision in the git log. """ import argparse import html import json import os import re import sys # Harvested names that are not track-scoped cart variables. See the docstring. BASELINE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cartVarsNotCataloged.txt") # Floor on how many names a working scan finds; well under the real count. See # the check in reconcile(). MIN_TREE_NAMES = 150 # --------------------------------------------------------------------------- # how a track-scoped name is built # --------------------------------------------------------------------------- NAMING = { "canonical": ".", "legacy": "_", "legacyNote": "The underscore form predates the dot form. cartRemoveAllForTdb() " "(hg/lib/cart.c:3306) removes both prefixes plus the bare track name, " "and carries the comment 'All should be {track}.{varName}'. Any " "rewrite should keep reading both.", "lookupOrder": [ ".", "..", ".", ], "lookupOrderSrc": "hg/lib/cart.c:cartLookUpVariableClosestToHome", "lookupOrderNote": "parentLevel=TRUE starts the search at the parent, so a subtrack " "setting always wins over its view, which wins over its composite. " "The JSON form needs to preserve that three-level fallback, not " "flatten it.", "trackNamePrefixes": [ {"prefix": "hub__", "what": "track hub track", "src": "hg/lib/trackHub.c"}, {"prefix": "ct_", "what": "custom track", "src": "hg/lib/customTrack.c"}, {"prefix": "dup__", "what": "duplicated track", "src": "hg/inc/dupTrack.h:DUP_TRACK_PREFIX"}, ], "valueEncoding": "Everything is a string in the cart hash. The 'type' column below is " "how the reader interprets it, and is what the JSON form could encode " "natively. 'list' vars are multi-valued: the same name appears more " "than once in the var=val encoding and must become a JSON array.", } # --------------------------------------------------------------------------- # helpers # --------------------------------------------------------------------------- def v(name, type_, src, sep=".", values=None, default=None, note=None, tdb=None, multi=False, aliases=None, valuesSrc=None): """One catalog entry. name variable name after the prefix and separator type_ bool | int | float | string | enum | list | color | hidden src file:line where the tree reads or writes it sep '.' (canonical) or '_' (legacy) values the values the CART may hold, exactly as the reader compares them aliases other spellings a writer may use -> the canonical cart value. Usually the trackDb vocabulary, which is often not the cart vocabulary; see the autoScale entry for the case that proved it. valuesSrc the C array the values were checked against. Name it for any enum whose value list is not obvious from src, so the next reader can re-verify instead of trusting this file. tdb trackDb setting that supplies the default, if differently named """ d = {"name": name, "type": type_, "sep": sep, "src": src} if values: d["values"] = values if aliases: d["aliases"] = aliases if valuesSrc: d["valuesSrc"] = valuesSrc if default is not None: d["default"] = default if tdb: d["tdbDefault"] = tdb if multi: d["multi"] = True if note: d["note"] = note return d # --------------------------------------------------------------------------- # LEVEL 2: variables every track can have, whatever its type # --------------------------------------------------------------------------- COMMON = { "visibility": { "what": "What the track is set to. The only var that is the bare " "track name with no suffix at all.", "vars": [ v("", "enum", "hg/lib/hui.c:9954", sep="", values=["hide", "dense", "squish", "pack", "full"], note="Bare . For a superTrack the values are " "'show'/'hide' instead (hg/lib/hui.c:9887). This is the " "var #37838 discussed moving under a 'vis.' prefix."), v("_sel", "bool", "hg/lib/hui.c:5434", sep="_", note="Subtrack/composite-member checkbox. Four-state in the " "UI (checked, unchecked, checked-disabled) but stored 0/1; " "see fourState* in hg/inc/hui.h."), v("_hideKids", "bool", "hg/hgTracks/hgTracks.c:7652", sep="_", note="Container collapsed in hgTracks, children not drawn."), v("_faux", "hidden", "hg/lib/hui.c:5524", sep="_", note="UI-only element id, not persisted state."), v("_toggle", "hidden", "hg/lib/hui.c:5529", sep="_", note="UI-only element id, not persisted state."), ], }, "layout": { "what": "Where the track sits in the image and how tall it is.", "vars": [ v("priority", "float", "hg/lib/hui.c:tdbAddPrioritiesFromCart", tdb="priority", note="User drag-reorder override of the trackDb priority."), v("group", "string", "hg/hgTracks/hgTracks.c:groupTracks", note="User moved the track into a different track group."), v("_imgOrd", "int", "hg/hgTracks/imageV2.c:flatTracksSort", sep="_", note="Row order in the image, set by drag-reorder in " "hgTracks.js. Distinct from priority."), v("heightPer", "int", "hg/inc/wiggle.h:HEIGHTPER", default="128", tdb="maxHeightPixels", note="Track height in pixels. Shared by wig, lolly, interact, " "long, sample and vcf haplotype displays."), ], }, "color": { "what": "Per-track color override, offered for any track whose type " "supports it (tdbSupportsColorOverride).", "vars": [ v("colorOverride", "color", "hg/lib/hui.c:colorTrackOption", note="RGB as '#rrggbb'."), v("colorOverrideOn", "bool", "hg/lib/hui.c:colorTrackOption"), ], }, "ui": { "what": "State of the hgTrackUi page itself, not of the drawing.", "vars": [ v("section_
_close", "bool", "hg/lib/jsHelper.c:411", note="One per collapsible section on the track's config page " "(jsBeginCollapsibleSection). Known sections include " "colorByAttribute, superDescription, superMembers."), v("_button", "hidden", "hg/hgTracks/config.c:321", sep="_"), v("_defaultBut", "hidden", "hg/hgTracks/config.c:348", sep="_"), v("_hideAllBut", "hidden", "hg/hgTracks/config.c:331", sep="_"), v("_showAllBut", "hidden", "hg/hgTracks/config.c:342", sep="_"), v("_edit", "hidden", "hg/hgTracks/hgTracks.c:10121", sep="_"), ], }, "dataFilters": { "what": "Filters offered for many types, not tied to one of them.", "vars": [ v("nameFilter", "string", "hg/lib/hui.c:filterNameOption", note="Wildcard match on item name."), v("doMergeItems", "bool", "hg/inc/hui.h:MERGESPAN_CART_SETTING", tdb="mergeSpannedItems", note="Collapse items that span the whole window into one."), v("doWiggle", "bool", "hg/lib/hui.c:wigOption", note="Draw a bed/genePred/psl/bam type as a coverage wiggle. " "When on, the whole wig type group below applies too."), v("squishyPackPoint", "float", "hg/lib/hui.c:squishyPackOption", note="Row count past which pack degrades to squish."), v("doSnake", "bool", "hg/lib/hui.c:snakeOption", note="Draw a chain/psl as a snake."), ], }, } # --------------------------------------------------------------------------- # LEVEL 2b: containers - composite, view, superTrack, multiWig, faceted # --------------------------------------------------------------------------- CONTAINER = { "composite": { "what": "Vars a composite parent owns. Subtrack selection lives on " "the child (_sel), not here.", "vars": [ v("displaySubtracks", "enum", "hg/lib/hui.c:compositeUiSubtracks", values=["all", "selected"]), v("hideEmptySubtracks", "bool", "hg/lib/hui.c:compositeHideEmptySubtracks", tdb="hideEmptySubtracks"), v("defaults", "int", "hg/hgTrackUi/hgTrackUi.c:3866", note="Set to 1 by the track UI's reset button. hgTrackUi " "reads it with cartUsualInt and then clears this " "container's cart variables and its children's, so it is " "a one-shot command that happens to travel as a cart " "variable. Read for a superTrack as well as a composite; " "the surrounding test is tdbIsContainer || " "tdbIsSuperTrack."), v("sortOrder", "string", "hg/lib/hui.c:sortOrderGet", note="Subtrack table sort, e.g. 'cellType=+ view=-'."), v("facetSortOrder", "string", "hg/hgTrackUi/hgTrackUi.c:3314", note="Same thing for a faceted composite's table, and the same " "'field=+ field2=-' syntax, which facetedComposite.js:921 " "says it copied from sortOrder above. Written only by " "JavaScript (facetedComposite.js:931), sent even when empty " "so the server clears a stale value, and read back in " "hgTrackUi to override trackDb's defaultSortField. The " "read treats it as untrusted, because the JSON it lands in " "goes inside a """) return "\n".join(p) def main(): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--json") ap.add_argument("--html") ap.add_argument("--check", action="store_true") ap.add_argument("--reconcile", action="store_true") ap.add_argument("--verbose", action="store_true", help="with --reconcile, also print the standing drift " "that needs no action") ap.add_argument("--update-baseline", dest="updateBaseline", action="store_true", help="rewrite %s from the current tree; read the diff " "before committing it" % os.path.basename(BASELINE_FILE)) args = ap.parse_args() cat = build() if args.updateBaseline: tree = harvested() if tree is None: return 1 cataloged, _, _ = cataloged_test() h = harvestModule() # harvested() above already proved it imports was = read_baseline() # The same two exclusions --reconcile makes, or accepting the backlog # would write back every filename the harvester reads as a name. now = set(n for n in tree if not cataloged(n) and not h.fileNameLike(n) and n not in h.hgConfNames(_RECORDS or [])) write_baseline(now, tree) print("wrote %s: %d names, %d added, %d dropped" % (BASELINE_FILE, len(now), len(now - was), len(was - now))) return 0 if args.reconcile: return 1 if reconcile(cat, verbose=args.verbose) else 0 if args.json: with open(args.json, "w") as f: json.dump(cat, f, indent=1) print("wrote %s" % args.json) if args.html: with open(args.html, "w") as f: f.write(render_html(cat)) print("wrote %s" % args.html) if args.check or not (args.json or args.html): c = counts(cat) for k in sorted(c): print("%-12s %s" % (k, c[k])) # every family referenced by a type must exist bad = 0 for name, t in TYPES.items(): for fam in t.get("families", []): key = fam.split(" ")[0] if key not in FAMILIES and key not in TYPES: print("unknown family %r referenced by type %r" % (fam, name), file=sys.stderr) bad += 1 if bad: return 1 print("families ok") bad += check_values() if bad: return 1 return 0 if __name__ == "__main__": sys.exit(main())