287583e8d72b4abc90ac33694147f604e1238e93 braney Sat Sep 5 08:17:39 2026 -0700 cartTrackVarCatalog: walk the 31 names left in the baseline, catalog 23. The baseline's own header said that a name in it was not evidence anybody had looked at it, because the first version was accepted wholesale. Every remaining name has now been read at its call site. Twenty-three turned out to be real cart variables and get rows; the other 31 are not cart variables at all, and the header now says which five things they are instead. The 23: My Variants edit form, 18 rows, a new group in OTHER_CGIS. The hgc details page prints a form named <track>_<field>, the form posts so the values land in the cart like any request variable, and hgTracks reads them out of the cart on the next render, writes the row to SQL and removes them. Cart variables by mechanism, one-shot commands by intent, which is why _id is the trigger and nothing has a default. <field>FilterLabel, <field>FilterValuesDefault, <field>HighlightType. Each of these settings is read through cartOrTdbString in three spellings: filterLabel.<field>, <field>.FilterLabel, <field>FilterLabel. The catalog had the first and, for four sibling settings, the third; for these three it had neither, which is the only reason they showed up as unknown while <field>FilterType did not. <container>.defaults, the track UI reset button, read with cartUsualInt for a composite or a superTrack and then used to clear that track's cart variables and its children's. <track>.minAc, the VCF minimum allele count, so a reader can hide singletons with 2 instead of a frequency cutoff. What the other 31 are: gvfItemName's item labels (_unk, _dnovo and six more appended to an item's name for display, which is not a variable at all), HTML element ids, table and db.table names, one submit button read with cgiOptionalString rather than from the cart, and one printf artifact. projectSelect is an id, not a name: the <select> that writes it carries no name attribute, so nothing is ever submitted. One cost, measured rather than guessed, and written into the new group's note. The My Variants suffixes are BED field names, so they collide with hgTables' per-field cart variables, whose last component is also a column name. In the 6,620 saved sessions that moves 24 names out of sessionCartAudit's "matched only by a catch-all" bucket and into the known one. The real cause is in that audit: peel() only tries suffixes beginning after a separator, so a left-anchored row such as hgta_fs.check.<db>.<table>.<field> can never match the name it describes, which is why 4,299 hgta_ names are in that bucket already. Fixing it would make the longer match win and take the 24 back. Not fixed here; it is the audit's own bug and its numbers are published. Catalog 341 rows to 364, baseline 54 names to 31. refs #37838 #37979 diff --git src/hg/utils/cartTrackVarCatalog/cartTrackVarCatalog.py src/hg/utils/cartTrackVarCatalog/cartTrackVarCatalog.py index 2f21bb6652a..cd293d5fe79 100755 --- src/hg/utils/cartTrackVarCatalog/cartTrackVarCatalog.py +++ src/hg/utils/cartTrackVarCatalog/cartTrackVarCatalog.py @@ -225,30 +225,38 @@ # --------------------------------------------------------------------------- # LEVEL 2b: containers - composite, view, superTrack, multiWig, faceted # --------------------------------------------------------------------------- CONTAINER = { "composite": { "what": "Vars a composite parent owns. Subtrack selection lives on " "the child (<subtrack>_sel), not here.", "vars": [ v("displaySubtracks", "enum", "hg/lib/hui.c:compositeUiSubtracks", values=["all", "selected"]), v("hideEmptySubtracks", "bool", "hg/lib/hui.c:compositeHideEmptySubtracks", tdb="hideEmptySubtracks"), + v("defaults", "int", "hg/hgTrackUi/hgTrackUi.c:3866", + note="Set to 1 by the track UI's reset button. hgTrackUi " + "reads it with cartUsualInt and then clears this " + "container's cart variables and its children's, so it is " + "a one-shot command that happens to travel as a cart " + "variable. Read for a superTrack as well as a composite; " + "the surrounding test is tdbIsContainer || " + "tdbIsSuperTrack."), v("sortOrder", "string", "hg/lib/hui.c:sortOrderGet", note="Subtrack table sort, e.g. 'cellType=+ view=-'."), v("facetSortOrder", "string", "hg/hgTrackUi/hgTrackUi.c:3314", note="Same thing for a faceted composite's table, and the same " "'field=+ field2=-' syntax, which facetedComposite.js:921 " "says it copied from sortOrder above. Written only by " "JavaScript (facetedComposite.js:931), sent even when empty " "so the server clears a stale value, and read back in " "hgTrackUi to override trackDb's defaultSortField. The " "read treats it as untrusted, because the JSON it lands in " "goes inside a <script> block: anything outside " "alphanumerics and _ . - + = space drops the whole value."), v("filterComp.<groupTag>", "list", "hg/lib/hui.c:3119", multi=True, note="One per ABC dimension of a filterComposite. 'All' means " "every option selected."), @@ -316,63 +324,79 @@ "authoritative list of these wildcard settings.", "pattern": ["filter.<field>", "<field>Filter"], "vars": [ v("filter.<field>", "float", "hg/inc/bigBedFilter.h:23", note="Value, or 'min:max' colon pair."), v("filter.<field>Min", "float", "hg/inc/hui.h:_MIN"), v("filter.<field>Max", "float", "hg/inc/hui.h:_MAX"), v("filterLimits.<field>", "string", "hg/lib/trackDb.c:282", note="trackDb-side bounds; cart values outside them are " "dropped (hg/hgTracks/bigBedTrack.c:88)."), v("filterByRange.<field>", "bool", "hg/lib/trackDb.c:286"), v("filterLabel.<field>", "string", "hg/inc/bigBedFilter.h:28"), v("filterPriority.<field>", "float", "hg/inc/bigBedFilter.h:29"), v("<field>FilterLimits", "string", "hg/lib/trackDb.c:298"), v("<field>FilterPriority", "float", "hg/lib/trackDb.c:302"), + v("<field>FilterLabel", "string", "hg/lib/hui.c:4027", + note="getLabelSetting tries three spellings through " + "cartOrTdbString, in order: filterLabel.<field> above, " + "then <field>.FilterLabel, then this one. All three are " + "legal cart names, not only trackDb settings. The " + "harvester reports the middle spelling, and because the " + "separator is stripped for comparison this row is what " + "accounts for it, the same way <field>FilterType does."), ], }, "textFilter": { "what": "Per-field text filter, wildcard or regex.", "vars": [ v("filterText.<field>", "string", "hg/inc/bigBedFilter.h:24"), v("filterType.<field>", "enum", "hg/inc/bigBedFilter.h:27", values=["wildcard", "regexp"]), v("<field>FilterText", "string", "hg/lib/trackDb.c:300"), v("<field>FilterType", "enum", "hg/lib/trackDb.c:296"), ], }, "filterBy": { "what": "Multi-select filter on a categorical field. Value 'All' " "means no filtering.", "vars": [ v("filterBy.<field>", "list", "hg/lib/hui.c:4058", multi=True), v("filterValues.<field>", "string", "hg/inc/bigBedFilter.h:25", note="trackDb-side value list."), v("filterValuesDefault.<field>", "string", "hg/inc/bigBedFilter.h:26"), + v("<field>FilterValuesDefault", "string", "hg/lib/hui.c:4009", + note="The CAP spelling of the row above. " + "getFilterValueDefaultsSetting reads all three forms " + "through cartOrTdbString; see <field>FilterLabel."), v("doAdvanced", "bool", "hg/lib/hui.c:filterBySetCfgUiGuts", note="Advanced filter box expanded."), ], }, "highlightBy": { "what": "Same shape as the filter family, but highlights instead of " "hiding. Every filter* name has a highlight* twin.", "vars": [ v("highlightBy.<field>", "list", "hg/lib/hui.c:4058", multi=True), v("highlight.<field>", "float", "hg/inc/bigBedFilter.h:64"), v("highlightText.<field>", "string", "hg/inc/bigBedFilter.h:65"), v("highlightType.<field>", "enum", "hg/inc/bigBedFilter.h:68"), + v("<field>HighlightType", "enum", "hg/hgTracks/bigBedTrack.c:162", + note="The CAP spelling of the row above. getHighlightType " + "reads all three forms through cartOrTdbString; see " + "<field>FilterLabel."), v("highlightLimits.<field>", "string", "hg/lib/trackDb.c:...", note="See isComplexSetting() for the full twin list."), ], }, "score": { "what": "The old score-based filters, offered by scoreCfgUi for bed " "and friends.", "vars": [ v("scoreFilter", "int", "hg/lib/hui.c:scoreCfgUi", tdb="scoreFilter"), v("scoreFilterMin", "int", "hg/lib/hui.c:7268"), v("scoreFilterMax", "int", "hg/lib/hui.c:7271"), v("filterTopScorersOn", "bool", "hg/lib/hui.c:7301"), v("filterTopScorersCt", "int", "hg/lib/hui.c:7307"), v("minGrayLevel", "int", "hg/inc/hui.h:MIN_GRAY_LEVEL", @@ -712,30 +736,36 @@ "cfgUi": "hg/lib/vcfUi.c:vcfCfgUi", "families": ["filterBy"], "vars": [ v("hapClusterEnabled", "bool", "hg/inc/vcfUi.h:16"), v("hapClusterHeight", "int", "hg/inc/vcfUi.h:13"), v("hapClusterMethod", "enum", "hg/inc/vcfUi.h:17", values=["centerWeighted", "fileOrder", "treeFile"]), v("hapClusterColorBy", "enum", "hg/inc/vcfUi.h:23", values=["altOnly", "function", "refAlt", "base"]), v("hapClusterTreeAngle", "enum", "hg/inc/vcfUi.h:30", values=["triangle", "rectangle"]), v("sampleColorFile", "string", "hg/inc/vcfUi.h:35"), v("applyMinQual", "bool", "hg/inc/vcfUi.h:40"), v("minQual", "float", "hg/inc/vcfUi.h:44"), v("minFreq", "float", "hg/inc/vcfUi.h:52"), + v("minAc", "int", "hg/inc/vcfUi.h:56", default=0, + note="Minimum allele count, read off the INFO column's AC, so " + "a reader can hide singletons with 2 instead of working " + "out a frequency cutoff. vcfCfgMinAc writes the form " + "field as <track>.minAc and reads the value back with " + "cartOrTdbInt on the bare name (vcfUi.c:452)."), v("excludeFilterValues", "list", "hg/inc/vcfUi.h:49", multi=True), v("showHardyWeinberg", "bool", "hg/inc/vcfUi.h:38"), v("vcfSampleOrder", "string", "hg/inc/vcfUi.h:57", note="Drag-and-drop sample order, comma separated."), v("doDefaultLabel", "bool", "hg/inc/vcfUi.h:59"), v("doAliasLabel", "bool", "hg/inc/vcfUi.h:60"), v("hideParents", "bool", "hg/inc/vcfUi.h:61"), v("sortChildBelow", "bool", "hg/inc/vcfUi.h:63"), v("vcfPhasedColorBy", "enum", "hg/inc/vcfUi.h:65", values=["mendelDiff", "deNovo", "function", "noColor"]), v("centerVariantChrom", "hidden", "hg/lib/vcfUi.c:33", note="Haplotype-sorting anchor. Position state stored on the " "track, which is unusual and worth flagging in a " "rewrite."), v("centerVariantPos", "hidden", "hg/lib/vcfUi.c:35"), @@ -1287,30 +1317,128 @@ note="INCONSISTENCY: hui.c writes this as " "<track>.pairEndsByName (dot), hgc reads it with an " "underscore, so the details page does not follow the UI " "setting. Worth reconciling before either form is baked " "into a schema."), v("_geneTrack", "list", "hg/hgc/hgc.c:19635", sep="_", multi=True, note="snp125-era gene tracks for functional annotation, " "written by hgTrackUi.c:191. Not to be confused with the " "bigDbSnp <track>.geneTrack: separate features, each " "internally consistent."), v("interProXref", "bool", "hg/hgc/lowelab.c:753", note="Lowe lab tracks."), ], }, + "hgc My Variants edit": { + "what": "One edit to one My Variants annotation, in flight. The " + "details page (hg/hgc/myVariantsClick.c:doMyVariantsDetails) " + "prints a form whose fields are named <track>_<field>; the " + "form posts, so the values land in the cart the way any " + "request variable does; and on the next render hgTracks " + "(hg/hgTracks/myVariantsTrack.c:myVariantsEditOrDelete) reads " + "them out of the cart, writes the row to SQL and removes " + "them. So these are cart variables by mechanism and one-shot " + "commands by intent, which is why nothing here has a default " + "and why _id is the trigger: its presence is what says an " + "edit was submitted.\n" + "The track name is always myVariants_<encoded>, so these " + "suffixes are only ever seen under that prefix, but the " + "harvester reports a suffix without its prefix and the " + "catalog compares them that way, so ordinary words like id " + "and name are registered here bare. The whole feature is " + "behind the doMyVariants hg.conf gate, which defaults FALSE, " + "and none of these names is in any saved session.\n" + "Registering them costs something, and it was measured rather " + "than guessed. These are BED field names, so they collide " + "with hgTables' per-field variables, whose last component is " + "also a column name: hgta_fs.check.<db>.<table>.name now " + "matches the _name row here. In the 6,620 saved sessions " + "that moves 24 names out of sessionCartAudit's honest " + "\"matched only by a catch-all\" bucket. The underlying " + "reason is a gap in that audit rather than in this file: " + "peel() only tries suffixes that start after a separator, so " + "a left-anchored row like hgta_fs.check.<db>.<table>.<field> " + "can never match the name it describes, which is also why " + "4,299 hgta_ names sit in that bucket already. Fix that and " + "the longer, correct match wins and these 24 go back.", + "vars": [ + v("_id", "hidden", "hg/hgTracks/myVariantsTrack.c:736", sep="_", + note="Row id of the annotation being edited, written as a " + "hidden field at myVariantsClick.c:238. hgTracks removes " + "it first so an edit is applied once, and every other " + "variable here is ignored without it."), + v("_name", "string", "hg/hgc/myVariantsClick.c:243", sep="_", + note="The annotation's label, as drawn in the browser."), + v("_description", "string", "hg/hgc/myVariantsClick.c:250", + sep="_", note="Longer notes, shown on the details page."), + v("_chromStart", "int", "hg/hgc/myVariantsClick.c:265", sep="_", + note="Range-checked against the chromosome size in the form " + "and validated again server-side."), + v("_chromEnd", "int", "hg/hgc/myVariantsClick.c:270", sep="_"), + v("_thickStart", "int", "hg/hgc/myVariantsClick.c:279", sep="_", + note="CDS start. Offered only for a transcript annotation."), + v("_thickEnd", "int", "hg/hgc/myVariantsClick.c:284", sep="_", + note="CDS end. Transcript only."), + v("_blockCount", "int", "hg/hgTracks/myVariantsTrack.c:647", + sep="_", + note="Exon count. This and the two below are hidden fields " + "kept in sync by the block widget, and updateBlocksFields " + "validates the three together with loadAndValidateBed " + "before touching SQL: a missing one is a no-op, and a " + "failed validation removes all three so a bad edit cannot " + "corrupt the next one."), + v("_blockSizes", "string", "hg/hgTracks/myVariantsTrack.c:648", + sep="_", note="Comma-separated exon sizes."), + v("_chromStarts", "string", "hg/hgTracks/myVariantsTrack.c:649", + sep="_", note="Comma-separated exon starts, BED12 relative."), + v("_itemRgb", "color", "hg/hgc/myVariantsClick.c:343", sep="_", + note="#RRGGBB from a text input. updateTextField special-cases " + "any name ending in itemRgb and stores the integer " + "htmlColorForCode returns, so an unparseable colour is " + "dropped rather than written."), + v("_mouseover", "string", "hg/hgc/myVariantsClick.c:441", sep="_", + note="Hover text for the item."), + v("_cnvType", "enum", "hg/hgc/myVariantsClick.c:364", sep="_", + values=["deletion", "duplication", "insertion", "inversion", + "translocation", "complex", "breakend"], + valuesSrc="hg/lib/myVariants.c:18 myVariantsCnvTypes", + note="CNV annotations only. The vocabulary follows gnomAD, " + "says the form's own help icon."), + v("_ref", "string", "hg/hgc/myVariantsClick.c:386", sep="_", + note="Reference allele. Offered for a plain variant, not for " + "a transcript or a CNV."), + v("_alt", "string", "hg/hgc/myVariantsClick.c:391", sep="_", + note="Alternate allele for a variant. A CNV reuses the same " + "field for the inserted or duplicated sequence " + "(myVariantsClick.c:378), which is worth knowing before " + "either meaning is written into a schema."), + v("_project", "string", "hg/hgc/myVariantsClick.c:406", sep="_", + note="Free-text project name, for grouping annotations. A " + "shared track shows the project and does not offer the " + "field, so this arrives only from the owner."), + v("_delete", "hidden", "hg/hgTracks/myVariantsTrack.c:753", + sep="_", + note="The Delete button's name, so only its presence matters; " + "read with cartVarExists and removed before the row is " + "deleted. Not offered on a shared track."), + v("_cancel", "hidden", "hg/hgTracks/myVariantsTrack.c:745", + sep="_", + note="The Cancel button. Read before delete and before any " + "field update, and returns without applying anything."), + ], + }, "hgc extended DNA": { "what": "The Extended DNA page (hg/hgc/hgc.c:doGetDnaExtended1) keeps " "one case/color set per track. A real per-track group that " "no cfgUi function knows about.", "vars": [ v("_case", "bool", "hg/hgc/hgc.c:5746", sep="_", note="Upper case this track's bases."), v("_u", "bool", "hg/hgc/hgc.c:5748", sep="_", note="Underline."), v("_b", "bool", "hg/hgc/hgc.c:5750", sep="_", note="Bold."), v("_i", "bool", "hg/hgc/hgc.c:5752", sep="_", note="Italic."), v("_red", "int", "hg/hgc/hgc.c:5754", sep="_", note="0-255."), v("_green", "int", "hg/hgc/hgc.c:5756", sep="_", note="0-255."), v("_blue", "int", "hg/hgc/hgc.c:5758", sep="_", note="0-255."), ], }, @@ -1675,51 +1803,62 @@ def write_baseline(names, sites=None, path=BASELINE_FILE): """Write the baseline, annotating each name with the file that reads it. The comment carries the file but not the line, so that ordinary edits above a call site do not rewrite hundreds of lines here and bury the one name that actually changed. """ with open(path, "w") as f: f.write("""\ # cartVarsNotCataloged.txt - names that harvestCartVars.py finds at a # cart*ClosestToHome() or safef("%s.%s") call site but that cartTrackVarCatalog.py # does not describe as a track-scoped cart variable. Refs #37838. # -# Most are not cart variables at all: the scan cannot tell one from a table -# name or an SQL fragment, so _gold comes out of it too. Some are cart -# variables that simply have not been cataloged yet. +# None of them is a cart variable: the scan cannot tell one from a table name +# or an SQL fragment, so _gold comes out of it too. See the walk below for +# the five classes they fall into. # # Filenames are NOT in here. A "%s.tmp" built from a filename has the same # shape as a "%s.heightPer" built from a track name, and 15 names of that kind # used to sit below with a new one arriving every few weeks. The harvester now # reads them from the trailing extension instead: harvestCartVars.py # --filenames lists what that rule claims and explains it. # # cartTrackVarCatalog.py --reconcile complains about any harvested name in # neither the catalog nor this file, so this is what keeps a nightly run quiet # until something actually changes. Regenerate with --update-baseline, then read # the diff before committing: a name appearing here is a decision that it is not # a cart variable worth cataloging, and a name disappearing means its call site # went away. # # The first version of this file was accepted wholesale, as a snapshot of the -# gap on the day reconcile learned to fail. So a name being in here is not -# evidence that anybody has looked at it; only the ones added since, which -# arrive a few at a time in a reviewable diff, carry that weight. +# gap on the day reconcile learned to fail, so for a year a name being in here +# was not evidence that anybody had looked at it. That is no longer true: on +# 2026-09-05 every remaining name was read at its call site, and what was left +# falls into five classes. Item labels that only look like variables +# (gvfItemName appends _unk, _dnovo and six more to an item's name for +# display). HTML element ids, built the same way a cart name is and never +# sent to the server, which is most of them. Table and db.table names. One +# submit-button name that is read with cgiOptionalString and never from the +# cart (snp125Defaults_coloring). And one printf artifact, "\\n". +# +# So a name in here now means somebody decided it is not a cart variable. A +# name that arrives later does not: read its call site before believing this +# file about it. Twenty-three names left in that walk and are described in +# cartTrackVarCatalog.py, eighteen of them the My Variants edit form's. # # Names are stored with the leading separator stripped, which is how reconcile # compares them. """) sites = sites or {} for n in sorted(names): where = sites.get(n, "").rsplit(":", 1)[0] if where: f.write("%-34s # %s\n" % (n, where)) else: f.write("%s\n" % n) def harvestModule(): """The harvester next door, or None if it cannot be imported.