fd12b362b0ccddedbc7055cc08f7fbe34b1bbdea
mspeir
  Fri Aug 21 07:59:17 2026 -0700
G2P otto: report the raw unrecognized confidence value, refs #38142

Code-review item from Jairo. The tally is keyed on the normalized value, so the
log printed the folded form rather than the text that is actually in the CSV.
Keep one raw example alongside each count and print that, so the value can be
found in the source file.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

diff --git src/hg/utils/otto/g2p/doG2p.py src/hg/utils/otto/g2p/doG2p.py
index 8e1145b23cc..9c5f7e3a36d 100755
--- src/hg/utils/otto/g2p/doG2p.py
+++ src/hg/utils/otto/g2p/doG2p.py
@@ -141,49 +141,55 @@
             continue
         fields = line.split("\t")[:8]
         name = fields[3]                       # e.g. "HGNC:36036"
         hgncId = name.split("HGNC:")[-1]
         if hgncId in wanted:
             coordMap.setdefault(hgncId, []).append(fields)
     return coordMap
 
 
 def joinAndWrite(g2pData, coords, outputFile):
     """Join G2P records and HGNC coordinates into BED 9+20 and write to outputFile.
 
     Returns a stats dict, both counts in G2P records so they are comparable:
       "unmatched"         -> count of G2P records whose HGNC ID had no coordinate
                              match in this assembly's HGNC track (they are skipped).
-      "unknownConfidence" -> {normalized confidence value: count of records} for
+      "unknownConfidence" -> {normalized confidence value: (count of records, one
+                             example of the value as it appeared in the CSV)} for
                              values not in CONFIDENCE_COLORS (colored black).
     """
     unmatched = 0
     unknownConfidence = {}
     with open(outputFile, "w", newline="", encoding="utf-8") as out:
         writer = csv.writer(out, delimiter="\t")
         for hgncId, rows in g2pData.items():
             matches = coords.get(hgncId, [])
             if not matches:
                 unmatched += len(rows)
                 continue
             for row in rows:
                 # Counted once per G2P record, not once per output line: an HGNC ID
                 # can carry several coordinate rows, which would inflate the tally.
                 rgb = confidenceToColor(row["confidence"])
                 if rgb is None:
+                    # Tally on the folded value so case and stray whitespace do not split
+                    # one unknown value into several, but keep a raw example alongside it:
+                    # the folded form is not what is in the CSV, so it is not what someone
+                    # reading the log would grep for.
                     key = normalizeConfidence(row["confidence"])
-                    unknownConfidence[key] = unknownConfidence.get(key, 0) + 1
+                    count, example = unknownConfidence.get(key, (0, row["confidence"]))
+                    unknownConfidence[key] = (count + 1, example)
                     rgb = DEFAULT_COLOR
 
                 # G2P 20 fields
                 g2pId       = row["g2p id"]
                 geneMim     = row["gene mim"]
                 hgncIdVal   = row["hgnc id"]
                 prevSymbols = row["previous gene symbols"].replace(";", ",")
                 diseaseName = row["disease name"]
                 diseaseMim  = row["disease mim"]
                 diseaseMondo = row["disease MONDO"]
                 allelicReq  = row["allelic requirement"]
                 crossMod    = row["cross cutting modifier"]
                 confidence  = row["confidence"]
                 varConseq   = row["variant consequence"]
                 varTypes    = row["variant types"]
@@ -267,33 +273,33 @@
 
     coordsByDb = {db: loadCoordinates(db, hgncIds) for db in DBS}
     for db in DBS:
         print("Loaded %s %s HGNC IDs" % (len(coordsByDb[db]), db))
 
     builtBb = {}
     for db in DBS:
         bedFile = "%s/%s_g2p_all.bed" % (buildDir, db)
         bbFile = "%s/%s_g2p.bb" % (buildDir, db)
         twoBit = "/gbdb/%s/%s.2bit" % (db, db)
         stats = joinAndWrite(g2pData, coordsByDb[db], bedFile)
         print("Wrote %s" % bedFile)
         if stats["unmatched"]:
             print("%s: %d G2P record(s) had no HGNC coordinate match and were skipped"
                   % (db, stats["unmatched"]))
-        for conf, n in sorted(stats["unknownConfidence"].items()):
+        for conf, (n, example) in sorted(stats["unknownConfidence"].items()):
             print("%s: unrecognized confidence value %r on %d record(s); colored black"
-                  % (db, conf, n))
+                  % (db, example, n))
         bash("bedToBigBed -type=bed9+20 -tab -sort "
              "-as=%s -sizesIs2Bit -extraIndex=name,g2p_id,gene_mim,hgnc_id %s %s %s"
              % (AS_FILE, bedFile, twoBit, bbFile))
         print("Built %s" % bbFile)
         builtBb[db] = bbFile
 
     # Safety check before swapping anything live.
     for db in DBS:
         checkItemCount(db, builtBb[db])
 
     for db in DBS:
         install(db, builtBb[db])
 
     bash("mv %s %s" % (NEW_CSV, PREV_CSV))
     print("G2P updated %s" % date)