62fad287d92a5937f768173e48381544043e7b3e
mspeir
  Fri Aug 21 13:03:58 2026 -0700
G2P otto: stop csv.writer putting quotes into the track text, refs #38142

The BED was written with csv.writer, whose default dialect treats the double
quote as its own quote character. Any field containing a quotation therefore
came out wrapped in quotes with the inner quotes doubled, and nine G2P comments
carry one, so that punctuation is in the released track today:

"Note, a 7-residue ""hot spot"" within the so-called hinge domain ...

Fields now go through bedField(), which keeps the text as G2P wrote it and only
takes out the tab and newline that would break the row. The same change drops
the CRLF line endings the excel dialect was emitting; bedToBigBed was already
stripping those, so they did no harm, but the BED is a plain tab file now.

Rebuilding hg38 from the same CSV changes exactly those nine records and
nothing else: same 4,214 items, same 29 fields, coordinates untouched.
/gbdb/{hg19,hg38}/g2p/g2p.bb still point at the previous build, so this needs a
rebuild to reach the track.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

diff --git src/hg/utils/otto/g2p/doG2p.py src/hg/utils/otto/g2p/doG2p.py
index 9c5f7e3a36d..dc6b089ee56 100755
--- src/hg/utils/otto/g2p/doG2p.py
+++ src/hg/utils/otto/g2p/doG2p.py
@@ -100,30 +100,45 @@
 }
 DEFAULT_COLOR = "0,0,0"          # black, for unrecognized confidence values
 
 
 def normalizeConfidence(confidence):
     """Fold a confidence string to its lookup form, so that case and stray
     whitespace do not make one value look like several."""
     return confidence.lower().strip()
 
 
 def confidenceToColor(confidence):
     """Return the itemRgb color for a confidence string, or None if unrecognized."""
     return CONFIDENCE_COLORS.get(normalizeConfidence(confidence))
 
 
+def bedField(value):
+    """Flatten one CSV value into a single tab-separated BED field.
+
+    This used to go through csv.writer, whose default QUOTE_MINIMAL treats the
+    double quote as its own quote character: any field holding one came out wrapped
+    in quotes with the inner quotes doubled. Nine G2P comments carry a quotation, so
+    that punctuation reached the live track and showed up on the details page.
+    bedToBigBed wants the raw text, and only needs the field and line separators
+    kept out of it.
+    """
+    if value is None:
+        return ""
+    return str(value).replace("\t", " ").replace("\r", " ").replace("\n", " ")
+
+
 def loadG2p(filePath):
     """Load G2P CSV into a dict keyed by HGNC ID (each value is a list of rows)."""
     g2pMap = {}
     numOfRows = 0
     with open(filePath, newline="", encoding="utf-8") as csvfile:
         reader = csv.DictReader(csvfile)
         for row in reader:
             numOfRows += 1
             hgncId = row["hgnc id"].strip()
             g2pMap.setdefault(hgncId, []).append(row)
     print("Number of rows in file: %s" % numOfRows)
     return g2pMap
 
 
 def loadCoordinates(db, hgncIds):
@@ -148,31 +163,30 @@
 
 
 def joinAndWrite(g2pData, coords, outputFile):
     """Join G2P records and HGNC coordinates into BED 9+20 and write to outputFile.
 
     Returns a stats dict, both counts in G2P records so they are comparable:
       "unmatched"         -> count of G2P records whose HGNC ID had no coordinate
                              match in this assembly's HGNC track (they are skipped).
       "unknownConfidence" -> {normalized confidence value: (count of records, one
                              example of the value as it appeared in the CSV)} for
                              values not in CONFIDENCE_COLORS (colored black).
     """
     unmatched = 0
     unknownConfidence = {}
     with open(outputFile, "w", newline="", encoding="utf-8") as out:
-        writer = csv.writer(out, delimiter="\t")
         for hgncId, rows in g2pData.items():
             matches = coords.get(hgncId, [])
             if not matches:
                 unmatched += len(rows)
                 continue
             for row in rows:
                 # Counted once per G2P record, not once per output line: an HGNC ID
                 # can carry several coordinate rows, which would inflate the tally.
                 rgb = confidenceToColor(row["confidence"])
                 if rgb is None:
                     # Tally on the folded value so case and stray whitespace do not split
                     # one unknown value into several, but keep a raw example alongside it:
                     # the folded form is not what is in the CSV, so it is not what someone
                     # reading the log would grep for.
                     key = normalizeConfidence(row["confidence"])
@@ -201,37 +215,37 @@
                 panel       = row["panel"]
                 comments    = row["comments"]
                 dateReview  = row["date of last review"]
 
                 for coord in matches:
                     # BED 9 fields
                     chrom       = coord[0]
                     chromStart  = coord[1]
                     chromEnd    = coord[2]
                     name        = row["gene symbol"]
                     score       = coord[4]
                     strand      = coord[5]
                     thickStart  = coord[6]
                     thickEnd    = coord[7]
 
-                    writer.writerow([
+                    out.write("\t".join(bedField(f) for f in (
                         chrom, chromStart, chromEnd, name, score, strand, thickStart, thickEnd,
                         rgb, g2pId, geneMim, hgncIdVal, prevSymbols, diseaseName, diseaseMim,
                         diseaseMondo, allelicReq, crossMod, confidence, varConseq, varTypes,
                         molMech, molMechCat, molMechEv, phenotypes, publications, panel,
                         comments, dateReview,
-                    ])
+                    )) + "\n")
     return {"unmatched": unmatched, "unknownConfidence": unknownConfidence}
 
 
 def itemCount(bb):
     line = bash('bigBedInfo %s | grep "itemCount"' % bb)
     return int(line.rstrip().split("itemCount:")[1].replace(",", "").strip())
 
 
 def checkItemCount(db, newBb):
     """Abort if the item count moved more than COUNT_TOLERANCE vs the live track."""
     liveBb = GBDB_BB % db
     if not Path(liveBb).exists():
         print("%s: no live bigBed yet, skipping item-count check" % db)
         return
     old = itemCount(liveBb)