5d23e68307f894bdcd024c138adc588591eceaaa
lrnassar
  Mon Jun 29 12:11:44 2026 -0700
Polish TAD description pages and bold the track mouseOvers. refs #21599

Description pages (tads.html on hg19, hg38, mm10, mm39): add a Methods resolution
summary table (track / feature / calling method / resolution), placed at the end of
the Description section; add GitHub source links (makedoc, build scripts, trackDb.ra)
to the Methods section; sentence-case the "How to use these tracks" heading.

mouseOvers: wrap the field labels in <b> across all subtracks - Dixon ("Cell type:"),
Schmitt ("Sample:"), ENCODE ("Biosample:" / "Arrowhead corner score:", now on separate
lines), 3D Genome Browser ("3DGB TAD domain:"), and McArthur boundary stability
("Boundary recurrent:" / "Percentile:"). Updated the ENCODE and 3D Genome Browser build
scripts so regenerated stanzas carry the same formatting.

diff --git src/hg/makeDb/scripts/tad/buildTads3dgb.py src/hg/makeDb/scripts/tad/buildTads3dgb.py
index d0861fbd556..fc582ebdc54 100644
--- src/hg/makeDb/scripts/tad/buildTads3dgb.py
+++ src/hg/makeDb/scripts/tad/buildTads3dgb.py
@@ -1,222 +1,222 @@
 #!/usr/bin/env python3
 # Build the tads3dgb faceted bigBed composite: all 464 human hg38 3D Genome Browser
 # TAD-domain datasets, one subtrack each, faceted by metadata, organ-colored.
 # Literal data: only a format normalization (10-col 3DGB bed -> bed4). No re-calling,
 # merging, lifting, or recurrence scoring. refs #21599
 #
 # Reads:  3DGB source beds + datasets_api.json + classification.tsv (project dir)
 #         organ_colors.json (TAD-owned /hive/data/outside/tad copy; has Colon/Bladder/Cervix added)
 # Writes: build/hg38/tads3dgb/<id>.bb, tads3dgb_metadata.tsv, tads3dgb.ra
 
 import json, csv, os, subprocess, sys, collections, re
 
 # Assembly-aware: `python3 buildTads3dgb.py [hg38|mm10]` (default hg38). hg38 iterates the human
 # classification.tsv (which carries the curated Condition/Treatment/Provenance facets); mm10 iterates
 # the mouse TAD-called datasets straight from the API (no curated facets -> those 3 are omitted).
 # mm39 is produced by lifting the mm10 bigBeds (see liftTads3dgbMouse.sh).
 ASM    = sys.argv[1] if len(sys.argv) > 1 else "hg38"
 APIJS  = "/hive/users/lrnassar/claude/RM21599/3dgenome/datasets_api.json"
 CLASS  = "/hive/users/lrnassar/claude/RM21599/3dgenome/classification.tsv"
 COLORS = "/hive/data/outside/tad/organ_colors.json"   # TAD-owned (symlinked into /gbdb/<asm>/bbi/tad/)
 ASCFG = {
  "hg38": dict(beds="/hive/users/lrnassar/claude/RM21599/3dgenome/tad_beds",
               chroms="/hive/data/genomes/hg38/chrom.sizes",
               out="/hive/data/outside/tad/3dgenome/build/hg38", species="human",
               default_on={"63","67","122","103"}),   # GM12878, H1-ESC, IMR-90, HMEC
  "mm10": dict(beds="/hive/data/outside/tad/3dgenome/mouse/tad_beds",
               chroms="/hive/data/genomes/mm10/chrom.sizes",
               out="/hive/data/outside/tad/3dgenome/build/mm10", species="mouse",
               default_on=set()),                      # mouse ships all-off (per user)
 }
 C = ASCFG[ASM]
 BEDS=C["beds"]; CHROMS=C["chroms"]; OUT=C["out"]; SPECIES=C["species"]; DEFAULT_ON=C["default_on"]
 HAS_CTP = (ASM == "hg38")          # Condition/Treatment/Provenance facets (human curation only)
 GBDB    = "/gbdb/%s/bbi/tad" % ASM
 BBDIR  = os.path.join(OUT, "tads3dgb")
 TSV    = os.path.join(OUT, "tads3dgb_metadata.tsv")
 RA     = os.path.join(OUT, "tads3dgb.ra")
 
 # 3DGB organ value -> organ_colors.json key (case/vocabulary normalization).
 # The 5 case-only fixes map to existing keys; Colon/Bladder/Cervix were added to the JSON.
 ORGAN_NORM = {
     "Blood Vessel":      "Blood vessel",
     "Lymphoid Tissue":   "Lymphoid tissue",
     "Small Intestine":   "Small intestine",
     "Adrenal Gland":     "Adrenal gland",
     "Connective Tissue": "Connective tissue",
 }
 
 def hex2rgb(h):
     h = h.lstrip("#")
     return "%d,%d,%d" % (int(h[0:2],16), int(h[2:4],16), int(h[4:6],16))
 
 def clean(s):
     """Collapse whitespace; never return empty (faceted JS throws on blank cells)."""
     s = "" if s is None else str(s)
     s = re.sub(r"\s+", " ", s.replace("\t", " ")).strip()
     return s
 
 STOP = {"the","of","a","an","in","on","to","and","for","with","at","by"}
 def shortlab(name, maxlen=22):
     """A concise, sensical shortLabel: underscores->spaces, truncate only at whole-word
     boundaries (never mid-word), and drop a trailing connective word/punctuation."""
     s = re.sub(r"\s+", " ", name.replace("_", " ")).strip()
     if " of " in s.lower():        # "A of B" -> organ-first "B A" (avoids dangling-preposition cuts)
         i = s.lower().find(" of "); head, tail = s[:i].strip(), s[i+4:].strip()
         if head and tail:
             if head[:1].isupper() and head[1:2].islower():
                 head = head[0].lower() + head[1:]
             s = tail + " " + head
     if len(s) > maxlen:
         out = ""
         for t in s.split(" "):
             if len((out + " " + t).strip()) <= maxlen:
                 out = (out + " " + t).strip()
             else:
                 break
         s = out if out else s[:maxlen]
     toks = s.split(" ")
     while len(toks) > 1 and toks[-1].lower().strip(",;-") in STOP:
         toks.pop()
     return " ".join(toks).rstrip(" ,;-")
 
 def scap(s):
     """Sentence-case: capitalize the first letter only if the first word is all-lowercase
     (so acronyms/proper tokens like GM12878, Hi-C, RAD21 are preserved)."""
     if not s:
         return s
     first = s.split(" ", 1)[0]
     if first[:1].isalpha() and first == first.lower():
         return s[:1].upper() + s[1:]
     return s
 
 def main():
     os.makedirs(BBDIR, exist_ok=True)
 
     api = {str(r["id"]): r for r in json.load(open(APIJS))}
     colors = json.load(open(COLORS))["Organ"]
 
     if ASM == "hg38":
         rows = []
         with open(CLASS) as fh:
             for r in csv.DictReader(fh, delimiter="\t"):
                 rows.append({k: (v.strip() if v else "") for k, v in r.items()})
     else:
         # mm10: the iteration source is the set of mouse TAD-called datasets (one <id>.bed per
         # dataset in BEDS); all per-dataset metadata (year, refNo, organ, ...) comes from the API.
         have = sorted(int(f[:-4]) for f in os.listdir(BEDS) if f.endswith(".bed"))
         rows = [{"id": str(i), "bed": "%d.bed" % i} for i in have]
 
     meta = []          # (id, dict of tsv cols, color rgb, shortLabel, longLabel, mouseOver)
     missing_color = collections.Counter()
     fail = []
 
     for r in rows:
         did = r["id"]
         bed = os.path.join(BEDS, r["bed"])
         if not os.path.exists(bed):
             fail.append((did, "missing bed")); continue
         a = api.get(did, {})
 
         organ_raw = clean(a.get("organ")) or "Unknown"
         organ = ORGAN_NORM.get(organ_raw, organ_raw)
         if organ not in colors:
             missing_color[organ] += 1
         rgb = hex2rgb(colors.get(organ, "#000000"))
 
         name   = clean(a.get("name")) or ("dataset " + did)
         cell   = scap(clean(a.get("cellType"))) or "(unspecified)"
         assay  = scap(clean(a.get("dataType"))) or "Unknown"
         cond   = {"normal": "Normal", "cancer": "Cancer"}.get(r.get("normal_cancer",""), "Unknown")
         treat  = {"baseline": "Baseline", "pert": "Perturbation"}.get(r.get("baseline_pert",""), "Unknown")
         prov   = {"have": "Also in another UCSC track", "novel": "Novel to browser"}.get(r.get("have_novel",""), "Unknown")
         yr     = r.get("year") or a.get("year", "")     # classification (human) or API (mouse)
         try:
             year = str(int(float(yr))) if yr else "Unknown"
         except ValueError:
             year = "Unknown"
         study  = clean(r.get("refNo") or a.get("refNo")) or "Unknown"
         desc   = clean(a.get("description")) or name
 
         # display name: underscores -> spaces, sentence-cased (logic still keys on id/name)
         dname = scap(name.replace("_", " "))
         short = scap(shortlab(name))
         long  = "%s TAD domains (%s, %s, %s)" % (dname, organ, assay, study)
-        mouse = "3DGB TAD domain: %s (%s, %s)" % (dname, organ, assay)
+        mouse = "<b>3DGB TAD domain:</b> %s (%s, %s)" % (dname, organ, assay)
 
         # bed4: format-only. sort -k1,1 -k2,2n, then chrom/start/end + display name (-tab: name has spaces).
         bb = os.path.join(BBDIR, did + ".bb")
         tmp = os.path.join(BBDIR, did + ".bed4")
         with open(tmp, "w") as out:
             p1 = subprocess.run(["sort", "-k1,1", "-k2,2n", bed],
                                 stdout=subprocess.PIPE, check=True, text=True)
             for line in p1.stdout.splitlines():
                 f = line.split("\t")
                 if len(f) < 3:
                     continue
                 out.write("%s\t%s\t%s\t%s\n" % (f[0], f[1], f[2], dname))
         rc = subprocess.run(["bedToBigBed", "-type=bed4", "-tab", tmp, CHROMS, bb],
                             stderr=subprocess.PIPE, text=True)
         os.remove(tmp)
         if rc.returncode != 0:
             fail.append((did, rc.stderr.strip())); continue
         od = [("DatasetId", did), ("Organ", organ), ("Cell_type", cell), ("Assay", assay)]
         if HAS_CTP:   # human-only curated facets
             od += [("Condition", cond), ("Treatment", treat), ("Provenance", prov)]
         od += [("Year", year), ("Study", study), ("_Description", desc)]
         meta.append((did, collections.OrderedDict(od), rgb, short, long, mouse))
 
     if fail:
         print("FAILURES (%d):" % len(fail), file=sys.stderr)
         for d, e in fail[:20]:
             print("  ", d, e, file=sys.stderr)
     if missing_color:
         print("ORGANS WITHOUT COLOR KEY:", dict(missing_color), file=sys.stderr)
     print("converted %d / %d datasets" % (len(meta), len(rows)))
 
     # metadata TSV (tab-sep; primaryKey + facet cols + hidden _Description)
     cols = list(meta[0][1].keys())
     with open(TSV, "w") as fh:
         fh.write("\t".join(cols) + "\n")
         for _, m, *_ in meta:
             fh.write("\t".join(m[c] for c in cols) + "\n")
 
     # faceted composite stanza
     with open(RA, "w") as fh:
         fh.write(
 """track tads3dgb
 parent tads
 priority 3
 compositeTrack faceted
 shortLabel 3D Genome Browser
 longLabel TAD domains across %d %s Hi-C/Micro-C datasets (3D Genome Browser 2.0)
 type bigBed 4
 group regulation
 visibility hide
 metaDataUrl %s/tads3dgb_metadata.tsv
 primaryKey DatasetId
 colorSettingsUrl %s/organ_colors.json
 maxCheckboxes 50
 html tads3dgb
 
 """ % (len(meta), SPECIES, GBDB, GBDB))
         for did, m, rgb, short, long, mouse in meta:
             onoff = "on" if did in DEFAULT_ON else "off"
             fh.write(
 """    track tads3dgb_%s
     parent tads3dgb %s
     shortLabel %s
     longLabel %s
     type bigBed 4
     bigDataUrl %s/tads3dgb/%s.bb
     color %s
     visibility dense
     mouseOver %s
 
 """ % (did, onoff, short, long, GBDB, did, rgb, mouse))
 
     print("wrote %d subtracks -> %s" % (len(meta), RA))
     print("metadata -> %s" % TSV)
 
 if __name__ == "__main__":
     main()