17b7d3c37be41135afaf8e91e365e3847af96ca5
lrnassar
  Mon Jun 22 10:56:56 2026 -0700
Add TAD (topologically associating domains) track set on hg19, hg38, mm10, mm39. refs #21599

New "tads" superTrack collecting published TAD calls, alpha-gated via include tad.ra
alpha in each assembly's trackDb.ra.

hg38 (all five sources): Dixon 2012 domains, Schmitt 2016 boundaries, McArthur & Capra
2021 boundary stability, ENCODE contact domains (faceted composite over 117 biosamples),
and 3D Genome Browser 2.0 domains (faceted composite over 464 datasets).
hg19: the three sources with hg19-compatible data (Dixon, Schmitt, McArthur).
mm10/mm39 (domains only; the boundary sources have no mouse data): Dixon, ENCODE
(faceted, 16 biosamples), and 3D Genome Browser (faceted, 30 datasets); mm39 lifted
from mm10, lift noted in the long labels.

Faceted composites are organ-colored from a TAD-owned organ_colors.json symlinked into
/gbdb/<asm>/bbi/tad/. Build scripts and autoSql are version-controlled under
makeDb/scripts/tad/ and symlinked into the per-source build dirs. Provenance and fetch
for every dataset are documented in the makedocs (doc/hg38/tad.txt, doc/mm10/tad.txt,
doc/mm39/tad.txt, and the hg19 TAD section in doc/hg19.txt).

diff --git src/hg/makeDb/scripts/tad/buildTads3dgb.py src/hg/makeDb/scripts/tad/buildTads3dgb.py
new file mode 100644
index 00000000000..d0861fbd556
--- /dev/null
+++ src/hg/makeDb/scripts/tad/buildTads3dgb.py
@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+# Build the tads3dgb faceted bigBed composite: all 464 human hg38 3D Genome Browser
+# TAD-domain datasets, one subtrack each, faceted by metadata, organ-colored.
+# Literal data: only a format normalization (10-col 3DGB bed -> bed4). No re-calling,
+# merging, lifting, or recurrence scoring. refs #21599
+#
+# Reads:  3DGB source beds + datasets_api.json + classification.tsv (project dir)
+#         organ_colors.json (TAD-owned /hive/data/outside/tad copy; has Colon/Bladder/Cervix added)
+# Writes: build/hg38/tads3dgb/<id>.bb, tads3dgb_metadata.tsv, tads3dgb.ra
+
+import json, csv, os, subprocess, sys, collections, re
+
+# Assembly-aware: `python3 buildTads3dgb.py [hg38|mm10]` (default hg38). hg38 iterates the human
+# classification.tsv (which carries the curated Condition/Treatment/Provenance facets); mm10 iterates
+# the mouse TAD-called datasets straight from the API (no curated facets -> those 3 are omitted).
+# mm39 is produced by lifting the mm10 bigBeds (see liftTads3dgbMouse.sh).
+ASM    = sys.argv[1] if len(sys.argv) > 1 else "hg38"
+APIJS  = "/hive/users/lrnassar/claude/RM21599/3dgenome/datasets_api.json"
+CLASS  = "/hive/users/lrnassar/claude/RM21599/3dgenome/classification.tsv"
+COLORS = "/hive/data/outside/tad/organ_colors.json"   # TAD-owned (symlinked into /gbdb/<asm>/bbi/tad/)
+ASCFG = {
+ "hg38": dict(beds="/hive/users/lrnassar/claude/RM21599/3dgenome/tad_beds",
+              chroms="/hive/data/genomes/hg38/chrom.sizes",
+              out="/hive/data/outside/tad/3dgenome/build/hg38", species="human",
+              default_on={"63","67","122","103"}),   # GM12878, H1-ESC, IMR-90, HMEC
+ "mm10": dict(beds="/hive/data/outside/tad/3dgenome/mouse/tad_beds",
+              chroms="/hive/data/genomes/mm10/chrom.sizes",
+              out="/hive/data/outside/tad/3dgenome/build/mm10", species="mouse",
+              default_on=set()),                      # mouse ships all-off (per user)
+}
+C = ASCFG[ASM]
+BEDS=C["beds"]; CHROMS=C["chroms"]; OUT=C["out"]; SPECIES=C["species"]; DEFAULT_ON=C["default_on"]
+HAS_CTP = (ASM == "hg38")          # Condition/Treatment/Provenance facets (human curation only)
+GBDB    = "/gbdb/%s/bbi/tad" % ASM
+BBDIR  = os.path.join(OUT, "tads3dgb")
+TSV    = os.path.join(OUT, "tads3dgb_metadata.tsv")
+RA     = os.path.join(OUT, "tads3dgb.ra")
+
+# 3DGB organ value -> organ_colors.json key (case/vocabulary normalization).
+# The 5 case-only fixes map to existing keys; Colon/Bladder/Cervix were added to the JSON.
+ORGAN_NORM = {
+    "Blood Vessel":      "Blood vessel",
+    "Lymphoid Tissue":   "Lymphoid tissue",
+    "Small Intestine":   "Small intestine",
+    "Adrenal Gland":     "Adrenal gland",
+    "Connective Tissue": "Connective tissue",
+}
+
+def hex2rgb(h):
+    h = h.lstrip("#")
+    return "%d,%d,%d" % (int(h[0:2],16), int(h[2:4],16), int(h[4:6],16))
+
+def clean(s):
+    """Collapse whitespace; never return empty (faceted JS throws on blank cells)."""
+    s = "" if s is None else str(s)
+    s = re.sub(r"\s+", " ", s.replace("\t", " ")).strip()
+    return s
+
+STOP = {"the","of","a","an","in","on","to","and","for","with","at","by"}
+def shortlab(name, maxlen=22):
+    """A concise, sensical shortLabel: underscores->spaces, truncate only at whole-word
+    boundaries (never mid-word), and drop a trailing connective word/punctuation."""
+    s = re.sub(r"\s+", " ", name.replace("_", " ")).strip()
+    if " of " in s.lower():        # "A of B" -> organ-first "B A" (avoids dangling-preposition cuts)
+        i = s.lower().find(" of "); head, tail = s[:i].strip(), s[i+4:].strip()
+        if head and tail:
+            if head[:1].isupper() and head[1:2].islower():
+                head = head[0].lower() + head[1:]
+            s = tail + " " + head
+    if len(s) > maxlen:
+        out = ""
+        for t in s.split(" "):
+            if len((out + " " + t).strip()) <= maxlen:
+                out = (out + " " + t).strip()
+            else:
+                break
+        s = out if out else s[:maxlen]
+    toks = s.split(" ")
+    while len(toks) > 1 and toks[-1].lower().strip(",;-") in STOP:
+        toks.pop()
+    return " ".join(toks).rstrip(" ,;-")
+
+def scap(s):
+    """Sentence-case: capitalize the first letter only if the first word is all-lowercase
+    (so acronyms/proper tokens like GM12878, Hi-C, RAD21 are preserved)."""
+    if not s:
+        return s
+    first = s.split(" ", 1)[0]
+    if first[:1].isalpha() and first == first.lower():
+        return s[:1].upper() + s[1:]
+    return s
+
+def main():
+    os.makedirs(BBDIR, exist_ok=True)
+
+    api = {str(r["id"]): r for r in json.load(open(APIJS))}
+    colors = json.load(open(COLORS))["Organ"]
+
+    if ASM == "hg38":
+        rows = []
+        with open(CLASS) as fh:
+            for r in csv.DictReader(fh, delimiter="\t"):
+                rows.append({k: (v.strip() if v else "") for k, v in r.items()})
+    else:
+        # mm10: the iteration source is the set of mouse TAD-called datasets (one <id>.bed per
+        # dataset in BEDS); all per-dataset metadata (year, refNo, organ, ...) comes from the API.
+        have = sorted(int(f[:-4]) for f in os.listdir(BEDS) if f.endswith(".bed"))
+        rows = [{"id": str(i), "bed": "%d.bed" % i} for i in have]
+
+    meta = []          # (id, dict of tsv cols, color rgb, shortLabel, longLabel, mouseOver)
+    missing_color = collections.Counter()
+    fail = []
+
+    for r in rows:
+        did = r["id"]
+        bed = os.path.join(BEDS, r["bed"])
+        if not os.path.exists(bed):
+            fail.append((did, "missing bed")); continue
+        a = api.get(did, {})
+
+        organ_raw = clean(a.get("organ")) or "Unknown"
+        organ = ORGAN_NORM.get(organ_raw, organ_raw)
+        if organ not in colors:
+            missing_color[organ] += 1
+        rgb = hex2rgb(colors.get(organ, "#000000"))
+
+        name   = clean(a.get("name")) or ("dataset " + did)
+        cell   = scap(clean(a.get("cellType"))) or "(unspecified)"
+        assay  = scap(clean(a.get("dataType"))) or "Unknown"
+        cond   = {"normal": "Normal", "cancer": "Cancer"}.get(r.get("normal_cancer",""), "Unknown")
+        treat  = {"baseline": "Baseline", "pert": "Perturbation"}.get(r.get("baseline_pert",""), "Unknown")
+        prov   = {"have": "Also in another UCSC track", "novel": "Novel to browser"}.get(r.get("have_novel",""), "Unknown")
+        yr     = r.get("year") or a.get("year", "")     # classification (human) or API (mouse)
+        try:
+            year = str(int(float(yr))) if yr else "Unknown"
+        except ValueError:
+            year = "Unknown"
+        study  = clean(r.get("refNo") or a.get("refNo")) or "Unknown"
+        desc   = clean(a.get("description")) or name
+
+        # display name: underscores -> spaces, sentence-cased (logic still keys on id/name)
+        dname = scap(name.replace("_", " "))
+        short = scap(shortlab(name))
+        long  = "%s TAD domains (%s, %s, %s)" % (dname, organ, assay, study)
+        mouse = "3DGB TAD domain: %s (%s, %s)" % (dname, organ, assay)
+
+        # bed4: format-only. sort -k1,1 -k2,2n, then chrom/start/end + display name (-tab: name has spaces).
+        bb = os.path.join(BBDIR, did + ".bb")
+        tmp = os.path.join(BBDIR, did + ".bed4")
+        with open(tmp, "w") as out:
+            p1 = subprocess.run(["sort", "-k1,1", "-k2,2n", bed],
+                                stdout=subprocess.PIPE, check=True, text=True)
+            for line in p1.stdout.splitlines():
+                f = line.split("\t")
+                if len(f) < 3:
+                    continue
+                out.write("%s\t%s\t%s\t%s\n" % (f[0], f[1], f[2], dname))
+        rc = subprocess.run(["bedToBigBed", "-type=bed4", "-tab", tmp, CHROMS, bb],
+                            stderr=subprocess.PIPE, text=True)
+        os.remove(tmp)
+        if rc.returncode != 0:
+            fail.append((did, rc.stderr.strip())); continue
+        od = [("DatasetId", did), ("Organ", organ), ("Cell_type", cell), ("Assay", assay)]
+        if HAS_CTP:   # human-only curated facets
+            od += [("Condition", cond), ("Treatment", treat), ("Provenance", prov)]
+        od += [("Year", year), ("Study", study), ("_Description", desc)]
+        meta.append((did, collections.OrderedDict(od), rgb, short, long, mouse))
+
+    if fail:
+        print("FAILURES (%d):" % len(fail), file=sys.stderr)
+        for d, e in fail[:20]:
+            print("  ", d, e, file=sys.stderr)
+    if missing_color:
+        print("ORGANS WITHOUT COLOR KEY:", dict(missing_color), file=sys.stderr)
+    print("converted %d / %d datasets" % (len(meta), len(rows)))
+
+    # metadata TSV (tab-sep; primaryKey + facet cols + hidden _Description)
+    cols = list(meta[0][1].keys())
+    with open(TSV, "w") as fh:
+        fh.write("\t".join(cols) + "\n")
+        for _, m, *_ in meta:
+            fh.write("\t".join(m[c] for c in cols) + "\n")
+
+    # faceted composite stanza
+    with open(RA, "w") as fh:
+        fh.write(
+"""track tads3dgb
+parent tads
+priority 3
+compositeTrack faceted
+shortLabel 3D Genome Browser
+longLabel TAD domains across %d %s Hi-C/Micro-C datasets (3D Genome Browser 2.0)
+type bigBed 4
+group regulation
+visibility hide
+metaDataUrl %s/tads3dgb_metadata.tsv
+primaryKey DatasetId
+colorSettingsUrl %s/organ_colors.json
+maxCheckboxes 50
+html tads3dgb
+
+""" % (len(meta), SPECIES, GBDB, GBDB))
+        for did, m, rgb, short, long, mouse in meta:
+            onoff = "on" if did in DEFAULT_ON else "off"
+            fh.write(
+"""    track tads3dgb_%s
+    parent tads3dgb %s
+    shortLabel %s
+    longLabel %s
+    type bigBed 4
+    bigDataUrl %s/tads3dgb/%s.bb
+    color %s
+    visibility dense
+    mouseOver %s
+
+""" % (did, onoff, short, long, GBDB, did, rgb, mouse))
+
+    print("wrote %d subtracks -> %s" % (len(meta), RA))
+    print("metadata -> %s" % TSV)
+
+if __name__ == "__main__":
+    main()