5d23e68307f894bdcd024c138adc588591eceaaa
lrnassar
  Mon Jun 29 12:11:44 2026 -0700
Polish TAD description pages and bold the track mouseOvers. refs #21599

Description pages (tads.html on hg19, hg38, mm10, mm39): add a Methods resolution
summary table (track / feature / calling method / resolution), placed at the end of
the Description section; add GitHub source links (makedoc, build scripts, trackDb.ra)
to the Methods section; sentence-case the "How to use these tracks" heading.

mouseOvers: wrap the field labels in <b> across all subtracks - Dixon ("Cell type:"),
Schmitt ("Sample:"), ENCODE ("Biosample:" / "Arrowhead corner score:", now on separate
lines), 3D Genome Browser ("3DGB TAD domain:"), and McArthur boundary stability
("Boundary recurrent:" / "Percentile:"). Updated the ENCODE and 3D Genome Browser build
scripts so regenerated stanzas carry the same formatting.

diff --git src/hg/makeDb/scripts/tad/buildTadsEncode.py src/hg/makeDb/scripts/tad/buildTadsEncode.py
index a75ad07a72b..446f308331a 100644
--- src/hg/makeDb/scripts/tad/buildTadsEncode.py
+++ src/hg/makeDb/scripts/tad/buildTadsEncode.py
@@ -1,316 +1,316 @@
 #!/usr/bin/env python3
 # Rebuild tadsEncode as a faceted bigBed composite across ALL 117 single-biosample
 # ENCODE contact-domain datasets (replaces the old 14-subtrack subGroup composite).
 # refs #21599
 #
 # ENCODE's contact-domain files are heterogeneous: the canonical product is the
 # 16-column Juicer/Arrowhead "blocks" bedpe (chr1 x1 x2 chr2 y1 y2 name score
 # strand1 strand2 color cornerScore uVarScore lVarScore upSign loSign), but the
 # pool also contains 12/11-col bedpe variants, 24-col loop files (different anchors,
 # no "chr" prefix), and 5-col hg19-lifted bed. We therefore tier the files and use
 # the best available per biosample:
 #   tier 16 : 16-col bedpe (full Arrowhead scores)            -> 111 biosamples
 #   tier 12 : 12-col bedpe (cornerScore only; others 0)       -> A549
 #   tier 11 : 11-col bedpe (no scores; all 0)
 #   tier  5 : hg19-lifted bed5 (no scores; all 0)             -> 5 biosamples
 # Loop files (anchors differ, x!=y) and non-hg38 chroms are dropped by the parser.
 #
 # Per biosample: pick one representative experiment (best file tier; preferred_default;
 # assay priority intact>in situ>dilution; then total size). Pool that experiment's
 # files of the chosen tier (union; drop a domain whose endpoints both fall within one
 # 5 kb bin of a higher-cornerScore domain). Write bigBed 4+5 (tadDomainEncode).
 # Conversion verified vs the retired tadsEncodeGM12878.bb (scores taken verbatim;
 # duplicate resolution keeps the higher cornerScore).
 
 import json, os, re, gzip, subprocess, sys
 from collections import defaultdict, Counter
 
 # Assembly-aware: `python3 buildTadsEncode.py [hg38|mm10]` (default hg38, the original human build).
 # mm10 is the native mouse build; mm39 is produced by lifting the mm10 bigBeds (see liftEncodeMouse.sh).
 ASM = sys.argv[1] if len(sys.argv) > 1 else "hg38"
 _HUMAN = "/hive/users/lrnassar/claude/RM21599/encode"
 _MOUSE = "/hive/data/outside/tad/encode/mouse"
 CFG = {
  "hg38": dict(src=_HUMAN+"/contact_domains",
               manif="/hive/data/outside/tad/encode/source/_cd_select.json",
               meta="/hive/data/outside/tad/encode/source/encode_meta_all.json",
               pert="/hive/data/outside/tad/encode/source/encode_perturbed.json",
               chroms="/hive/data/genomes/hg38/chrom.sizes", out="/hive/data/outside/tad/encode/build/hg38",
               species="human",
               default_on={"GM12878","K562","HepG2","HCT116","IMR-90","A549","HL-60/S4",
                 "heart left ventricle","heart right ventricle","dorsolateral prefrontal cortex",
                 "ovary","pancreas","transverse colon","motor neuron"}),
  "mm10": dict(src=_MOUSE+"/contact_domains", manif=_MOUSE+"/source/_cd_select.json",
               meta=_MOUSE+"/source/encode_meta_all.json", pert=_MOUSE+"/source/encode_perturbed.json",
               chroms="/hive/data/genomes/mm10/chrom.sizes", out="/hive/data/outside/tad/encode/build/mm10",
               species="mouse",
               default_on={"mouse embryonic stem cell","CH12.LX","heart","left cerebral cortex"}),
 }
 C = CFG[ASM]
 SRC=C["src"]; MANIF=C["manif"]; META=C["meta"]; PERT=C["pert"]; CHROMS=C["chroms"]
 SPECIES=C["species"]; DEFAULT_ON=C["default_on"]
 COLORS = "/hive/data/outside/tad/organ_colors.json"   # TAD-owned (symlinked into /gbdb/<asm>/bbi/tad/)
 ASFILE = "/hive/data/outside/tad/tadDomainEncode.as"
 GBDB   = "/gbdb/%s/bbi/tad" % ASM
 OUT    = C["out"]
 BBDIR  = os.path.join(OUT, "tadsEncode")
 TSV    = os.path.join(OUT, "tadsEncode_metadata.tsv")
 RA     = os.path.join(OUT, "tadsEncode.ra")
 BIN    = 5000
 
 ASSAY_PRIORITY = {"intact Hi-C":0, "in situ Hi-C":1, "dilution Hi-C":2}
 
 SLIM2KEY = {
  'bodily fluid':'Blood','blood':'Blood','epithelium':'Epithelium',
  'heart':'Heart','pericardium':'Heart','lung':'Lung',
  'vasculature':'Blood vessel','blood vessel':'Blood vessel',
  'arterial blood vessel':'Blood vessel','vein':'Blood vessel',
  'colon':'Colon','large intestine':'Large intestine','intestine':'Large intestine',
  'brain':'Brain','spinal cord':'Spinal cord','nerve':'Nerve','eye':'Eye',
  'musculature of body':'Muscle','skin of body':'Skin','limb':'Limb',
  'mammary gland':'Breast','breast':'Breast',
  'embryo':'Embryo','extraembryonic component':'Placenta','placenta':'Placenta',
  'kidney':'Kidney','liver':'Liver','pancreas':'Pancreas','esophagus':'Esophagus','stomach':'Stomach',
  'connective tissue':'Connective tissue','bone marrow':'Bone marrow','bone element':'Bone',
  'prostate gland':'Prostate','uterus':'Uterus','vagina':'Vagina','ovary':'Ovary','testis':'Testis',
  'adrenal gland':'Adrenal gland','thyroid gland':'Thyroid','spleen':'Spleen',
  'immune organ':'Lymphoid tissue','lymph node':'Lymphoid tissue',
  'exocrine gland':'Epithelium','endocrine gland':'Epithelium','gonad':'Epithelium',
 }
 PRIORITY = ['Heart','Lung','Brain','Liver','Kidney','Pancreas','Prostate','Breast','Ovary','Testis',
  'Uterus','Vagina','Cervix','Stomach','Esophagus','Colon','Large intestine','Small intestine',
  'Spleen','Thyroid','Adrenal gland','Eye','Nerve','Spinal cord','Skin','Blood vessel','Muscle',
  'Bone marrow','Bone','Connective tissue','Placenta','Limb','Embryo','Lymphoid tissue',
  'Blood','Epithelium']
 PRANK = {k:i for i,k in enumerate(PRIORITY)}
 ORGAN_OVERRIDE = {'GM11168':'Blood','Calu3':'Lung','Ramos':'Blood',
                   'activated CD8-positive, alpha-beta T cell':'Blood',
                   # mouse biosamples with empty organ_slims (else fall back to Epithelium):
                   'CH12F3':'Blood','v-Abl transformed pro-B cells':'Blood',
                   'mouse trophoblast stem cell':'Placenta'}
 CLASS_LABEL = {'tissue':'Tissue','cell line':'Cell line','primary cell':'Primary cell',
                'in vitro differentiated cells':'In vitro differentiated'}
 
 def pick_organ(slims, term):
     keys=[SLIM2KEY[s] for s in slims if s in SLIM2KEY]
     return ORGAN_OVERRIDE.get(term,'Epithelium') if not keys else min(keys,key=lambda k:PRANK.get(k,999))
 
 def clean(s):
     s="" if s is None else str(s); return re.sub(r"\s+"," ",s.replace("\t"," ")).strip()
 def scap(s):
     """Sentence-case: capitalize the first letter only if the first word is all-lowercase
     (so acronyms/proper tokens like hTERT, GM12878, CD4+, Hi-C are preserved)."""
     if not s: return s
     first=s.split(" ",1)[0]
     if first[:1].isalpha() and first==first.lower(): return s[:1].upper()+s[1:]
     return s
 def hex2rgb(h):
     h=h.lstrip("#"); return "%d,%d,%d"%(int(h[0:2],16),int(h[2:4],16),int(h[4:6],16))
 
 # ASCII-safe abbreviations for long ENCODE biosample names, applied before truncation
 ENC_ABBR=[(r'CD4-positive','CD4+'),(r'CD8-positive','CD8+'),(r'CD34-positive','CD34+'),
           (r'CD14-positive','CD14+'),(r'thymus-derived ',''),(r', alpha-beta',''),
           (r' myocardium',''),(r'-positive','+'),(r'-negative','-')]
 STOP={"the","of","a","an","in","on","to","and","for","with","at","by"}
 def shortlab(name, maxlen=22, abbr=None):
     """Concise, sensical shortLabel: abbreviate, truncate only at whole-word boundaries
     (never mid-word), drop a trailing connective word/punctuation."""
     s=name.replace("_"," ")
     for pat,rep in (abbr or []): s=re.sub(pat,rep,s)
     s=re.sub(r"\s+"," ",s).strip()
     if " of " in s.lower():        # "A of B" -> organ-first "B A" (avoids dangling-preposition cuts)
         i=s.lower().find(" of "); head,tail=s[:i].strip(),s[i+4:].strip()
         if head and tail:
             if head[:1].isupper() and head[1:2].islower(): head=head[0].lower()+head[1:]
             s=tail+" "+head
     if len(s)>maxlen:
         out=""
         for t in s.split(" "):
             if len((out+" "+t).strip())<=maxlen: out=(out+" "+t).strip()
             else: break
         s=out if out else s[:maxlen]
     toks=s.split(" ")
     while len(toks)>1 and toks[-1].lower().strip(",;-") in STOP: toks.pop()
     return " ".join(toks).rstrip(" ,;-")
 
 CHROMSZ={l.split('\t')[0]: int(l.split('\t')[1]) for l in open(CHROMS)}
 
 def file_tier(acc, fmt):
     """First domain row's column count -> tier; 0 if not usable."""
     p=os.path.join(SRC, acc+('.bedpe.gz' if fmt=='bedpe' else '.bed.gz'))
     if not os.path.exists(p): return 0,None
     with gzip.open(p,'rt') as fh:
         for line in fh:
             if line.startswith('#'): continue
             c=line.rstrip('\n').split('\t')
             if len(c)>1 and c[1]=='x1': continue
             n=len(c)
             if fmt=='bed' and n>=5: return 5,p
             # exact column counts only: 16-col Arrowhead blocks (full scores), 12/11-col
             # variants. 24-col files are loops (different anchors) -> tier 0, excluded.
             if fmt=='bedpe' and n==16: return 16,p
             if fmt=='bedpe' and n==12: return 12,p
             if fmt=='bedpe' and n==11: return 11,p
             return 0,p
     return 0,p
 
 def parse_domains(path, fmt, tier, name):
     """Yield bed4+5 rows: domain intervals only, chrom-normalized, scores per tier."""
     out=[]
     with gzip.open(path,'rt') as fh:
         for line in fh:
             if line.startswith('#'): continue
             c=line.rstrip('\n').split('\t')
             if len(c)<3 or c[1]=='x1': continue
             ch=c[0] if c[0].startswith('chr') else 'chr'+c[0]
             if ch not in CHROMSZ: continue
             if fmt=='bedpe':
                 if len(c)<6: continue
                 # domain row: same chrom both anchors, same interval (x==y); else it's a loop
                 if c[1]!=c[4] or c[2]!=c[5]: continue
                 s,e=c[1],c[2]
                 if tier==16: sc=[c[11],c[12],c[13],c[14],c[15]]
                 elif tier==12: sc=[c[11],"0","0","0","0"]
                 else: sc=["0","0","0","0","0"]
             else:  # bed5
                 s,e=c[1],c[2]; sc=["0","0","0","0","0"]
             si,ei=int(s),int(e)
             if si<0 or ei>CHROMSZ[ch] or si>=ei: continue   # drop out-of-bounds/degenerate
             out.append([ch,si,ei,name]+sc+[float(sc[0])])
     return out
 
 def main():
     os.makedirs(BBDIR, exist_ok=True)
     recs=json.load(open(MANIF)); recs=recs if isinstance(recs,list) else recs.get("@graph",recs)
     meta=json.load(open(META)); colors=json.load(open(COLORS))["Organ"]
     pert=json.load(open(PERT))   # experiment accession -> {perturbed, summary}
 
     bybs=defaultdict(lambda: defaultdict(list))
     for r in recs:
         bo=r.get("biosample_ontology")
         if not isinstance(bo,dict): continue
         if "/aggregate" in str(r.get("dataset","")): continue
         bybs[bo["term_name"]][r["dataset"]].append(r)
 
     # annotate each file with its tier + path (cache)
     for term,exps in bybs.items():
         for ds,fs in exps.items():
             for f in fs:
                 f["_tier"],f["_path"]=file_tier(f["accession"],f["file_format"])
 
     def exp_rank(ds, efiles):
         besttier=max((f["_tier"] for f in efiles), default=0)
         encsr=ds.strip("/").split("/")[-1]
         perturbed=1 if pert.get(encsr,{}).get("perturbed") else 0  # prefer unperturbed
         has_pref=any(f.get("preferred_default") for f in efiles)
         assay=min((ASSAY_PRIORITY.get(f.get("assay_title"),9) for f in efiles), default=9)
         size=sum(f.get("file_size",0) for f in efiles if f["_tier"]==besttier)
         return (-besttier, perturbed, 0 if has_pref else 1, assay, -size)
 
     subtracks=[]; fail=[]
     for term,exps in bybs.items():
         exp,efiles=min(exps.items(), key=lambda kv:(exp_rank(kv[0],kv[1]), kv[0]))
         encsr=exp.strip("/").split("/")[-1]
         tier=max(f["_tier"] for f in efiles)
         if tier==0: fail.append((term,"no usable file")); continue
         use=[f for f in efiles if f["_tier"]==tier]
         fmt=use[0]["file_format"]
         assay=scap(sorted(use,key=lambda f:ASSAY_PRIORITY.get(f.get("assay_title"),9))[0].get("assay_title"))
         dterm=scap(term)   # sentence-cased display name (term kept original for logic/lookups)
 
         rows=[]
         for f in use:
             rows+=parse_domains(f["_path"], fmt, tier, dterm)
         if not rows: fail.append((term,"no domain rows")); continue
 
         kept=[]; bychrom=defaultdict(list)
         for r in rows: bychrom[r[0]].append(r)
         for ch,rs in bychrom.items():
             rs.sort(key=lambda r:-r[-1]); acc=[]
             for r in rs:
                 if any(abs(r[1]-k[1])<=BIN and abs(r[2]-k[2])<=BIN for k in acc): continue
                 acc.append(r)
             kept.extend(acc)
 
         symbol=re.sub("[^A-Za-z0-9]","",term)
         m=meta.get(encsr,{})
         organ=pick_organ(m.get("organ_slims",[]),term)
         if organ not in colors: organ="Epithelium"
         rgb=hex2rgb(colors[organ])
         btype=CLASS_LABEL.get(m.get("classification",""),"Unknown")
         ls=m.get("life_stage",[]); life="Unknown" if not ls else (ls[0].capitalize() if len(ls)==1 else "Mixed")
         lifted=(tier==5)
         calls="Lifted from hg19" if lifted else "Arrowhead (%s)"%ASM
         summary=clean(m.get("summary")) or term
 
         bb=os.path.join(BBDIR,symbol+".bb"); tmp=os.path.join(BBDIR,symbol+".bed")
         kept.sort(key=lambda r:(r[0],r[1],r[2]))
         with open(tmp,"w") as o:
             for r in kept: o.write("\t".join([r[0],str(r[1]),str(r[2]),r[3],r[4],r[5],r[6],r[7],r[8]])+"\n")
         rc=subprocess.run(["bedToBigBed","-type=bed4+5","-tab","-as="+ASFILE,tmp,CHROMS,bb],
                           stderr=subprocess.PIPE,text=True)
         os.remove(tmp)
         if rc.returncode!=0: fail.append((term,rc.stderr.strip())); continue
 
         subtracks.append({"symbol":symbol,"term":term,"dterm":dterm,"organ":organ,"rgb":rgb,"encsr":encsr,
             "assay":assay,"btype":btype,"life":life,"calls":calls,
             "summary":summary,"lifted":lifted,"tier":tier,"n":len(kept),
             "on":("on" if term in DEFAULT_ON else "off")})
 
     if fail:
         print("FAILURES (%d):"%len(fail),file=sys.stderr)
         for t,e in fail: print("  ",t,e,file=sys.stderr)
     print("built %d / %d biosamples"%(len(subtracks),len(bybs)))
     print("default-on:",sum(1 for s in subtracks if s["on"]=="on"))
     print("tiers:",Counter(s["tier"] for s in subtracks))
 
     # primaryKey = the representative ENCODE experiment accession (linked via subtrackUrls,
     # like the wgEncodeReg4 reference). _Biosample (readable name) and _Description (the full
     # ENCODE biosample summary) are shown in the metadata table but excluded from faceting.
     cols=["Accession","_Biosample","Organ","Biosample_type","Assay","Life_stage","Calls","_Description"]
     with open(TSV,"w") as fh:
         fh.write("\t".join(cols)+"\n")
         for s in sorted(subtracks,key=lambda s:s["encsr"]):
             fh.write("\t".join([s["encsr"],s["dterm"],s["organ"],s["btype"],s["assay"],
                                 s["life"],s["calls"],s["summary"]])+"\n")
 
     with open(RA,"w") as fh:
         fh.write(
 """track tadsEncode
 parent tads
 priority 2
 compositeTrack faceted
 shortLabel ENCODE TADs
 longLabel ENCODE contact domains (TADs) across %d %s biosamples (Arrowhead/Hi-C)
 type bigBed 4 + 5
 group regulation
 visibility pack
 metaDataUrl %s/tadsEncode_metadata.tsv
 primaryKey Accession
 colorSettingsUrl %s/organ_colors.json
 subtrackUrls Accession=https://www.encodeproject.org/experiments/$$/
 maxCheckboxes 50
 html tadsEncode
 
 """ % (len(subtracks), SPECIES, GBDB, GBDB))
         for s in sorted(subtracks,key=lambda s:(s["on"]!="on", s["term"].lower())):
             short=scap(shortlab(s["term"],22,ENC_ABBR))
             long="ENCODE TADs in %s (%s)"%(s["dterm"],s["encsr"])
-            mouse=("Biosample: %s | TAD lifted from hg19 (no Arrowhead score)"%s["dterm"] if s["lifted"]
-                   else "Biosample: %s | Arrowhead corner score: $cornerScore"%s["dterm"])
+            mouse=("<b>Biosample:</b> %s<br>TAD lifted from hg19 (no Arrowhead score)"%s["dterm"] if s["lifted"]
+                   else "<b>Biosample:</b> %s<br><b>Arrowhead corner score:</b> $cornerScore"%s["dterm"])
             fh.write(
 """    track tadsEncode_%s
     parent tadsEncode %s
     shortLabel %s
     longLabel %s
     type bigBed 4 + 5
     bigDataUrl %s/tadsEncode/%s.bb
     color %s
     visibility dense
     mouseOver %s
 
 """%(s["encsr"],s["on"],short,long,GBDB,s["symbol"],s["rgb"],mouse))
     print("wrote %d subtracks -> %s"%(len(subtracks),RA))
 
 if __name__=="__main__":
     main()