7713a08da69691ba499d5b9d44379c43e8cdb608
markd
  Fri Sep 18 09:27:52 2026 -0700
Adding Transcription Start container with ENCODE 4 PRO-cap and ProCapNet tracks. refs #35528

New superTrack transcriptionStart in the rna group, holding two faceted
composites: encode4ProCap with PRO-cap measurements and proCapNet with the
model predictions and sequence-contribution scores.  hg38 has all three data
types, hs1 the predictions only.

ENCODE 4 PRO-cap comes from the portal rather than the submitter's hub copy.
For each of the six experiments only the plus and minus strand signal of unique
reads files of that experiment's default analysis are taken, which drops files
superseded by a later reprocessing.  ENCODE publishes no pooled file, so the
per-replicate files are summed per cell line and strand, the same merge the
ProCapNet models were trained on.  Total signal is conserved exactly.

The published ProCapNet prediction bigWigs store one bedGraph interval per base
and hold a literal NaN at every unresolved (N) base on hg38, which makes
autoScale and every summary statistic NaN.  They are re-encoded into fixedStep
sections, a third smaller with no value changed, dropping 164,268,582 NaN bases
of 3,088,269,832 on hg38 and none on hs1.  Losslessness verified against the
originals on random windows across five chromosomes.

The composites are faceted rather than plain because a container multiWig under
a plain composite is flattened away by hgTrackUi and never drawn.  Each cell
line is one row with a checkbox per data type, a Sample class facet, ENCODE
accession links and a Files column linking each bigWig on hgdownload.

Scripts and the cell line configuration are in makeDb/outside/proCapNet; the
trackDb stanzas and the faceted metadata tables are generated, not hand edited.

Claude-Session: https://claude.ai/code/session_01LAB6jWshLvB7eNXQKWVuW5

diff --git src/hg/makeDb/outside/proCapNet/proCapNetEncodeMeta src/hg/makeDb/outside/proCapNet/proCapNetEncodeMeta
new file mode 100755
index 00000000000..a0769292190
--- /dev/null
+++ src/hg/makeDb/outside/proCapNet/proCapNetEncodeMeta
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""Query the ENCODE portal for the PRO-cap signal bigWigs of each experiment
+and write the file manifest used to build the observed PRO-cap tracks.
+
+Only files belonging to the experiment's default analysis are kept, which drops
+superseded files from older analyses of the same experiment.  Both biological
+and sequencing-run level files are kept; they are summed downstream.
+"""
+import argparse
+import json
+import urllib.request
+from pycbio.sys import cli, fileOps
+from pycbio.tsv import TsvReader, TsvWriter
+
+ENCODE_URL = "https://www.encodeproject.org"
+STRAND_OUTPUT_TYPES = {"plus strand signal of unique reads": "pos",
+                       "minus strand signal of unique reads": "neg"}
+OUT_COLUMNS = ("cell", "encodeAcc", "analysisAcc", "fileAcc", "strand",
+               "bioRep", "techRep", "size", "url")
+
+def parseArgs():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--assembly", default="GRCh38",
+                        help="ENCODE assembly name to select files for")
+    parser.add_argument("experimentsTsv")
+    parser.add_argument("outTsv")
+    return cli.parseOptsArgsWithLogging(parser)
+
+def encodeGet(path):
+    req = urllib.request.Request(ENCODE_URL + path, headers={"accept": "application/json"})
+    with urllib.request.urlopen(req, timeout=120) as fh:
+        return json.load(fh)
+
+def defaultAnalysis(encodeAcc):
+    exp = encodeGet(f"/experiments/{encodeAcc}/?frame=page")
+    analysis = exp.get("default_analysis")
+    if analysis is None:
+        raise cli.PycbioException(f"experiment {encodeAcc} has no default_analysis; "
+                                  "pick the analysis explicitly and extend this script")
+    return analysis
+
+def experimentFiles(encodeAcc, assembly):
+    "signal bigWigs of the default analysis, as (strand, fileJson) pairs"
+    analysis = defaultAnalysis(encodeAcc)
+    found = encodeGet(f"/search/?type=File&dataset=/experiments/{encodeAcc}/"
+                      f"&file_format=bigWig&frame=object&limit=all")["@graph"]
+    selected = [(STRAND_OUTPUT_TYPES[f["output_type"]], f) for f in found
+                if (analysis in (f.get("analyses") or []))
+                and (f["output_type"] in STRAND_OUTPUT_TYPES)
+                and (f.get("assembly") == assembly)
+                and (f["status"] == "released")]
+    if len(selected) == 0:
+        raise cli.PycbioException(f"no released {assembly} signal bigWigs found in analysis "
+                                  f"{analysis} of experiment {encodeAcc}")
+    return analysis, selected
+
+def writeExperiment(tw, exp, assembly):
+    analysis, selected = experimentFiles(exp.procapAcc, assembly)
+    for strand, f in selected:
+        tw.writeColumns(cell=exp.cell, encodeAcc=exp.procapAcc,
+                        analysisAcc=analysis.split("/")[2], fileAcc=f["accession"],
+                        strand=strand, bioRep=",".join(map(str, f["biological_replicates"])),
+                        techRep=",".join(f["technical_replicates"]), size=f["file_size"],
+                        url=ENCODE_URL + f["href"])
+    return len(selected)
+
+def proCapNetEncodeMeta(opts, args):
+    experiments = list(TsvReader(args.experimentsTsv))
+    fileOps.ensureFileDir(args.outTsv)
+    with fileOps.AtomicFileCreate(args.outTsv) as tmpTsv:
+        with TsvWriter(tmpTsv, columns=OUT_COLUMNS) as tw:
+            for exp in experiments:
+                cnt = writeExperiment(tw, exp, opts.assembly)
+                print(f"{exp.cell}\t{exp.procapAcc}\t{cnt} files")
+
+def main():
+    opts, args = parseArgs()
+    with cli.ErrorHandler(noStackExcepts=(OSError, cli.PycbioException)):
+        proCapNetEncodeMeta(opts, args)
+
+main()