7713a08da69691ba499d5b9d44379c43e8cdb608
markd
  Fri Sep 18 09:27:52 2026 -0700
Adding Transcription Start container with ENCODE 4 PRO-cap and ProCapNet tracks. refs #35528

New superTrack transcriptionStart in the rna group, holding two faceted
composites: encode4ProCap with PRO-cap measurements and proCapNet with the
model predictions and sequence-contribution scores.  hg38 has all three data
types, hs1 the predictions only.

ENCODE 4 PRO-cap comes from the portal rather than the submitter's hub copy.
For each of the six experiments only the plus and minus strand signal of unique
reads files of that experiment's default analysis are taken, which drops files
superseded by a later reprocessing.  ENCODE publishes no pooled file, so the
per-replicate files are summed per cell line and strand, the same merge the
ProCapNet models were trained on.  Total signal is conserved exactly.

The published ProCapNet prediction bigWigs store one bedGraph interval per base
and hold a literal NaN at every unresolved (N) base on hg38, which makes
autoScale and every summary statistic NaN.  They are re-encoded into fixedStep
sections, a third smaller with no value changed, dropping 164,268,582 NaN bases
of 3,088,269,832 on hg38 and none on hs1.  Losslessness verified against the
originals on random windows across five chromosomes.

The composites are faceted rather than plain because a container multiWig under
a plain composite is flattened away by hgTrackUi and never drawn.  Each cell
line is one row with a checkbox per data type, a Sample class facet, ENCODE
accession links and a Files column linking each bigWig on hgdownload.

Scripts and the cell line configuration are in makeDb/outside/proCapNet; the
trackDb stanzas and the faceted metadata tables are generated, not hand edited.

Claude-Session: https://claude.ai/code/session_01LAB6jWshLvB7eNXQKWVuW5

diff --git src/hg/makeDb/outside/proCapNet/proCapNetPredToFixedStep src/hg/makeDb/outside/proCapNet/proCapNetPredToFixedStep
new file mode 100755
index 00000000000..3e0a69d86d0
--- /dev/null
+++ src/hg/makeDb/outside/proCapNet/proCapNetPredToFixedStep
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Re-encode a genome-wide ProCapNet prediction bigWig into fixedStep sections.
+
+The published files store one bedGraph-style interval per base, which costs 12
+raw bytes per base where a fixedStep section costs 4.  The hg38 files also store
+a literal NaN at every unresolved (N) reference base, which makes autoScale and
+every summary statistic NaN.  This drops the NaN bases and keeps every other
+value bit-for-bit, since bigWig values are float32 either way.
+"""
+import argparse
+import numpy as np
+import pyBigWig
+from pycbio.sys import cli, fileOps
+
+CHUNK_SIZE = 10000000
+
+def parseArgs():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--chunkSize", type=int, default=CHUNK_SIZE,
+                        help="bases held in memory at a time")
+    parser.add_argument("chromSizes")
+    parser.add_argument("inBigWig")
+    parser.add_argument("outBigWig")
+    return cli.parseOptsArgsWithLogging(parser)
+
+def loadChromSizes(chromSizes):
+    sizes = {}
+    for line in fileOps.iterLines(chromSizes):
+        chrom, size = line.split("\t")[0:2]
+        sizes[chrom] = int(size)
+    return sizes
+
+def checkChroms(inBw, sizes):
+    "input chroms in chrom.sizes order, validating names and sizes"
+    for chrom, size in inBw.chroms().items():
+        if chrom not in sizes:
+            raise cli.PycbioException(f"chromosome {chrom} is not in the chrom.sizes file; "
+                                      "add it or filter it out upstream")
+        if sizes[chrom] != size:
+            raise cli.PycbioException(f"chromosome {chrom} is {size} bases, chrom.sizes says "
+                                      f"{sizes[chrom]}; wrong assembly?")
+    return [c for c in sizes if c in inBw.chroms()]
+
+def dataRuns(values):
+    "(start, end) of maximal runs of non-NaN values, as offsets into values"
+    hasData = ~np.isnan(values)
+    edges = np.diff(np.concatenate(([0], hasData.view(np.int8), [0])))
+    return np.flatnonzero(edges == 1), np.flatnonzero(edges == -1)
+
+def writeChunk(outBw, chrom, chunkStart, values):
+    starts, ends = dataRuns(values)
+    for start, end in zip(starts, ends):
+        outBw.addEntries(chrom, int(chunkStart + start), values=values[start:end],
+                         span=1, step=1)
+    return int((ends - starts).sum())
+
+def convertChrom(inBw, outBw, chrom, size, chunkSize):
+    kept = 0
+    for chunkStart in range(0, size, chunkSize):
+        chunkEnd = min(chunkStart + chunkSize, size)
+        values = inBw.values(chrom, chunkStart, chunkEnd, numpy=True)
+        kept += writeChunk(outBw, chrom, chunkStart, values)
+    return kept
+
+def proCapNetPredToFixedStep(opts, args):
+    sizes = loadChromSizes(args.chromSizes)
+    inBw = pyBigWig.open(args.inBigWig)
+    chroms = checkChroms(inBw, sizes)
+    fileOps.ensureFileDir(args.outBigWig)
+    total = kept = 0
+    with fileOps.AtomicFileCreate(args.outBigWig) as tmpBw:
+        outBw = pyBigWig.open(tmpBw, "w")
+        outBw.addHeader([(c, sizes[c]) for c in chroms], maxZooms=10)
+        for chrom in chroms:
+            total += sizes[chrom]
+            kept += convertChrom(inBw, outBw, chrom, sizes[chrom], opts.chunkSize)
+        outBw.close()
+    inBw.close()
+    print(f"{args.outBigWig}: {len(chroms)} chroms, {total} bases, {kept} with data, "
+          f"{total - kept} dropped as no-data or NaN")
+
+def main():
+    opts, args = parseArgs()
+    with cli.ErrorHandler(noStackExcepts=(OSError, cli.PycbioException)):
+        proCapNetPredToFixedStep(opts, args)
+
+main()