bdf448295c10bee74b6baa773f6733ce33171996 markd Thu Oct 1 20:56:37 2026 -0700 Store the ProCapNet minus strand negated, and fix group autoscale. refs #35528 Two of the three problems Jairo found on the composite. The negate control did not work on ProCapNet. Its minus-strand files held positive values and were flipped at display time with trackDb negateValues. The composite's control sets one shared value, which replaced the per-track setting and sent both strands the same way, with no route back to the default short of a cart reset. encode4ProCap was fine because ENCODE publishes its minus strand negative already. So put the sign in the data: proCapNetPredToFixedStep gains --negate, proCapNetPredBuild passes it for the minus strand, and negateValues is gone from the stanzas. The twelve existing files were rewritten rather than rebuilt, since the downloads had been deleted; each was checked against its original for identical nBasesCovered, a mirrored value range and exactly negated values. The originals are kept in previousPred, about 197 GB, until QA is done. Group autoscale on the composite did nothing, which was my error: the multiWig containers and the contribution leaves each carried autoScale on, and the child setting wins. The wiggle settings now live only on the composite. The third, an empty row when both strands of a cell line are deselected, is #38441 and is not fixed here. Note in both makedocs and in the generator that this arrangement only partly works and is expected to become a superTrack of multiWig overlays: under a composite, hgTrackUi lists leaves rather than containers, so an overlay gets no inline config block and leaves an empty row when deselected. diff --git src/hg/makeDb/outside/proCapNet/proCapNetPredToFixedStep src/hg/makeDb/outside/proCapNet/proCapNetPredToFixedStep index 3e0a69d86d0..1d04677aafa 100755 --- src/hg/makeDb/outside/proCapNet/proCapNetPredToFixedStep +++ src/hg/makeDb/outside/proCapNet/proCapNetPredToFixedStep @@ -1,87 +1,97 @@ #!/usr/bin/env python3 """Re-encode a genome-wide ProCapNet prediction bigWig into fixedStep sections. The published files store one bedGraph-style interval per base, which costs 12 raw bytes per base where a fixedStep section costs 4. The hg38 files also store a literal NaN at every unresolved (N) reference base, which makes autoScale and every summary statistic NaN. This drops the NaN bases and keeps every other value bit-for-bit, since bigWig values are float32 either way. """ import argparse import numpy as np import pyBigWig from pycbio.sys import cli, fileOps CHUNK_SIZE = 10000000 def parseArgs(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--chunkSize", type=int, default=CHUNK_SIZE, help="bases held in memory at a time") + parser.add_argument("--negate", action="store_true", + help="store the values negated, for a minus-strand file that is " + "to be drawn below the baseline. Negating here rather than " + "with the trackDb negateValues setting keeps the sign in the " + "data, so the composite's negate control behaves the same way " + "for these tracks as for ones whose minus strand is already " + "stored negative") parser.add_argument("chromSizes") parser.add_argument("inBigWig") parser.add_argument("outBigWig") return cli.parseOptsArgsWithLogging(parser) def loadChromSizes(chromSizes): sizes = {} for line in fileOps.iterLines(chromSizes): chrom, size = line.split("\t")[0:2] sizes[chrom] = int(size) return sizes def checkChroms(inBw, sizes): "input chroms in chrom.sizes order, validating names and sizes" for chrom, size in inBw.chroms().items(): if chrom not in sizes: raise cli.PycbioException(f"chromosome {chrom} is not in the chrom.sizes file; " "add it or filter it out upstream") if sizes[chrom] != size: raise cli.PycbioException(f"chromosome {chrom} is {size} bases, chrom.sizes says " f"{sizes[chrom]}; wrong assembly?") return [c for c in sizes if c in inBw.chroms()] def dataRuns(values): "(start, end) of maximal runs of non-NaN values, as offsets into values" hasData = ~np.isnan(values) edges = np.diff(np.concatenate(([0], hasData.view(np.int8), [0]))) return np.flatnonzero(edges == 1), np.flatnonzero(edges == -1) -def writeChunk(outBw, chrom, chunkStart, values): +def writeChunk(outBw, chrom, chunkStart, values, negate): starts, ends = dataRuns(values) + if negate: + values = -values for start, end in zip(starts, ends): outBw.addEntries(chrom, int(chunkStart + start), values=values[start:end], span=1, step=1) return int((ends - starts).sum()) -def convertChrom(inBw, outBw, chrom, size, chunkSize): +def convertChrom(inBw, outBw, chrom, size, chunkSize, negate): kept = 0 for chunkStart in range(0, size, chunkSize): chunkEnd = min(chunkStart + chunkSize, size) values = inBw.values(chrom, chunkStart, chunkEnd, numpy=True) - kept += writeChunk(outBw, chrom, chunkStart, values) + kept += writeChunk(outBw, chrom, chunkStart, values, negate) return kept def proCapNetPredToFixedStep(opts, args): sizes = loadChromSizes(args.chromSizes) inBw = pyBigWig.open(args.inBigWig) chroms = checkChroms(inBw, sizes) fileOps.ensureFileDir(args.outBigWig) total = kept = 0 with fileOps.AtomicFileCreate(args.outBigWig) as tmpBw: outBw = pyBigWig.open(tmpBw, "w") outBw.addHeader([(c, sizes[c]) for c in chroms], maxZooms=10) for chrom in chroms: total += sizes[chrom] - kept += convertChrom(inBw, outBw, chrom, sizes[chrom], opts.chunkSize) + kept += convertChrom(inBw, outBw, chrom, sizes[chrom], opts.chunkSize, + opts.negate) outBw.close() inBw.close() print(f"{args.outBigWig}: {len(chroms)} chroms, {total} bases, {kept} with data, " f"{total - kept} dropped as no-data or NaN") def main(): opts, args = parseArgs() with cli.ErrorHandler(noStackExcepts=(OSError, cli.PycbioException)): proCapNetPredToFixedStep(opts, args) main()