75e828960283291546d2c1a27845e2cf3823adcd max Mon Sep 14 05:29:02 2026 -0700 uniprot otto: the miniprot cluster job needs absolute paths GRCz12ab failed with the parasol job crashing four times, return 1, no output. The wrapper I wrote ran miniprot -t 16 --gff protToGenome/GRCz12ab/.../genome.fa fasta/7955.fa > $1 and a parasol job runs with its working directory set to the batch directory, not to the directory the pipeline runs in, so neither input existed from the job's point of view. The BLAST batch next door gets away with relative paths because it cds into its own workdir and its jobList is written relative to that; this batch directory sits a level deeper and its paths were relative to the otto root. Every path in the wrapper, the jobList command and the output check is now absolute. Verified on the cluster against the real 1.48 Gb zebrafish genome: successful batch, a 195 MB GFF with 93518 mRNA records. refs #38300 diff --git src/hg/makeDb/scripts/fiberSeq/fiberSeqTrackDb.py src/hg/makeDb/scripts/fiberSeq/fiberSeqTrackDb.py index f71e5574b7b..978de73183c 100755 --- src/hg/makeDb/scripts/fiberSeq/fiberSeqTrackDb.py +++ src/hg/makeDb/scripts/fiberSeq/fiberSeqTrackDb.py @@ -32,30 +32,38 @@ DEFAULT_OVERLAY = [ ("PM00001", "230,159,0"), # GM12878, orange ("PM00004", "86,180,233"), # K562, sky blue ("PM00005", "0,158,115"), # HepG2, bluish green ("PM00010", "240,228,66"), # H1, yellow ("PM00008", "0,114,178"), # Hap1, blue ("PM00012", "213,94,0"), # Hek293T, vermillion ("PM00009", "204,121,167"), # Jurkat, reddish purple ] # Samples selected on a first visit. A clean cross-product with the three # default data types, so the facet table comes up as a tidy grid rather than the # ragged per-sample mix the source hub had. DEFAULT_SELECTED = ["PM00001", "PM00004", "PM00005"] +# Nucleosome density is mirrored for all 41 samples but is not shown. The lab +# asked us to hold it back on 2026-09-14, while they settle internally on how +# they want nucleosomes displayed, and to keep the track set on FIRE peaks and +# CpG methylation for now. The files stay in place and the download and check +# scripts still fetch and verify them, so turning this back on is this one flag +# plus a regenerate; nothing has to be downloaded again. +INCLUDE_NUC = False + HAP1_COLOR = "0,114,178" # Okabe-Ito blue HAP2_COLOR = "213,94,0" # Okabe-Ito vermillion # CpG haplotype-difference thresholds: file suffix, label, color. A sequential # yellow-to-red ramp over nested significance cutoffs, monotonic in lightness. DIFF_LEVELS = [ ("cpg.diffs_all.bw", "All", "137,143,143"), ("cpg.diffs_p0.01.bw", "p < 0.01", "235,229,52"), ("cpg.diffs_p0.001.bw", "p < 0.001", "245,148,22"), ("cpg.diffs_p0.0001.bw", "p < 0.0001", "255,0,0"), ] # Shown behind an info icon on the Sample class column heading. SAMPLE_CLASS_DESCRIPTION = ( "HPRC = Lymphoblastoid (B-lymphocyte, EBV) cell lines from the NHGRI " @@ -186,199 +194,212 @@ s = byAcc[acc] out += stanza(8, [ "track fiberSeqAcc_%s" % acc, "parent fiberSeqAcc", "type bigWig", "bigDataUrl %s/%s/all.percent.accessible.bw" % (gbdb, acc), "color %s" % color, "shortLabel %s" % s["sample"], "longLabel %s Fiber-seq percent-accessible chromatin, both haplotypes" % s["sample"], ]) return out def compendium(gbdb, dataUrlDir, samples): - """One faceted composite over all seven per-sample data types. + """One faceted composite over the per-sample data types. Accessibility and CpG methylation are read off the same molecules in the same experiment, so they belong in one table: the user picks a sample once, and cartDump.c assigns priority with the data element as the outer loop and - the data type as the inner one, which keeps a sample's seven subtracks + the data type as the inner one, which keeps a sample's subtracks contiguous in the image. Split across two composites they would draw as an accessibility block followed by a methylation block, and comparing the two for one sample would mean reading across every other sample. """ out = stanza(4, [ "track fiberSeqCompendium", "parent fiberSeq", "compositeTrack faceted", "type bigWig", "shortLabel Fiber-seq Compendium", - "longLabel Fiber-seq accessibility, peaks, nucleosomes and CpG " - "methylation in %d samples" % len(samples), + # Two literal wordings rather than one with a slot, to keep both under + # the 80 characters a longLabel should stay within. + ("longLabel Fiber-seq accessibility, peaks, nucleosomes and CpG " + "methylation in %d samples" % len(samples)) if INCLUDE_NUC else + ("longLabel Fiber-seq accessibility, FIRE peaks and CpG methylation " + "in %d samples" % len(samples)), "metaDataUrl %s/fiberSeqCompendium_metadata.tsv" % dataUrlDir, "colorSettingsUrl %s/fiberSeqCompendium_colors.json" % dataUrlDir, "primaryKey Accession", # Declared order sets the order of the data type checkboxes, and of the # subtracks within each sample. No data type name may contain an # underscore: hgTrackUi globs "<composite>_*_<dataType>_sel". 'dataTypes acc|"Percent accessible" peaks|"FIRE peaks" ' - 'hap|"Haplotype accessibility" nuc|"Nucleosome density" ' - 'cpg|"CpG methylation" ' + 'hap|"Haplotype accessibility" ' + + ('nuc|"Nucleosome density" ' if INCLUDE_NUC else "") + + 'cpg|"CpG methylation" ' 'cpgHap|"Haplotype CpG" cpgDiff|"CpG haplotype difference"', "defaultSortField Accession", "defaultGroupBy sample", "maxCheckboxes 50", "noInherit on", "visibility hide", "priority 2", ]) for i, s in enumerate(samples): acc, name = s["accession"], s["sample"] # A clean cross-product on a first visit: three samples, and the three # data types that answer the question the merge is for, accessibility # next to methylation. on = "on" if acc in DEFAULT_SELECTED else "off" # Without an explicit priority the subtracks fall back to a label sort, # which on a first visit puts a sample's data types in an arbitrary # order (Peaks, CpG, Acc). Sample order outer and declared data type # order inner matches the row of checkboxes across the top of the table. # cartDump.c replaces these with its own priorities as soon as the user - # submits a selection, so this only sets the starting order. - pri = lambda j: "priority %d" % (i * 10 + j + 1) + # submits a selection, so this only sets the starting order. A running + # counter rather than a fixed index per data type, so the numbering stays + # contiguous whether or not INCLUDE_NUC adds one in the middle. + priN = [0] + def pri(): + priN[0] += 1 + return "priority %d" % (i * 10 + priN[0]) out += stanza(8, [ "track fiberSeqCompendium_%s_acc" % acc, "parent fiberSeqCompendium %s" % on, "type bigWig", "bigDataUrl %s/%s/all.percent.accessible.bw" % (gbdb, acc), "shortLabel %s Acc" % name, "longLabel %s Fiber-seq percent accessible, both haplotypes" % name, "color 0,0,0", "viewLimits 0:100", "autoScale off", "alwaysZero on", "graphTypeDefault bar", "windowingFunction maximum", "maxHeightPixels 100:40:8", "onlyVisibility full", - pri(0), + pri(), ]) # bigNarrowPeak, so the point-source offset in the tenth column is drawn # as a tick inside the peak (lfFromEncodePeak() sets tallStart/tallEnd # from it). Its filters are the ENCODE peak settings, signalFilter and # qValueFilter, which encodePeakCfgUi() already knows how to draw; the # bigBed-generic filter.<field> settings are not read by this type. # Both defaults are the full range, so nothing is hidden until the user # narrows it. signalValue tops out near 90 and the FIRE pipeline caps # qValue at 100. pValue is -1 throughout, so no filter is offered. out += stanza(8, [ "track fiberSeqCompendium_%s_peaks" % acc, "parent fiberSeqCompendium %s" % on, "type bigNarrowPeak", "bigDataUrl %s/%s/fire-peaks.ucsc.bb" % (gbdb, acc), "shortLabel %s Peaks" % name, "longLabel %s Fiber-seq FIRE peaks" % name, "signalFilter 0", "signalFilterLimits 0:100", "qValueFilter 0", "qValueFilterLimits 0:100", "scoreFilter 0", "scoreFilterLimits 0:1000", "mouseOver <b>%s FIRE peak</b><br>FIRE score: ${signalValue}" "<br>-log10 FDR: ${qValue}<br>Score: ${score}" % name, "onlyVisibility dense", - pri(1), + pri(), ]) out += hapOverlay(gbdb, acc, name, "hap", "hap1.percent.accessible.bw", "hap2.percent.accessible.bw", "%s Hap1/2" % name, "%s Fiber-seq percent accessible, haplotype 1 (blue) " "and 2 (orange)" % name, "%s Fiber-seq percent accessible, haplotype" % name, - "maximum", pri(2)) - - # Nucleosome density is read depth, not a percentage, so unlike every - # other wiggle here it cannot have fixed viewLimits: the per-sample mean - # runs from 25 (PS00971) to 142 (GM12878) purely with sequencing depth, - # and single loci spike into the hundred thousands. autoScale per - # window is the only setting that shows all 41 samples usefully, and - # absolute values are not comparable between samples anyway. + "maximum", pri()) + + # Held back, see INCLUDE_NUC. Nucleosome density is read depth, not a + # percentage, so unlike every other wiggle here it cannot have fixed + # viewLimits: the per-sample mean runs from 25 (PS00971) to 142 + # (GM12878) purely with sequencing depth, and single loci spike into the + # hundred thousands. autoScale per window is the only setting that + # shows all 41 samples usefully, and absolute values are not comparable + # between samples anyway. + if INCLUDE_NUC: out += stanza(8, [ "track fiberSeqCompendium_%s_nuc" % acc, "parent fiberSeqCompendium off", "type bigWig", "bigDataUrl %s/%s/all.nucleosome.coverage.bw" % (gbdb, acc), "shortLabel %s Nuc" % name, "longLabel %s Fiber-seq nucleosome density, both haplotypes" % name, "color 0,158,115", "autoScale on", "alwaysZero on", "windowingFunction mean", "maxHeightPixels 100:40:8", "onlyVisibility full", - pri(3), + pri(), ]) out += stanza(8, [ "track fiberSeqCompendium_%s_cpg" % acc, "parent fiberSeqCompendium %s" % on, "type bigWig 0 100", "bigDataUrl %s/%s/cpg.combined.bw" % (gbdb, acc), "shortLabel %s CpG" % name, "longLabel %s CpG methylation, both haplotypes" % name, "color 0,0,0", "viewLimits 0:100", "autoScale off", "windowingFunction mean", "maxHeightPixels 100:40:8", "onlyVisibility full", - pri(4), + pri(), ]) out += hapOverlay(gbdb, acc, name, "cpgHap", "cpg.hap1.bw", "cpg.hap2.bw", "%s CpG Hap1/2" % name, "%s CpG methylation, haplotype 1 (blue) and 2 (orange)" % name, "%s CpG methylation, haplotype" % name, - "mean", pri(5)) + "mean", pri()) out += stanza(8, [ "track fiberSeqCompendium_%s_cpgDiff" % acc, "parent fiberSeqCompendium off", "container multiWig", "aggregate solidOverlay", "showSubtrackColorOnUi on", "type bigWig -100 100", "viewLimits -100:100", "autoScale off", "windowingFunction mean", "maxHeightPixels 100:50:8", "shortLabel %s CpG diff" % name, "longLabel %s CpG methylation difference between haplotypes, " "by significance threshold" % name, "onlyVisibility full", - pri(6), + pri(), ]) # Least significant first, so the more significant levels draw on top. - for i, (fname, label, color) in enumerate(DIFF_LEVELS): + # Not "i": that is the sample index pri() builds its priority from. + for level, (fname, label, color) in enumerate(DIFF_LEVELS): out += stanza(12, [ - "track fiberSeqCompendium_%s_cpgDiff_l%d" % (acc, i), + "track fiberSeqCompendium_%s_cpgDiff_l%d" % (acc, level), "parent fiberSeqCompendium_%s_cpgDiff" % acc, "type bigWig", "bigDataUrl %s/%s/%s" % (gbdb, acc, fname), "color %s" % color, "shortLabel %s %s" % (name, label), "longLabel %s CpG haplotype difference, %s" % (name, label), ]) return out def hapOverlay(gbdb, acc, name, dataType, file1, file2, shortLabel, longLabel, childLongLabel, windowing, priority): """A haplotype 1 / haplotype 2 transparent overlay, used for both the accessibility and the CpG haplotype data types.""" out = stanza(8, [ @@ -437,25 +458,25 @@ f.write("# Fiber-seq: chromatin accessibility, FIRE regulatory elements and CpG\n" "# methylation from PacBio HiFi Fiber-seq, Stergachis and Vollger labs.\n" "# Generated by hg/makeDb/scripts/fiberSeq/fiberSeqTrackDb.py.\n" "# Do not edit by hand, edit the script and regenerate.\n\n") f.write(stanza(0, [ "track fiberSeq", "superTrack on show", "shortLabel Fiber-seq", "longLabel Fiber-seq chromatin accessibility, regulatory elements and CpG methylation", "group regulation", "priority 2.5", ])) f.write(accOverlay(args.gbdb_dir, samples)) f.write(compendium(args.gbdb_dir, args.gbdb_dir, samples)) - # Per sample: acc, peaks, nuc, cpg, three container stanzas (hap, cpgHap, - # cpgDiff) and their 2 + 2 + 4 children. - nSub = len(DEFAULT_OVERLAY) + len(samples) * (4 + 3 + 8) + # Per sample: acc, peaks, cpg (plus nuc when enabled), three container + # stanzas (hap, cpgHap, cpgDiff) and their 2 + 2 + 4 children. + nSub = len(DEFAULT_OVERLAY) + len(samples) * (3 + int(INCLUDE_NUC) + 3 + 8) print("wrote %s" % raPath) print(" %d samples, %d track stanzas" % (len(samples), nSub + 3)) print(" metadata and colors in %s" % args.data_dir) if __name__ == "__main__": main()