d886333094fd66bac7a872f685c3464a6d667244
max
  Fri Sep 11 09:52:30 2026 -0700
uniprot otto: run miniprot on the cluster, with the RAM and CPUs it actually needs

para's default RAM per job is the node's RAM divided by its CPU count, which for a
16 CPU job is far less than miniprot wants, and without -cpu parasol would pack
more of these onto a node than it has cores for. Both are now passed.

The RAM figure is measured rather than guessed. Peak RSS is linear in genome size
at about 11 GB per Gb of sequence (2.60 GB for 227 Mb, 4.69 GB for 424 Mb, 8.93 GB
for 811 Mb) and is flat in the thread count, because the index is built once and
shared: 2.49 GB at -t 1 against 2.60 GB at -t 16 on the same sequence. So the
reservation is sized off the genome alone, with headroom, and floored at 8g.
Zebrafish comes out at 21g and marmoset at 42g.

Note that -cpu is a scheduling reservation, not a limit: the job still sees every
core on the node. It stops parasol oversubscribing the machine, and pairing it
with miniprot's own -t is what makes the two agree.

Checked with a real one-job batch at -cpu=16 -ram=42g: accepted, ran on a compute
node, and the binary is reachable from there.

refs #38300

diff --git src/hg/utils/otto/uniprot/doUniprot src/hg/utils/otto/uniprot/doUniprot
index aa3ecf69557..7f855ed2169 100755
--- src/hg/utils/otto/uniprot/doUniprot
+++ src/hg/utils/otto/uniprot/doUniprot
@@ -1482,47 +1482,84 @@
 
     ret = {}
     for fname in fnames:
         for row in iterTsvRows(open(fname)):
             for acc in getAllIsoAccs(row):
                 ret[acc] = row
     return ret
 
 def twoBitFname(db):
     " the assembly sequence, wherever this assembly keeps it "
     hubDir = genArkHubDir(db)
     if hubDir is not None:
         return genArkTwoBit(db, hubDir)
     return "/gbdb/{db}/{db}.2bit".format(db=db)
 
+def miniprotRamGb(genomeFa):
+    """ how much RAM to reserve for a miniprot run on this genome, in whole GB.
+    Measured on real assemblies: peak RSS is linear in genome size at about 11 GB per Gb
+    of sequence and is flat in the thread count, because the index is built once and shared
+    (2.49 GB at -t 1 versus 2.60 GB at -t 16 on the same 227 Mb). So this is sized off the
+    sequence only, with headroom, and never below a small floor.
+    """
+    bases = os.path.getsize(genomeFa) # near enough, fasta is one byte per base plus headers
+    gb = int(bases / 1e9 * 14) + 1
+    return max(gb, 8)
+
+def runMiniprotOnCluster(db, genomeFa, protFa, gffName, workDir):
+    """ run one miniprot job on the parasol cluster.
+    It has to be told both numbers: para's default RAM is the node's RAM divided by its CPU
+    count, which for a 16 CPU job is far less than miniprot needs, and without -cpu parasol
+    would pack more of these onto a node than it has cores for.
+    """
+    ram = miniprotRamGb(genomeFa)
+    jobDir = join(workDir, "cluster")
+    if not isdir(jobDir):
+        os.makedirs(jobDir)
+
+    # a wrapper, so the jobList line stays free of the redirection and quoting that
+    # parasol's job parser does not accept
+    jobSh = join(jobDir, "runMiniprot.sh")
+    with open(jobSh, "w") as ofh:
+        ofh.write("#!/bin/sh\nset -e\n")
+        ofh.write("%s -t %d --gff %s %s > $1\n" % (miniprotBin, miniprotThreads, genomeFa, protFa))
+    os.chmod(jobSh, 0o755)
+
+    jobList = join(jobDir, "jobList")
+    with open(jobList, "w") as ofh:
+        ofh.write("%s {check out exists %s}\n" % (jobSh, gffName))
+
+    logging.info("%s: miniprot on the cluster, -cpu=%d -ram=%dg" % (db, miniprotThreads, ram))
+    run("cd %s && para make -cpu=%d -ram=%dg jobList" % (jobDir, miniprotThreads, ram))
+
 def miniprotProteins(fullFaFname, db, mapFname, stats):
     """ align the UniProt proteins straight to the genome with miniprot and write a PSL.
     Used when the assembly has no gene models worth mapping through. This replaced both
     "blat -q=prot" and mapping through Augustus: Augustus is ab initio, so going through it
     stacks its errors on top of ours, and the BLAT protein search is very slow on a big
     genome.
     """
     workDir = mapFname+".miniprot.tmp"
     if not isdir(workDir):
         os.makedirs(workDir)
 
     # miniprot reads fasta, not 2bit
     genomeFa = join(workDir, "genome.fa")
     run(["twoBitToFa", twoBitFname(db), genomeFa])
 
     gffName = join(workDir, "miniprot.gff")
-    run("%s -t %d --gff %s %s > %s" % (miniprotBin, miniprotThreads, genomeFa, fullFaFname, gffName))
+    runMiniprotOnCluster(db, genomeFa, fullFaFname, gffName, workDir)
     os.remove(genomeFa)
 
     # Name each alignment after the UniProt accession it came from. miniprot calls them
     # MP000001 and puts the accession in Target=, and its ##PAF meta lines and capitalised
     # attributes make gff3ToGenePred unhappy, so both are cleaned up here. The ID has to
     # stay unique for GFF3, hence the .N suffix, which comes off again after the
     # conversion so that qName is the bare accession.
     namedName = join(workDir, "named.gff")
     cmd = """awk -F'\\t' -v OFS='\\t' '
       /^##PAF/ {next}
       /^#/     {print; next}
       $3=="mRNA" {
           id=""; acc=""
           if (match($9, /ID=[^;]+/))      id  = substr($9, RSTART+3, RLENGTH-3)
           if (match($9, /Target=[^ ;]+/)) acc = substr($9, RSTART+7, RLENGTH-7)