d97010481bcf72f263f4a6a21010e46bf667c272
angie
  Thu Jan 13 11:45:32 2022 -0800
Shorten time limit for matOptimize cluster run to 2 hours (probably could be 1) because it usually doesn't recognize when it's done.

diff --git src/hg/utils/otto/sarscov2phylo/usherClusterRun.sh src/hg/utils/otto/sarscov2phylo/usherClusterRun.sh
index ae2e336..6eb0965 100755
--- src/hg/utils/otto/sarscov2phylo/usherClusterRun.sh
+++ src/hg/utils/otto/sarscov2phylo/usherClusterRun.sh
@@ -110,49 +110,49 @@
 time $matUtils merge -T 32 --input-mat-1 merged.16.23.pb --input-mat-2 merged.24.31.pb -o merged.16.31.pb &
 wait
 
 time $matUtils merge -T 64 --input-mat-1 merged.0.15.pb --input-mat-2 merged.16.31.pb -o merged.pb
 
 # Mask Delta deletion sites where spurious "mutations" called by sloppy genome assembly pipelines
 # wreak havoc otherwise.
 $scriptDir/maskDelta.sh merged.pb merged.deltaMasked.pb
 
 # Optimize on the cluster too, with Cheng's MPI-parallelized matOptimize
 # Reserve parasol nodes with a "cluster run" of sleep commands
 mkdir -p reserveKuNodes
 cd reserveKuNodes
 jobCount=16
 threadCount=16
-reserveHours=8
+reserveHours=2
 reserveSeconds=$((3600 * $reserveHours))
 cp /dev/null jobList
 for ((i=0;  $i < $jobCount;  i++)); do
     echo "sleep $reserveSeconds" >> jobList
 done
 ssh ku "cd $ottoDir/$today/reserveKuNodes && para create -cpu=$threadCount jobList && para push"
 
 # Wait for all jobs to start (i.e. all nodes to be reserved):
 echo "Waiting for parasol to assign ku nodes"
-while (( $(ssh ku "parasol list jobs | grep $USER" | awk '$2 != "none"' | wc -l) < $jobCount )); do
+while (( $(ssh ku "parasol list jobs | grep $USER | grep sleep" | awk '$2 != "none"' | wc -l) < $jobCount )); do
     sleep 10
 done
 
 # Make hostfile for mpirun
 cd $ottoDir/$today
 ssh ku "parasol list jobs | grep $USER | grep sleep" \
 | awk '{print $2;}' \
 | sort \
 | uniq -c \
 | awk '{print $2, "slots=" '$threadCount'*$1;}' \
     > hostfile
 
 # mpirun matOptimize on first host in hostfile
 headNode=$(head -1 hostfile | awk '{print $1;}')
 radius=8
 ssh $headNode "cd $ottoDir/$today && \
-               $scriptDir/kuRunMatOptimize.sh -T $threadCount -r $radius \
+               $scriptDir/kuRunMatOptimize.sh -T $threadCount -r $radius -M $reserveHours \
                    -S move_log.cluster -i merged.deltaMasked.pb \
                    -o gisaidAndPublic.$today.masked.preTrim.pb \
                    >& matOptimize.cluster.log"
 
 # Release the ku nodes by stopping the parasol batch
-ssh ku "cd $ottoDir/$today/reserveKuNodes && para stop"
+ssh ku "cd $ottoDir/$today/reserveKuNodes && para stop || true"