d97010481bcf72f263f4a6a21010e46bf667c272 angie Thu Jan 13 11:45:32 2022 -0800 Shorten time limit for matOptimize cluster run to 2 hours (probably could be 1) because it usually doesn't recognize when it's done. diff --git src/hg/utils/otto/sarscov2phylo/usherClusterRun.sh src/hg/utils/otto/sarscov2phylo/usherClusterRun.sh index ae2e336..6eb0965 100755 --- src/hg/utils/otto/sarscov2phylo/usherClusterRun.sh +++ src/hg/utils/otto/sarscov2phylo/usherClusterRun.sh @@ -110,49 +110,49 @@ time $matUtils merge -T 32 --input-mat-1 merged.16.23.pb --input-mat-2 merged.24.31.pb -o merged.16.31.pb & wait time $matUtils merge -T 64 --input-mat-1 merged.0.15.pb --input-mat-2 merged.16.31.pb -o merged.pb # Mask Delta deletion sites where spurious "mutations" called by sloppy genome assembly pipelines # wreak havoc otherwise. $scriptDir/maskDelta.sh merged.pb merged.deltaMasked.pb # Optimize on the cluster too, with Cheng's MPI-parallelized matOptimize # Reserve parasol nodes with a "cluster run" of sleep commands mkdir -p reserveKuNodes cd reserveKuNodes jobCount=16 threadCount=16 -reserveHours=8 +reserveHours=2 reserveSeconds=$((3600 * $reserveHours)) cp /dev/null jobList for ((i=0; $i < $jobCount; i++)); do echo "sleep $reserveSeconds" >> jobList done ssh ku "cd $ottoDir/$today/reserveKuNodes && para create -cpu=$threadCount jobList && para push" # Wait for all jobs to start (i.e. all nodes to be reserved): echo "Waiting for parasol to assign ku nodes" -while (( $(ssh ku "parasol list jobs | grep $USER" | awk '$2 != "none"' | wc -l) < $jobCount )); do +while (( $(ssh ku "parasol list jobs | grep $USER | grep sleep" | awk '$2 != "none"' | wc -l) < $jobCount )); do sleep 10 done # Make hostfile for mpirun cd $ottoDir/$today ssh ku "parasol list jobs | grep $USER | grep sleep" \ | awk '{print $2;}' \ | sort \ | uniq -c \ | awk '{print $2, "slots=" '$threadCount'*$1;}' \ > hostfile # mpirun matOptimize on first host in hostfile headNode=$(head -1 hostfile | awk '{print $1;}') radius=8 ssh $headNode "cd $ottoDir/$today && \ - $scriptDir/kuRunMatOptimize.sh -T $threadCount -r $radius \ + $scriptDir/kuRunMatOptimize.sh -T $threadCount -r $radius -M $reserveHours \ -S move_log.cluster -i merged.deltaMasked.pb \ -o gisaidAndPublic.$today.masked.preTrim.pb \ >& matOptimize.cluster.log" # Release the ku nodes by stopping the parasol batch -ssh ku "cd $ottoDir/$today/reserveKuNodes && para stop" +ssh ku "cd $ottoDir/$today/reserveKuNodes && para stop || true"