cf519593dadf5adb63837c46b3e242f3c1ab08c3
angie
  Thu Jan 13 11:43:51 2022 -0800
Fetch from different sources in parallel.  Guard against non-zero return from grep (thx Jonathan!)

diff --git src/hg/utils/otto/sarscov2phylo/updatePublic.sh src/hg/utils/otto/sarscov2phylo/updatePublic.sh
index 6e286c9..7a3e9df 100755
--- src/hg/utils/otto/sarscov2phylo/updatePublic.sh
+++ src/hg/utils/otto/sarscov2phylo/updatePublic.sh
@@ -10,47 +10,49 @@
 }
 
 if [ $# != 1 ]; then
   usage
   exit 1
 fi
 
 problematicSitesVcf=$1
 
 ottoDir=/hive/data/outside/otto/sarscov2phylo
 gisaidDir=/hive/users/angie/gisaid
 
 today=$(date +%F)
 scriptDir=$(dirname "${BASH_SOURCE[0]}")
 
-$scriptDir/gisaidFromChunks.sh
+$scriptDir/gisaidFromChunks.sh &
 
 cogUkDir=$ottoDir/cogUk.$today
 mkdir -p $cogUkDir
-cd $cogUkDir
-time $scriptDir/getCogUk.sh >& getCogUk.log
+cd $cogUkDir && time $scriptDir/getCogUk.sh >& getCogUk.log &
 
 ncbiDir=$ottoDir/ncbi.$today
 mkdir -p $ncbiDir
-cd $ncbiDir
-time $scriptDir/getNcbi.sh >& getNcbi.log
+cd $ncbiDir && time $scriptDir/getNcbi.sh >& getNcbi.log &
+
+wait
 
 time $scriptDir/updateIdMapping.sh \
     $gisaidDir/{metadata_batch_$today.tsv.gz,sequences_batch_$today.fa.xz}
 
 buildDir=$ottoDir/$today
 mkdir -p $buildDir
 cd $buildDir
 
 prevDate=$(date -d yesterday +%F)
 time $scriptDir/updateCombinedTree.sh $prevDate $today $problematicSitesVcf \
     >& updateCombinedTree.log
 
 echo ""
 cat hgPhyloPlace.description.txt
 cat hgPhyloPlace.plusGisaid.description.txt
 
-grep skip annotate.pango annotate.nextclade
-grep 'Could not' annotate.pango annotate.nextclade
+set +o pipefail
+grep skip annotate.pango annotate.nextclade | cat
+grep 'Could not' annotate.pango annotate.nextclade | cat
+set -o pipefail
 
 # Clean up
 nice xz -f new*fa &