637a3d134a395adf8eb77c177ff11aa6972146c0
angie
  Fri Aug 28 08:25:27 2026 -0700
Use csvtk grep because we hit a wall with grep -Fwf performance.

diff --git src/hg/utils/otto/sarscov2phylo/makeNewMaskedMaple.sh src/hg/utils/otto/sarscov2phylo/makeNewMaskedMaple.sh
index e5452e2fb42..cbe5280aada 100755
--- src/hg/utils/otto/sarscov2phylo/makeNewMaskedMaple.sh
+++ src/hg/utils/otto/sarscov2phylo/makeNewMaskedMaple.sh
@@ -112,39 +112,39 @@
 
 # Remove duplicates and withdrawn sequences
 if [ -s prevNameToRemove ]; then
     rm -f prevDedup.pb.gz
     $matUtils extract -i $baseProtobuf \
         -p -s <(sort -u prevNameToRemove) \
         -u prevDedupNames \
         -o prevDedup.pb.gz
 else
     ln -sf $baseProtobuf prevDedup.pb.gz
     $matUtils extract -i prevDedup.pb.gz -u prevDedupNames
 fi
 
 function gbAccCogRenaming {
     # pipeline: one INSDC accession per line of stdin, acc to full name if COG-UK on stdout
-    grep -Fwf - $ncbiDir/ncbi_dataset.plusBioSample.tsv \
+    csvtk -t -U -H grep --pattern-file - $ncbiDir/ncbi_dataset.plusBioSample.tsv \
     | grep COG-UK/ \
     | tawk '{ if ($4 != "") { print $1, $4 "/" $6 "/" $3 "|" $1 "|" $3; } else { if ($3 != "") { print $1, $6  "/" $3 "|" $1 "|" $3; } else { print $1, $6 "|" $1 "|?"; } } }' \
     | sed -re 's@COG-UK/@@g; s/United Kingdom://; s/(\/[0-9]{4})(-[0-9]+)*/\1/; s/ //g;'
 }
 
 function gbAccNonCogRenaming {
     # pipeline: one INSDC accession per line of stdin, acc to full name if non-COG-UK on stdout
-    grep -Fwf - $ncbiDir/ncbi_dataset.plusBioSample.tsv \
+    csvtk -t -U -H grep --pattern-file - $ncbiDir/ncbi_dataset.plusBioSample.tsv \
     | grep -v COG-UK/ \
     | cleanGenbank \
     | tawk '{ if ($3 == "") { $3 = "?"; }
               if ($6 != "") { print $1 "\t" $6 "|" $1 "|" $3; }
               else { print $1 "\t" $1 "|" $3; } }' \
     | sed -re 's/ /_/g'
 }
 
 # To update names that have changed and simplify detection of new sequences to add, relate to acc.
 # Strip country and year from COG-UK names to get COG acc.
 awk -F\| '{ if ($3 == "") { print $1 "\t" $0; } else { print $2 "\t" $0; } }' prevDedupNames \
 | subColumn 1 -miss=/dev/null stdin <(cut -f 1,2 $epiToPublic) stdout \
 | sed -re 's@^(England|Northern_?Ireland|Scotland|Wales)/([A-Z]+[_-]?[A-Za-z0-9]+)/[0-9]+@COG:\2@;' \
 | sort \
     > accToPrevDedupName
@@ -155,35 +155,35 @@
 cut -f 1 accToPrevDedupName | grep -E '^COG:' | sed -re 's/^COG://;' > prevCogUk
 cut -f 1 accToPrevDedupName | grep -E '^EPI_ISL_' > prevGisaid
 cut -f 1 accToPrevDedupName | grep -vE '^([A-Z]{2}[0-9]{6}\.[0-9]+|COG:|EPI_ISL_)' > prevCncb
 
 # GenBank renaming has both COG-UK and non-COG-UK versions:
 gbAccCogRenaming < prevGbAcc > accToNewName
 gbAccNonCogRenaming < prevGbAcc >> accToNewName
 # Restore the COG:isolate format for non-GenBank COG-UK sequences:
 zcat $cogUkDir/cog_metadata.csv.gz \
 | grep -Fwf prevCogUk \
 | awk -F, '{print $1 "\t" $1 "|" $5;}' \
 | sed -re 's@^(England|Northern_?Ireland|Scotland|Wales)/([A-Z]+[_-]?[A-Za-z0-9]+)/[0-9]+@COG:\2@;' \
     >> accToNewName
 # GISAID:
 zcat $gisaidDir/metadata_batch_$today.tsv.gz \
-| grep -Fwf prevGisaid \
+| csvtk -t -U grep --pattern-file prevGisaid --fields gisaid_epi_isl \
 | tawk '$3 != "" {print $3 "\t" $1 "|" $3 "|" $5;}' \
     >> accToNewName
 # CNCB:
-grep -Fwf prevCncb $cncbDir/cncb.metadata.tsv \
+csvtk -t -U grep --pattern-file prevCncb --fields "Accession ID" $cncbDir/cncb.metadata.tsv \
 | cleanCncb \
 | sed -re 's/ /_/g;' \
 | tawk '{print $2 "\t" $1 "|" $2 "|" $10;}' \
     >> accToNewName
 join -t$'\t' accToPrevDedupName <(sort accToNewName) \
 | tawk '$2 != $3 {print $2, $3;}' \
     > prevDedupNameToNewName
 
 $matUtils mask -i prevDedup.pb.gz -r prevDedupNameToNewName -o prevRenamed.pb.gz \
     >& renaming.out
 rm renaming.out
 
 # OK, now that the tree names are updated, figure out which seqs are already in there and
 # which need to be added.
 # Add GenBank COG-UK sequences to prevCogUk so we don't add dups.
@@ -221,117 +221,114 @@
 curl -sS $lineageProposalsRecombinants  | tail -n+2 | cut -f 1 \
 | sed -re 's@(England|Northern[ _]?Ireland|Scotland|Wales)/([A-Z0-9_-]+).*@\2@;
            s/.*(EPI_ISL_[0-9]+|[A-Z]{2}[0-9]+{6}(\.[0-9]+)?).*/\1/;' \
     > tmp
 grep -Fwf tmp $epiToPublic | cut -f 2 | grep -E '^[A-Z]{2}[0-9]{6}' > tmp2
 sort -u tmp tmp2 > lpRecombinantIds
 rm tmp tmp2
 sort -u ../tooManyEpps.ids ../badBranchSeed.ids dropoutContam.ids refBackfill.ids \
 | grep -vFwf <(tail -n+2 $scriptDir/includeRecombinants.tsv | cut -f 1) \
 | grep -vFwf lpRecombinantIds \
     > exclude.ids
 
 # Get IDs of new GenBank sequences
 set +o pipefail
 cut -f 1 $ncbiDir/ncbi_dataset.plusBioSample.tsv \
-| grep -vFwf <(cat prevGbAcc exclude.ids) \
-| cat \
+| csvtk -t -U grep --invert --pattern-file <(cat prevGbAcc exclude.ids) \
     > gbNew
 
 # Get IDs of new COG-UK sequences
 zcat $cogUkDir/cog_metadata.csv.gz | cut -d, -f 1 \
 | grep -vFwf <(cat prevCogUk exclude.ids) \
-| cat \
+| tail -n+2 \
     > cogUkNew
 
 # Get IDs of new GISAID sequences
 zcat $gisaidDir/metadata_batch_$today.tsv.gz \
 | cut -f 3 \
-| grep -vFwf <(cat prevGisaid exclude.ids) \
-| cat \
+| csvtk -t -U grep --invert --pattern-file <(cat prevGisaid exclude.ids) \
     > gisaidNew
 
 # Get IDs of new CNCB sequences
 cut -f 2 $cncbDir/cncb.metadata.tsv \
-| grep -vFwf <(cat prevCncb exclude.ids) \
-| cat \
+| grep -v ^EPI_ISL_ \
+| csvtk -t -U grep --invert --pattern-file <(cat prevCncb exclude.ids) \
     > cncbNew
 
 # Now make a renaming that converts accessions back to full name|acc|year names.
 cp /dev/null $renaming
 if [ -s cogUkNew ]; then
     grep -Fwf cogUkNew <(zcat $cogUkDir/cog_metadata.csv.gz) \
     | awk -F, '{print $1 "\t" $1 "|" $5;}' \
         >> $renaming
 fi
 if [ -s gbNew ]; then
     # Special renaming for COG-UK sequences: strip COG-UK/, add back country and year
     cat gbNew \
     | gbAccCogRenaming \
         >> $renaming
     cat gbNew \
     | gbAccNonCogRenaming \
         >> $renaming
 fi
 if [ -s gisaidNew ]; then
     zcat $gisaidDir/metadata_batch_$today.tsv.gz \
-    | grep -Fwf gisaidNew \
+    | csvtk -t -U grep --pattern-file gisaidNew --fields gisaid_epi_isl \
     | tawk '$3 != "" {print $3 "\t" $1 "|" $3 "|" $5;}' \
         >> $renaming
 fi
 if [ -s cncbNew ]; then
     cleanCncb < $cncbDir/cncb.metadata.tsv \
     | sed -re 's/ /_/g;' \
-    | grep -Fwf cncbNew \
+    | csvtk -t -U grep --pattern-file cncbNew --fields Accession_ID \
     | tawk '{print $2 "\t" $1 "|" $2 "|" $10;}' \
         >> $renaming
 fi
 set -o pipefail
 wc -l $renaming
 
 # Make mask.bed from $problematicSitesVcf
 tawk '$7 == "mask" {print $1, $2-1, $2;}' $problematicSitesVcf > mask.bed
 
 # Make MAPLE diff from nextclade TSV output from each source
 # Strip out deletions because usher-sampled treats them as N and can infer substitutions
 # which usually means garbage.
 cp /dev/null new.masked.mpl.gz
 if [ -s gbNew ]; then
-    cat <(zcat $ncbiDir/nextclade.full.tsv.gz | head -1) \
-        <(zcat $ncbiDir/nextclade.full.tsv.gz | grep -Fwf gbNew) \
+    zcat $ncbiDir/nextclade.full.tsv.gz \
+    | csvtk -t grep --pattern-file gbNew --fields seqName \
     | nextcladeToMaple -refLen=29903 -renameOrPrune=$renaming -maskBed=mask.bed -maxSubst=200 \
         -minReal=$minReal stdin stdout \
     | grep -v '^-' \
     | pigz -p 8 \
         >> new.masked.mpl.gz
 fi
 if [ -s cogUkNew ]; then
-    cat <(zcat $cogUkDir/nextclade.full.tsv.gz | head -1) \
-        <(zcat $cogUkDir/nextclade.full.tsv.gz | grep -Fwf cogUkNew) \
+    zcat $cogUkDir/nextclade.full.tsv.gz \
+    | csvtk -t grep --pattern-file cogUkNew --fields seqName \
     | nextcladeToMaple -refLen=29903 -renameOrPrune=$renaming -maskBed=mask.bed -maxSubst=200 \
         -minReal=$minReal stdin stdout \
     | grep -v '^-' \
     | pigz -p 8 \
         >> new.masked.mpl.gz
 fi
 if [ -s gisaidNew ]; then
     # The GISAID nextclade.full.tsv.gz has full names not IDs; strip down to IDs so lack of
     # renaming doesn't lead to pruning.
-    cat <(zcat $gisaidDir/chunks/nextclade.full.tsv.gz | head -1) \
-        <(zcat $gisaidDir/chunks/nextclade.full.tsv.gz \
+    zcat $gisaidDir/chunks/nextclade.full.tsv.gz \
     | sed -re 's/[^\t]+\|(EPI_ISL_[0-9]+)\|[^\t]+/\1/' \
-          | grep -Fwf gisaidNew) \
+    | csvtk -t grep --pattern-file gisaidNew --fields seqName \
     | nextcladeToMaple -refLen=29903 -renameOrPrune=$renaming -maskBed=mask.bed -maxSubst=200 \
         -minReal=$minReal stdin stdout \
     | grep -v '^-' \
     | pigz -p 8 \
         >> new.masked.mpl.gz
 fi
 if [ -s cncbNew ]; then
-    cat <(zcat $cncbDir/nextclade.full.tsv.gz | head -1) \
-        <(zcat $cncbDir/nextclade.full.tsv.gz | grep -Fwf cncbNew) \
+    zcat $cncbDir/nextclade.full.tsv.gz \
+    | csvtk -t grep --pattern-file cncbNew --fields seqName \
     | nextcladeToMaple -refLen=29903 -renameOrPrune=$renaming -maskBed=mask.bed -maxSubst=200 \
         -minReal=$minReal stdin stdout \
     | grep -v '^-' \
     | pigz -p 8 \
         >> new.masked.mpl.gz
 fi