a5a4f89c3230564550036f73c9821617d3ab4bac angie Thu Mar 23 13:28:16 2023 -0700 No idea why I was skipping the first 3 lines of includeRecombinants.tsv rather than just the header line. diff --git src/hg/utils/otto/sarscov2phylo/makeNewMaskedVcf.sh src/hg/utils/otto/sarscov2phylo/makeNewMaskedVcf.sh index 0f948c6..3327343 100755 --- src/hg/utils/otto/sarscov2phylo/makeNewMaskedVcf.sh +++ src/hg/utils/otto/sarscov2phylo/makeNewMaskedVcf.sh @@ -179,31 +179,31 @@ cut -f 1 *.dropoutContam \ | awk -F\| '{ if ($3 == "") { print $1; } else { print $2; } }' \ > dropoutContam.ids # Also exclude sequences with unbelievably low numbers of mutations given sampling dates. zcat $gisaidDir/chunks/nextclade.full.tsv.gz | cut -f 1,10 \ | awk -F\| '{ if ($3 == "") { print $1 "\t" $2; } else { print $2 "\t" $3; } }' \ | $scriptDir/findRefBackfill.pl > gisaid.refBackfill zcat $ncbiDir/nextclade.full.tsv.gz | cut -f 1,10 | sort \ | join -t $'\t' <(cut -f 1,3 $ncbiDir/ncbi_dataset.plusBioSample.tsv | sort) - \ | $scriptDir/findRefBackfill.pl > gb.refBackfill zcat $cogUkDir/nextclade.full.tsv.gz | cut -f 1,10 | sort \ | join -t $'\t' <(cut -d, -f 1,5 $cogUkDir/cog_metadata.csv | tr , $'\t' | sort) - \ | $scriptDir/findRefBackfill.pl > cog.refBackfill cut -f 1 *.refBackfill > refBackfill.ids sort -u ../tooManyEpps.ids ../badBranchSeed.ids dropoutContam.ids refBackfill.ids \ -| grep -vFwf <(tail -n+4 $scriptDir/includeRecombinants.tsv | cut -f 1) \ +| grep -vFwf <(tail -n+2 $scriptDir/includeRecombinants.tsv | cut -f 1) \ > exclude.ids # Get new GenBank sequences with at least $minReal non-N bases. # Exclude seqs in the tree with EPI IDs that that have been mapped in the very latest $epiToPublic. set +o pipefail egrep $'\t''[A-Z][A-Z][0-9]{6}\.[0-9]+' $epiToPublic \ | grep -Fwf prevGisaid - \ | grep -vFwf prevGbAcc \ | cat \ >> prevGbAcc set -o pipefail xzcat $ncbiDir/genbank.fa.xz \ | faSomeRecords -exclude stdin <(cat prevGbAcc exclude.ids) newGenBank.fa faSize -veryDetailed newGenBank.fa \ | tawk '$4 < '$minReal' {print $1;}' \