4d34cfc382f4e45a7119129b2baa6a8645b80542 angie Fri Oct 1 14:21:27 2021 -0700 Max pointed out that 8 threads would be much more reasonable for xz than 50. diff --git src/hg/utils/otto/sarscov2phylo/gisaidFromChunks.sh src/hg/utils/otto/sarscov2phylo/gisaidFromChunks.sh index 60267d1..b29043f 100755 --- src/hg/utils/otto/sarscov2phylo/gisaidFromChunks.sh +++ src/hg/utils/otto/sarscov2phylo/gisaidFromChunks.sh @@ -15,31 +15,31 @@ today=$(date +%F) # Run pangolin and nextclade on any chunks that need it cd /hive/users/angie/gisaid/chunks make nextclade.tsv make -j10 cd /hive/users/angie/gisaid # Glom all the chunks together. # Remove initial "hCoV-19/" and remove spaces a la nextmeta (e.g. "Hong Kong" -> "HongKong"). # Strip single quotes (e.g. "Cote d'Ivoire" --> "CotedIvoire"). # Also remove a stray comma in a name that caused Newick parsing error ("Hungary/US-32533w,/2020"). # Keep the strain|epiId|date "full names". time xzcat chunks/gisaid_epi_isl_*.fa.xz \ | sed -re 's@^>hCo[Vv]-19/+@>@; s/[ '"'"',()]//g; s/\r$//;' \ -| xz -T 50 \ +| xz -T 8 \ > gisaid_fullNames_$today.fa.xz # Make tmp files with a fullName key and various columns that we'll join together. fastaNames gisaid_fullNames_$today.fa.xz \ | awk -F\| -vOFS="\t" '{print $0, $1, $2, $3;}' \ | sort \ > tmp.first3 # Sequence length faSize -detailed <(xzcat gisaid_fullNames_$today.fa.xz) | sort > tmp.lengths # Lineage & clade assignments sort chunks/pangolin.tsv \ > tmp.lineage sort chunks/nextclade.tsv \ > tmp.clade # Countries -- go back to unstripped sequence names: @@ -75,20 +75,20 @@ set -o pipefail zcat $lastRealNextmeta \ | tail -n+2 \ | sort -k 3,3 \ | join -t$'\t' -a 1 -2 3 \ -o 1.2,2.2,1.1,2.4,1.3,2.6,1.7,2.8,2.9,2.10,2.11,2.12,2.13,1.4,2.15,2.16,2.17,1.5,1.6,2.20,2.21,2.22,2.23,2.24,2.25,2.26,2.27 \ tmp.epiToLocalMeta - \ | sort \ >> metadata_batch_$today.tsv wc -l metadata_batch_$today.tsv gzip -f metadata_batch_$today.tsv # Make fasta with strain-name headers a la nextfasta. xzcat gisaid_fullNames_$today.fa.xz \ | sed -re 's/\|.*//' \ -| xz -T 50 \ +| xz -T 8 \ > sequences_batch_$today.fa.xz # Clean up rm tmp.*