e1df6d7bb91a0f8027129677be73ad720d79449d angie Fri Aug 26 11:38:42 2022 -0700 Update nextstrain URL and use curl --compressed (thanks @tsibley). diff --git src/hg/utils/otto/nextstrainNcov/doUpdate.sh src/hg/utils/otto/nextstrainNcov/doUpdate.sh index 2b59b2c..1afd407 100755 --- src/hg/utils/otto/nextstrainNcov/doUpdate.sh +++ src/hg/utils/otto/nextstrainNcov/doUpdate.sh @@ -1,34 +1,33 @@ #!/bin/bash source ~/.bashrc set -beEu -o pipefail # Do not modify this script, modify the source tree copy: # kent/src/hg/utils/nextstrainNcov/doUpdate.sh ottoDir=/hive/data/outside/otto/nextstrainNcov chromSizes=/hive/data/genomes/wuhCor1/chrom.sizes gbdbDir=/gbdb/wuhCor1/nextstrain -ncovJsonUrl='https://nextstrain.org/charon/getDataset?prefix=/ncov/open/global' +ncovJsonUrl='https://nextstrain.org/charon/getDataset?prefix=/ncov/open/global/6m' cd $ottoDir -# The file is named .json, but is actually gzipped, so gunzip it. rm -f ncov.json -curl -s "$ncovJsonUrl" \ -| gunzip -c > ncov.json +# HT Thomas Sibley: use --compressed flag to send "Accept-Encoding: gzip" and auto-decompress: +curl -sS --compressed "$ncovJsonUrl" > ncov.json curl -s -I "$ncovJsonUrl" \ | grep Last-Mod | sed -re 's/Last-Modified: //; s/\r//;' \ > ncov.json.date latestDate=$(cat ncov.json.date) oldDate=$(cat old.ncov.json.date) if [ $(date -d "$latestDate" +%s) -le $(date -d "$oldDate" +%s) ]; then exit 0 fi today=`date +%F` mkdir -p $today ncovTime=$(date -d "$latestDate" +%F-%H:%M) cp -p ncov.json $today/ncov.$ncovTime.json mv -f ncov.json old.ncov.json mv ncov.json.date old.ncov.json.date