8b26d7af3e7e4915eab698dcc6485ebccc87ddab hiram Mon Jun 2 15:56:23 2025 -0700 hgdownload2 disk is full send no more data to hgdownload2 refs #35415 diff --git src/hg/makeDb/doc/asmHubs/asmHubs.mk src/hg/makeDb/doc/asmHubs/asmHubs.mk index 7be65fb43c6..17223b1576c 100644 --- src/hg/makeDb/doc/asmHubs/asmHubs.mk +++ src/hg/makeDb/doc/asmHubs/asmHubs.mk @@ -29,31 +29,30 @@ makeDirs: mkdir -p ${destDir} sanityCheck: @goodBad=$$(cut -d'_' -f1-2 ${orderList} | sort | uniq -c | awk '$$1 > 1' | wc -l); \ if [ $$goodBad -ne 0 ]; then \ tsvFile=$$(basename ${orderList}); \ echo "ERROR: duplicate accession in '$$tsvFile'"; \ cut -d'_' -f1-2 ${orderList} | sort | uniq -c | awk '$$1 > 1'; \ exit 255; \ fi sshKeyDownload: ssh -o PasswordAuthentication=no qateam@${downloadDest1} date - ssh -o PasswordAuthentication=no qateam@${downloadDest2} date ssh -o PasswordAuthentication=no qateam@${downloadDest3} date sshKeyDynablat: ssh -o PasswordAuthentication=no qateam@dynablat-01 date sshKeyCheck: sshKeyDownload sshKeyDynablat @printf "# ssh keys to hgdownload and dynablat-01 are good\n" mkJson:: if [ "$(name)" = "VGP" ]; then \ sort -u *.orderList.tsv | ${toolsDir}/tsvToJson.py stdin > ${destDir}/assemblyList.json 2> ${name}.jsonData.txt; \ else \ ${toolsDir}/tsvToJson.py ${orderList} > ${destDir}/assemblyList.json 2> ${name}.jsonData.txt; \ fi @@ -135,125 +134,92 @@ rm -f ${destDir}/${testHubFile}.txt rm -f ${destDir}/groups.txt rm -f ${destDir}/${genomesTxt}.txt rm -f ${destDir}/${indexName}.html rm -f ${destDir}/${testIndexName}.html rm -f ${destDir}/${statsName}.html rm -f ${destDir}/${testStatsName}.html sendDownload:: sshKeyCheck ${toolsDir}/mkSendList.pl ${orderList} | while read F; do \ ((N=N+1)); printf "### count %5d\t%s\t%s\n" $${N} $${F} "`date '+%F %T %s'`"; ${toolsDir}/sendToHgdownload.sh $$F < /dev/null; done rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/assemblyList.json \ qateam@${downloadDest1}:/mirrordata/hubs/${name}/ rsync -L -a -P \ - /usr/local/apache/htdocs-hgdownload/hubs/${name}/assemblyList.json \ - qateam@${downloadDest2}:/mirrordata/hubs/${name}/ - rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/assemblyList.json \ qateam@${downloadDest3}:/mirrordata/hubs/${name}/ rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/groups.txt \ qateam@${downloadDest1}:/mirrordata/hubs/${name}/ rsync -L -a -P \ - /usr/local/apache/htdocs-hgdownload/hubs/${name}/groups.txt \ - qateam@${downloadDest2}:/mirrordata/hubs/${name}/ - rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/groups.txt \ qateam@${downloadDest3}:/mirrordata/hubs/${name}/ rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/${hubFile}.txt \ qateam@${downloadDest1}:/mirrordata/hubs/${name}/ rsync -L -a -P \ - /usr/local/apache/htdocs-hgdownload/hubs/${name}/${hubFile}.txt \ - qateam@${downloadDest2}:/mirrordata/hubs/${name}/ - rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/${hubFile}.txt \ qateam@${downloadDest3}:/mirrordata/hubs/${name}/ rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${indexName}.html \ qateam@${downloadDest1}:/mirrordata/hubs/${name}/${indexName}.html rsync -L -a -P \ - /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${indexName}.html \ - qateam@${downloadDest2}:/mirrordata/hubs/${name}/${indexName}.html - rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${indexName}.html \ qateam@${downloadDest3}:/mirrordata/hubs/${name}/${indexName}.html rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${statsName}.html \ qateam@${downloadDest1}:/mirrordata/hubs/${name}/${statsName}.html rsync -L -a -P \ - /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${statsName}.html \ - qateam@${downloadDest2}:/mirrordata/hubs/${name}/${statsName}.html - rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${statsName}.html \ qateam@${downloadDest3}:/mirrordata/hubs/${name}/${statsName}.html rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${dataName}.html \ qateam@${downloadDest1}:/mirrordata/hubs/${name}/${dataName}.html rsync -L -a -P \ - /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${dataName}.html \ - qateam@${downloadDest2}:/mirrordata/hubs/${name}/${dataName}.html - rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${dataName}.html \ qateam@${downloadDest3}:/mirrordata/hubs/${name}/${dataName}.html # no longer sending genomes.txt file 2024-10-23 - becomes static obsolete: rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${genomesTxt}.txt \ qateam@${downloadDest1}:/mirrordata/hubs/${name}/${genomesTxt}.txt rsync -L -a -P \ - /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${genomesTxt}.txt \ - qateam@${downloadDest2}:/mirrordata/hubs/${name}/${genomesTxt}.txt - rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${genomesTxt}.txt \ qateam@${downloadDest3}:/mirrordata/hubs/${name}/${genomesTxt}.txt verifyTestDownload: ${toolsDir}/verifyOnDownload.sh api-test.gi.ucsc.edu ${orderList} verifyDownload: ${toolsDir}/verifyOnDownload.sh apibeta.soe.ucsc.edu ${orderList} verifyDynamicBlat: grep -v "^#" ${orderList} | cut -d'_' -f1-2 | while read asmId; do \ ${toolsDir}/testDynBlat.sh $$asmId < /dev/null; done sendIndexes:: rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/assemblyList.json \ qateam@${downloadDest1}:/mirrordata/hubs/${name}/ rsync -L -a -P \ - /usr/local/apache/htdocs-hgdownload/hubs/${name}/assemblyList.json \ - qateam@${downloadDest2}:/mirrordata/hubs/${name}/ - rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/assemblyList.json \ qateam@${downloadDest3}:/mirrordata/hubs/${name}/ rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${indexName}.html \ qateam@${downloadDest1}:/mirrordata/hubs/${name}/${indexName}.html rsync -L -a -P \ - /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${indexName}.html \ - qateam@${downloadDest2}:/mirrordata/hubs/${name}/${indexName}.html - rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${indexName}.html \ qateam@${downloadDest3}:/mirrordata/hubs/${name}/${indexName}.html rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${statsName}.html \ qateam@${downloadDest1}:/mirrordata/hubs/${name}/${statsName}.html rsync -L -a -P \ - /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${statsName}.html \ - qateam@${downloadDest2}:/mirrordata/hubs/${name}/${statsName}.html - rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${statsName}.html \ qateam@${downloadDest3}:/mirrordata/hubs/${name}/${statsName}.html rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${dataName}.html \ qateam@${downloadDest1}:/mirrordata/hubs/${name}/${dataName}.html rsync -L -a -P \ - /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${dataName}.html \ - qateam@${downloadDest2}:/mirrordata/hubs/${name}/${dataName}.html - rsync -L -a -P \ /usr/local/apache/htdocs-hgdownload/hubs/${name}/download.${dataName}.html \ qateam@${downloadDest3}:/mirrordata/hubs/${name}/${dataName}.html