351ed2fc24a13b86cf2a1fc9695309e2c0da9de5
lrnassar
  Wed Jun 3 09:33:28 2026 -0700
Make otto fetch failures loud and clinGen silent on no-update. refs #37697

Several otto jobs used 'wget -q'/'curl -s' under 'set -e', so a broken download
aborted with no output and cron (which mails on output, not exit code) sent nothing
-- the mode by which mastermind failed silently for months.

- mastermind, malacards: add an ERR trap that echoes a failure message; switch
mastermind's 'curl -s' to 'curl -sf' so an HTTP error fails loudly instead of
writing a challenge page into the release file.
- geneReviews: add an ERR trap (kept silent on no-update via the wrapper's mail -E).
- All three use 'set -eE' (errtrace) so the ERR trap also fires for failures inside
functions/subshells, not just top-level commands.
- clinGen: capture each sub-build's exit status instead of swallowing it under
'set +e', and report which sub-build failed. Also drop the routine 'No ... update'
echoes and the unconditional 'ClinGen update done.' line, and switch the wrapper to
'mail -E', so clinGen is now silent on no-update and only emails on a real update
or a failure.

diff --git src/hg/utils/otto/clinGen/makeCnv.sh src/hg/utils/otto/clinGen/makeCnv.sh
index 6462b6b8f21..29cfc511ff4 100755
--- src/hg/utils/otto/clinGen/makeCnv.sh
+++ src/hg/utils/otto/clinGen/makeCnv.sh
@@ -1,57 +1,55 @@
 #!/bin/bash
 
 # downloads and builds the curated ClinGen Curated CNVS track
 # assumes running in the build directory:
 # /hive/data/outside/otto/clinGen/
 set -beEu -o pipefail
 
 WORKDIR=$1
 mkdir -p ${WORKDIR}/clinGenCnv
 cd ${WORKDIR}/clinGenCnv
 
 echo "user anonymous otto@soe.ucsc.edu
 cd /pub/dbVar/data/Homo_sapiens/by_study/gvf
 ls nstd45*
 bye" > ftp.isca.rsp
 
 if [ -e release.list ]
 then
     mv release.list prev.release.list
 fi
 touch prev.release.list
 rm -f release.list
 
 # connect and list a directory, result to file: ls.check
 ftp -n -v -i ftp.ncbi.nlm.nih.gov 2>&1 < ftp.isca.rsp &> ls.check
 grep "nstd45.*gvf.gz" ls.check | sort > release.list || echo "Error - no gvf files found"
 
 # see if anything is changing, if so, notify, download, and build
 diff prev.release.list release.list > release.diff || true
 count=`wc -l release.diff | cut -d' ' -f1`
 if [ "${count}" -gt 1 ]
 then
     echo "New ClinGen CNV update"
     today=`date +%F`
     mkdir -p ${today}/{download,output}
     cd ${today}/download
     hgsql -Ne 'select 0, ca.alias, size, ca.chrom, size from chromInfo ci join chromAlias ca on ci.chrom = ca.chrom where source = "refseq"' hg19 > hg19.lift
     hgsql -Ne 'select 0, ca.alias, size, ca.chrom, size from chromInfo ci join chromAlias ca on ci.chrom = ca.chrom where source = "refseq"' hg38 > hg38.lift
     for grc in GRCh37 GRCh38
     do
         wget -N -q "ftp://ftp.ncbi.nlm.nih.gov/pub/dbVar/data/Homo_sapiens/by_study/gvf/nstd45.${grc}.variant*.gvf.gz"
         if [ ${grc} == "GRCh37" ]
         then
             zcat nstd45.${grc}.* | ../../../processClinGenCnv.py stdin hg19.lift | sort -k1,1 -k2,2n | bedClip -truncate stdin /hive/data/genomes/hg19/chrom.sizes stdout > ../output/hg19.clinGenCnv.bed
             bedToBigBed -type=bed9+17 -as=../../clinGenCnv.as -tab ../output/hg19.clinGenCnv.bed /hive/data/genomes/hg19/chrom.sizes ../output/hg19.clinGenCnv.bb
             cp ../output/hg19.clinGenCnv.bb ${WORKDIR}/release/hg19/clinGenCnv.bb
         elif [ ${grc} == "GRCh38" ]
         then
             zcat nstd45.${grc}.* | ../../../processClinGenCnv.py stdin hg38.lift | sort -k1,1 -k2,2n | bedClip -truncate stdin /hive/data/genomes/hg38/chrom.sizes stdout  > ../output/hg38.clinGenCnv.bed
             bedToBigBed -type=bed9+17 -as=../../clinGenCnv.as -tab ../output/hg38.clinGenCnv.bed /hive/data/genomes/hg38/chrom.sizes ../output/hg38.clinGenCnv.bb
             cp ../output/hg38.clinGenCnv.bb ${WORKDIR}/release/hg38/clinGenCnv.bb
         fi
     done
     cd ..
-else
-    echo "No ClinGen CNV update"
 fi