d1fb9d605d77792dff66878cf1a8990320861049
lrnassar
  Tue May 12 12:57:38 2026 -0700
Make the OMIM otto fetch surface its wget log on failure and retry patiently.
Previously, "set -e" killed the script after a partial wget failure before the
"if [ $? -ne 0 ]" branch could run, so the build email showed only "fetching
files ..." with no hint that the public omim.org mim2gene.txt URL had returned
403 (Cloudflare). The fetch now retries on 403/408/429/5xx with linear backoff
(up to ~13 min of attempts) and prints wget.log into the email when it still
gives up. No RM.

diff --git src/hg/utils/otto/omim/checkOmim.sh src/hg/utils/otto/omim/checkOmim.sh
index ef8e4d7abad..433d56a6ec3 100755
--- src/hg/utils/otto/omim/checkOmim.sh
+++ src/hg/utils/otto/omim/checkOmim.sh
@@ -1,113 +1,121 @@
 #!/bin/sh -e
 
 #	Do not modify this script, modify the source tree copy:
 #	src/hg/utils/omim/checkOmim.sh
 
 # set EMAIL here for notification list
 #EMAIL="braney@soe.ucsc.edu"
 # set DEBUG_EMAIL here for notification of potential errors in the process
 #DEBUG_EMAIL="braney@soe.ucsc.edu"
 
 #	cron jobs need to ensure this is true
 umask 002
 
 WORKDIR=$1
 export WORKDIR
 export PATH=$WORKDIR":$PATH"
 
 #	this is where we are going to work
 if [ ! -d "${WORKDIR}" ]; then
     echo "ERROR in OMIM release watch, Can not find the directory:     ${WORKDIR}" 
     exit 255
 fi
 
 cd "${WORKDIR}"
 
 
 ######################
 # Checking if there's a new release
 
 today=`date +%F`
 mkdir -p $today
 cd $today
 
 # fetch remote files
+# Patient retries because the public omim.org static endpoint
+# occasionally returns 403 (Cloudflare) at our cron run time.
 echo fetching files ...
-wget -nv -i ../omimUrls.txt -o wget.log
-if [ $? -ne 0 ]; then
-    echo "Potential error in OMIM release fetch,
-check wget.log in ${WORKDIR}/${today}"
+if ! wget -nv \
+        --tries=40 --waitretry=60 --timeout=120 \
+        --retry-on-http-error=403,408,429,500,502,503,504 \
+        -U "UCSC Genome Browser otto OMIM build" \
+        -i ../omimUrls.txt -o wget.log
+then
+    echo "Potential error in OMIM release fetch, check wget.log in ${WORKDIR}/${today}"
+    echo "--- wget.log ---"
+    cat wget.log
+    echo "--- end wget.log ---"
     exit 255
 fi
 
 ls | grep -v wget.log | xargs -I % sh -c "echo -ne '%\t'; (grep -v '^#' % || true) | md5sum | awk '{print \$1}'" | sort > md5sum.txt
 
 if [ ! -e ../prev.md5sum.txt ]
 then
     touch ../prev.md5sum.txt
 fi
 
 diff md5sum.txt ../prev.md5sum.txt > release.diff || true
 
 WC=`cat release.diff | wc -l`
 if [ "${WC}" -gt 0 ]; then
 #####################
 # There's a new release!
     echo -e "New OMIM update noted at:\n" \
 "http://omim.org/\n"`comm -13 ../prev.md5sum.txt md5sum.txt`"/" 
 
 # build the new OMIM track tables for hg18, hg19, hg38
 for db in hg18 hg19 hg38
 do
   echo Running OMIM build for $db
   rm -rf $db
   mkdir -p $db
   cd $db
 
   ln -s ../genemap.txt ./genemap.txt
   ln -s ../genemap2.txt ./genemap2.txt
   ln -s ../allelicVariants.txt ./allelicVariants.txt
   ln -s ../mim2gene.txt ./mim2gene.txt
   #ln -s ../../doOmimPhenotype.pl ./doOmimPhenotype.pl
   ln -s ../../doOmimPhenotypeNew ./doOmimPhenotypeNew
 
   ../../buildOmimTracks.sh $db
   ../../flagOmimGene.py $db > omimGene2.prev.flagged
   ../../validateOmim.sh $db
   cd ..
 
   # install new tables
   for table in `cat ../omim.tables`
   do 
     new=$table"New"
     old=$table"Old"
     hgsqlSwapTables $db $new $table $old -dropTable3
   done
 
   # archive the data
   if [ ! -d ${WORKDIR}/archive/${db} ]; then
     mkdir -p ${WORKDIR}/archive/${db}
   fi
   cd ${WORKDIR}/archive/${db}
   mkdir -p ${today}
   cd ${today}
   printf "This directory contains a backup of the OMIM track data tables built on %s\n" "${today}" > README 
   for i in `cat ${WORKDIR}/omim.tables`
   do
     hgsql --raw -Ne "show create table ${i}" ${db} > ${i}.sql
     hgsql -Ne "select * from ${i}" ${db} | gzip >  ${i}.txt.gz
   done
   cd ${WORKDIR}/${today}
   
 done
 
 rm -f "${WORKDIR}"/prev.md5sum.txt
 cp -p "${WORKDIR}/${today}"/md5sum.txt "${WORKDIR}"/prev.md5sum.txt
 echo "Omim Installed `date`" 
 
 else
     echo "No update `date` "
     cd "${WORKDIR}"
     rm -rf "${today}"
 fi