3a85cb72b4b01663b80635ce8c7196100627d188
hiram
  Thu Apr 30 23:49:28 2026 -0700
cron jobs expose PATH issues refs #31811

diff --git src/hg/utils/otto/userRequests/ottoRequestWatch.sh src/hg/utils/otto/userRequests/ottoRequestWatch.sh
index 3577e617b90..21a29412ef7 100755
--- src/hg/utils/otto/userRequests/ottoRequestWatch.sh
+++ src/hg/utils/otto/userRequests/ottoRequestWatch.sh
@@ -35,31 +35,31 @@
 # comes along while we're running won't wipe our PID from the file
 # before its flock attempt fails.
 exec 9<>"${lockPath}"
 flock -n 9 || exit 0
 # we own the lock now safe to truncate and write our PID.  ': >file'
 # truncates via a separate FD; FD 9 keeps its position 0 from <>, so
 # the printf below starts writing at the beginning of the empty file.
 : >"${lockPath}"
 printf "%d\n" "$$" >&9
 ##############################################################################
 
 ##############################################################################
 ### errors - set error status in the table
 function setErrorStatus() {
   id="${1}"
-  hgsql -N -e \
+  /cluster/bin/x86_64/hgsql -N -e \
       "UPDATE ottoRequest SET status=7 WHERE id=${id};" hgcentraltest
 }
 ##############################################################################
 
 ##############################################################################
 ### liftOverUrl - build the public download URL for an over.chain.gz file
 ###   args: srcDb dstDb
 ###   GenArk:      https://hgdownload.soe.ucsc.edu/hubs/<3>/<3>/<3>/<3>/<acc>/liftOver/<srcDb>To<DstDb>.over.chain.gz
 ###   UCSC native: https://hgdownload.soe.ucsc.edu/goldenPath/<srcDb>/liftOver/<srcDb>To<DstDb>.over.chain.gz
 ###   DstDb is dstDb with the first letter upper-cased (matches the
 ###   filename convention used in installLinks()).
 ###   verifies the URL with a curl HEAD before printing; returns 1 if
 ###   the URL does not resolve to a 2xx response.
 ##############################################################################
 function liftOverUrl() {
@@ -89,31 +89,31 @@
 
 ##############################################################################
 ### sendNotification - email the requesting user that their alignment is done
 ###   args: reqId subject
 ###   message body is read from stdin
 ###   recipient: email column of ottoRequest table for that reqId
 ###   bcc: chain-file-request-group@ucsc.edu
 ###   envelope sender / Return-Path / bounce: genome-www@soe.ucsc.edu
 ###   returns 0 on success, non-zero on failure
 ##############################################################################
 function sendNotification() {
   local reqId="${1}"
   local subject="${2}"
   local msgBody="${3}"
   local toAddr
-  toAddr="$(hgsql -N -B -e \
+  toAddr="$(/cluster/bin/x86_64/hgsql -N -B -e \
     "SELECT email FROM ottoRequest WHERE id = ${reqId};" hgcentraltest)"
   if [ -z "${toAddr}" ]; then
     printf "ERROR: sendNotification: no email for request %s\n" "${reqId}" 1>&2
     return 1
   fi
   local bcc="chain-file-request-group@ucsc.edu"
   local bounce="genome-www@soe.ucsc.edu"
   # -f sets the envelope sender (becomes Return-Path at delivery and the
   #    bounce address); -t reads recipients from To:/Cc:/Bcc: headers;
   #    -oi prevents a lone "." in body from ending the message
   {
     printf "From: %s\n" "${bounce}"
     printf "To: %s\n" "${toAddr}"
     printf "Bcc: %s\n" "${bcc}"
     printf "Reply-To: %s\n" "${bounce}"
@@ -207,90 +207,90 @@
     ln -s "${overChain}" "${liftOverDir}/${over}"
 
     mkdir -p "${gbdbLiftOverDir}" "${gbdbQuickLiftDir}"
     rm -f "${gbdbLiftOverDir}/${over}"
     rm -f "${gbdbQuickLiftDir}/${quick}.bb"
     rm -f "${gbdbQuickLiftDir}/${quick}.link.bb"
     ln -s "${quickBb}" "${gbdbQuickLiftDir}/${quick}.bb"
     ln -s "${quickLinkBb}" "${gbdbQuickLiftDir}/${quick}.link.bb"
     ln -s "${liftOverDir}/${over}" "${gbdbLiftOverDir}/${over}"
 
     chainPath="${gbdbLiftOverDir}/${over}"
     quickPath="${gbdbQuickLiftDir}/${quick}.bb"
   fi
 
   # register both rows in hgcentraltest
-  if ! hgAddLiftOverChain -minMatch=0.1 -multiple \
+  if ! /cluster/bin/x86_64/hgAddLiftOverChain -minMatch=0.1 -multiple \
       -path="${chainPath}" "${tDb}" "${qDb}"; then
     printf "ERROR: installLinks: hgAddLiftOverChain failed for %s -> %s\n" \
       "${tDb}" "${qDb}" 1>&2
     return 1
   fi
   if ! "${HOME}/kent/src/hg/utils/automation/addQuickLift.py" \
       "${tDb}" "${qDb}" "${quickPath}"; then
     printf "ERROR: installLinks: addQuickLift.py failed for %s -> %s\n" \
       "${tDb}" "${qDb}" 1>&2
     return 1
   fi
   return 0
 }
 ##############################################################################
 
 ############################################################################
 # phase 1: new requests needing alignment setup - status=1 AND buildDir=''
 ############################################################################
 while read -r reqId; do
   printf "# starting alignment for request %s\n" "${reqId}" 1>&2
   if "${scriptDir}/ottoRequestAlign.sh" "${reqId}"; then
     printf "# alignment setup complete for request %s\n" "${reqId}" 1>&2
   else
     printf "# alignment setup FAILED for request %s\n" "${reqId}" 1>&2
     setErrorStatus "${reqId}"
   fi
-done < <(hgsql -N -B -e \
+done < <(/cluster/bin/x86_64/hgsql -N -B -e \
   "SELECT id FROM ottoRequest WHERE status = 1 AND buildDir = '' AND requestType = 'liftOver';" \
   hgcentraltest)
 
 ############################################################################
 # phase 2: in-progress requests needing workflow monitoring
 ############################################################################
 while IFS=$'\t' read -r reqId buildDir; do
   if [ ! -d "${buildDir}" ]; then
     printf "# WARNING: buildDir not found for request %s: %s\n" \
       "${reqId}" "${buildDir}" 1>&2
     continue
   fi
   # takes a while for the galaxy WF to start up, wait for this file to appear
   if [ ! -s "${buildDir}/pendingInvocationId.txt" ]; then
     continue
   fi
   printf "# monitoring request %s: %s\n" "${reqId}" "${buildDir}" 1>&2
   if "${scriptDir}/workflowMonitor.sh" "${reqId}" "${buildDir}"; then
     # workflowMonitor.sh exits 0 both when still running and when complete;
     # check for the success marker to distinguish
     if [ -s "${buildDir}/successInvocationId.txt" ]; then
-      hgsql -N -e \
+      /cluster/bin/x86_64/hgsql -N -e \
         "UPDATE ottoRequest SET status = 4, completeTime = NOW() \
          WHERE id = ${reqId};" hgcentraltest
       printf "# request %s completed successfully\n" "${reqId}" 1>&2
     fi
     # else: still running, will check again next invocation
   else
     printf "# workflow error for request %s\n" "${reqId}" 1>&2
     setErrorStatus "${reqId}"
   fi
-done < <(hgsql -N -B -e \
+done < <(/cluster/bin/x86_64/hgsql -N -B -e \
   "SELECT id, buildDir FROM ottoRequest \
    WHERE status = 2 AND buildDir != '' AND requestType = 'liftOver';" hgcentraltest)
 
 ############################################################################
 # phase 3: check for tracks done, setup symlinks set status=5 to indicate
 #          ready to push
 ############################################################################
 while IFS=$'\t' read -r reqId buildDir; do
   if [ ! -d "${buildDir}" ]; then
     printf "# WARNING: buildDir not found for request %s: %s\n" \
       "${reqId}" "${buildDir}" 1>&2
     continue
   fi
   source <(grep -E '^export (swapDir|targetDb|queryDb)=' "${buildDir}/kegAlign.sh")
   echo $buildDir
@@ -321,33 +321,33 @@
   ln -s "${swapWork}" "${swapData}/lastz.${targetDb}"
   printf "%s\n" "${swapTdb}"
   ${swapTdb} 1>&2
 
   # install liftOver and quickLift symlinks + register in hgcentraltest,
   # for both directions
   if ! installLinks "${targetDb}" "${queryDb}" "${buildDir}"; then
     setErrorStatus "${reqId}"
     continue
   fi
   if ! installLinks "${queryDb}" "${targetDb}" "${swapDir}"; then
     setErrorStatus "${reqId}"
     continue
   fi
 
-  hgsql -N -e \
+  /cluster/bin/x86_64/hgsql -N -e \
       "UPDATE ottoRequest SET status = 5 WHERE id=${reqId};" hgcentraltest
-done < <(hgsql -N -B -e \
+done < <(/cluster/bin/x86_64/hgsql -N -B -e \
   "SELECT id, buildDir FROM ottoRequest \
    WHERE status = 4 AND buildDir != '' AND requestType = 'liftOver';" hgcentraltest)
 
 ############################################################################
 # phase 4: check for push files is complete, send final notification
 #          clean up galaxy workflow
 ############################################################################
 
 while IFS=$'\t' read -r reqId fromDb toDb buildDir; do
   # time to clean up the galaxy history and workflow to release the space
   if [ -s "${buildDir}/successInvocationId.txt" ]; then
     invocationId=$(cut -f2 "${buildDir}/successInvocationId.txt")
     profileJson="${HOME}/.planemo/profiles/vgp/planemo_profile_options.json"
     if "${scriptDir}/galaxyCleanup.py" "${profileJson}" "${invocationId}"; then
       printf "# galaxy cleanup complete for request %s\n" "${reqId}" 1>&2
@@ -360,22 +360,22 @@
     setErrorStatus "${reqId}"
     continue
   fi
   if ! toUrl="$(liftOverUrl "${toDb}" "${fromDb}")"; then
     setErrorStatus "${reqId}"
     continue
   fi
   sendNotification "${reqId}" \
 "from UCSC: liftOverRequest complete: ${fromDb}<->${toDb}" \
 "Your lift over request is complete.  You can access the lift.over files at:
 
   ${fromUrl}
   ${toUrl}
 "
 
-  hgsql -N -e \
+  /cluster/bin/x86_64/hgsql -N -e \
       "UPDATE ottoRequest SET status=8, completeTime=now() WHERE id=${reqId};" hgcentraltest
 
-done < <(hgsql -N -B -e \
+done < <(/cluster/bin/x86_64/hgsql -N -B -e \
   "SELECT id, fromDb, toDb, buildDir FROM ottoRequest \
    WHERE status = 6 AND requestType = 'liftOver';" hgcentraltest)