476ebb957aebb1a7f20ea71301e894c5422bd57e
hiram
  Wed Apr 29 21:41:01 2026 -0700
subtle timing issue resolved and minor bugs fixed refs #31811

diff --git src/hg/utils/otto/userRequests/ottoRequestWatch.sh src/hg/utils/otto/userRequests/ottoRequestWatch.sh
index ba3925d229b..f8e63f2bd08 100755
--- src/hg/utils/otto/userRequests/ottoRequestWatch.sh
+++ src/hg/utils/otto/userRequests/ottoRequestWatch.sh
@@ -4,30 +4,31 @@
 #
 # Intended to run from cron under the user's own account (not the
 # web-server service user).  Picks up requests that ottoRequest.py has
 # acknowledged (status=1) and drives them through alignment setup
 # and workflow monitoring.
 #
 # Phase 1: new requests needing alignment setup - status=1 AND buildDir=''
 #          run ottoRequestAlign.sh to set up and launch the workflow
 # Phase 2: in-progress requests needing workflow monitoring
 #          run workflowMonitor.sh to poll Galaxy and install results
 #   0 pending, 1 notified, 2 in progress, 3 galaxy done, 4 tracks complete,
 #   5 ready to push, 6 push is done, 7 problems,
 #      8 final notification has been sent == process is complete
 
 set -eEu -o pipefail
+set -x
 
 export scriptDir=$(cd "$(dirname "$0")" && pwd)
 
 ##############################################################################
 ### errors - set error status in the table
 function setErrorStatus() {
   id="${1}"
   hgsql -N -e \
       "UPDATE ottoRequest SET status=7 WHERE id=${id};" hgcentraltest
 }
 ##############################################################################
 
 ##############################################################################
 ### sendNotification - email the requesting user that their alignment is done
 ###   args: reqId subject
@@ -188,30 +189,34 @@
     setErrorStatus "${reqId}"
   fi
 done < <(hgsql -N -B -e \
   "SELECT id FROM ottoRequest WHERE status = 1 AND buildDir = '' AND requestType = 'liftOver';" \
   hgcentraltest)
 
 ############################################################################
 # phase 2: in-progress requests needing workflow monitoring
 ############################################################################
 while IFS=$'\t' read -r reqId buildDir; do
   if [ ! -d "${buildDir}" ]; then
     printf "# WARNING: buildDir not found for request %s: %s\n" \
       "${reqId}" "${buildDir}" 1>&2
     continue
   fi
+  # takes a while for the galaxy WF to start up, wait for this file to appear
+  if [ ! -s "${buildDir}/pendingInvocationId.txt" ]; then
+    continue
+  fi
   printf "# monitoring request %s: %s\n" "${reqId}" "${buildDir}" 1>&2
   if "${scriptDir}/workflowMonitor.sh" "${reqId}" "${buildDir}"; then
     # workflowMonitor.sh exits 0 both when still running and when complete;
     # check for the success marker to distinguish
     if [ -s "${buildDir}/successInvocationId.txt" ]; then
       hgsql -N -e \
         "UPDATE ottoRequest SET status = 4, completeTime = NOW() \
          WHERE id = ${reqId};" hgcentraltest
       printf "# request %s completed successfully\n" "${reqId}" 1>&2
     fi
     # else: still running, will check again next invocation
   else
     printf "# workflow error for request %s\n" "${reqId}" 1>&2
     setErrorStatus "${reqId}"
   fi