430bb5b5b7601fa51ed1872e5c7a55def547c450
hiram
  Wed Apr 29 14:27:34 2026 -0700
refactor the workflowMonitor to get json operations into galaxyState.py and finalizing state diagram refs #31811

diff --git src/hg/utils/otto/userRequests/ottoRequestWatch.sh src/hg/utils/otto/userRequests/ottoRequestWatch.sh
index 70161329403..ba3925d229b 100755
--- src/hg/utils/otto/userRequests/ottoRequestWatch.sh
+++ src/hg/utils/otto/userRequests/ottoRequestWatch.sh
@@ -1,30 +1,31 @@
 #!/bin/bash
 
 # ottoRequestWatch.sh - drive the alignment pipeline from ottoRequest table
 #
 # Intended to run from cron under the user's own account (not the
 # web-server service user).  Picks up requests that ottoRequest.py has
 # acknowledged (status=1) and drives them through alignment setup
 # and workflow monitoring.
 #
-# Phase 1: status=1 with empty buildDir
+# Phase 1: new requests needing alignment setup - status=1 AND buildDir=''
 #          run ottoRequestAlign.sh to set up and launch the workflow
-# Phase 2: status=2 with buildDir set
+# Phase 2: in-progress requests needing workflow monitoring
 #          run workflowMonitor.sh to poll Galaxy and install results
 #   0 pending, 1 notified, 2 in progress, 3 galaxy done, 4 tracks complete,
-#   5 finish notification, 6 complete, 7 problems */
+#   5 ready to push, 6 push is done, 7 problems,
+#      8 final notification has been sent == process is complete
 
 set -eEu -o pipefail
 
 export scriptDir=$(cd "$(dirname "$0")" && pwd)
 
 ##############################################################################
 ### errors - set error status in the table
 function setErrorStatus() {
   id="${1}"
   hgsql -N -e \
       "UPDATE ottoRequest SET status=7 WHERE id=${id};" hgcentraltest
 }
 ##############################################################################
 
 ##############################################################################
@@ -175,101 +176,123 @@
 }
 ##############################################################################
 
 ############################################################################
 # phase 1: new requests needing alignment setup - status=1 AND buildDir=''
 ############################################################################
 while read -r reqId; do
   printf "# starting alignment for request %s\n" "${reqId}" 1>&2
   if "${scriptDir}/ottoRequestAlign.sh" "${reqId}"; then
     printf "# alignment setup complete for request %s\n" "${reqId}" 1>&2
   else
     printf "# alignment setup FAILED for request %s\n" "${reqId}" 1>&2
     setErrorStatus "${reqId}"
   fi
 done < <(hgsql -N -B -e \
-  "SELECT id FROM ottoRequest WHERE status = 1 AND buildDir = '';" \
+  "SELECT id FROM ottoRequest WHERE status = 1 AND buildDir = '' AND requestType = 'liftOver';" \
   hgcentraltest)
 
 ############################################################################
 # phase 2: in-progress requests needing workflow monitoring
 ############################################################################
 while IFS=$'\t' read -r reqId buildDir; do
   if [ ! -d "${buildDir}" ]; then
     printf "# WARNING: buildDir not found for request %s: %s\n" \
       "${reqId}" "${buildDir}" 1>&2
     continue
   fi
   printf "# monitoring request %s: %s\n" "${reqId}" "${buildDir}" 1>&2
   if "${scriptDir}/workflowMonitor.sh" "${reqId}" "${buildDir}"; then
     # workflowMonitor.sh exits 0 both when still running and when complete;
     # check for the success marker to distinguish
     if [ -s "${buildDir}/successInvocationId.txt" ]; then
       hgsql -N -e \
         "UPDATE ottoRequest SET status = 4, completeTime = NOW() \
          WHERE id = ${reqId};" hgcentraltest
       printf "# request %s completed successfully\n" "${reqId}" 1>&2
     fi
     # else: still running, will check again next invocation
   else
     printf "# workflow error for request %s\n" "${reqId}" 1>&2
     setErrorStatus "${reqId}"
   fi
 done < <(hgsql -N -B -e \
   "SELECT id, buildDir FROM ottoRequest \
-   WHERE status = 2 AND buildDir != '';" hgcentraltest)
+   WHERE status = 2 AND buildDir != '' AND requestType = 'liftOver';" hgcentraltest)
 
 ############################################################################
-# phase 3: check for tracks done, send notification email, mark complete
+# phase 3: check for tracks done, setup symlinks set status=5 to indicate
+#          ready to push
 ############################################################################
 while IFS=$'\t' read -r reqId buildDir; do
   if [ ! -d "${buildDir}" ]; then
     printf "# WARNING: buildDir not found for request %s: %s\n" \
       "${reqId}" "${buildDir}" 1>&2
     continue
   fi
   source <(grep -E '^export (swapDir|targetDb|queryDb)=' "${buildDir}/kegAlign.sh")
   echo $buildDir
   echo $swapDir
   echo targetDb $targetDb
   echo queryDb $queryDb
   export trackData="$(dirname "${buildDir}")"
   export swapData="$(dirname "${swapDir}")"
   export workDir="$(basename "${buildDir}")"
   export swapWork="$(basename "${swapDir}")"
   export doTdb="`dirname ${trackData}`/doTrackDb.bash"
   export swapTdb="`dirname ${swapData}`/doTrackDb.bash"
   if [ ! -x "${doTdb}" ]; then
-    printf "ERROR: can not find ${doTdb}\n" 1>&2
+    printf "ERROR: can not find %s\n" "${doTdb}" 1>&2
     setErrorStatus "${reqId}"
+    continue
   fi
   if [ ! -x "${swapTdb}" ]; then
-    printf "ERROR: can not find ${swapTdb}\n" 1>&2
+    printf "ERROR: can not find %s\n" "${swapTdb}" 1>&2
     setErrorStatus "${reqId}"
+    continue
   fi
   rm -f "${trackData}/lastz.${queryDb}"
   ln -s "${workDir}" "${trackData}/lastz.${queryDb}"
   printf "%s\n" "${doTdb}"
   ${doTdb} 1>&2
   rm -f "${swapData}/lastz.${targetDb}"
   ln -s "${swapWork}" "${swapData}/lastz.${targetDb}"
   printf "%s\n" "${swapTdb}"
   ${swapTdb} 1>&2
 
   # install liftOver and quickLift symlinks + register in hgcentraltest,
   # for both directions
   if ! installLinks "${targetDb}" "${queryDb}" "${buildDir}"; then
     setErrorStatus "${reqId}"
     continue
   fi
   if ! installLinks "${queryDb}" "${targetDb}" "${swapDir}"; then
     setErrorStatus "${reqId}"
     continue
   fi
+
+  hgsql -N -e \
+      "UPDATE ottoRequest SET status = 5 WHERE id=${reqId};" hgcentraltest
+done < <(hgsql -N -B -e \
+  "SELECT id, buildDir FROM ottoRequest \
+   WHERE status = 4 AND buildDir != '' AND requestType = 'liftOver';" hgcentraltest)
+
+############################################################################
+# phase 4: check for push files is complete, send final notification
+#          clean up galaxy workflow
+############################################################################
+
+while IFS=$'\t' read -r reqId fromDb toDb buildDir; do
+
+   # TBD using buildDir go clean up the galaxy workflow
+
    sendNotification "${reqId}" \
-"from UCSC: liftOverRequest complete: ${targetDb}<->${queryDb}" \
+"from UCSC: liftOverRequest complete: ${fromDb}<->${toDb}" \
 "Your lift over request is complete.  You can access the lift.over files at:"
+
   hgsql -N -e \
-      "UPDATE ottoRequest SET status=6 WHERE id=${reqId};" hgcentraltest
+      "UPDATE ottoRequest SET status=8 WHERE id=${reqId};" hgcentraltest
+
 done < <(hgsql -N -B -e \
-  "SELECT id, buildDir FROM ottoRequest \
-   WHERE status = 4 AND buildDir != '';" hgcentraltest)
+  "SELECT id, fromDb, toDb, buildDir FROM ottoRequest \
+   WHERE status = 6 AND requestType = 'liftOver';" hgcentraltest)
+