3976a8ec6b27464bd64e93916821dbcb52cd7671
hiram
  Thu Apr 30 23:01:52 2026 -0700
this actually might be working all the way through now refs #31811

diff --git src/hg/utils/otto/userRequests/ottoRequestWatch.sh src/hg/utils/otto/userRequests/ottoRequestWatch.sh
index 49a80162f75..3577e617b90 100755
--- src/hg/utils/otto/userRequests/ottoRequestWatch.sh
+++ src/hg/utils/otto/userRequests/ottoRequestWatch.sh
@@ -4,34 +4,57 @@
 #
 # Intended to run from cron under the user's own account (not the
 # web-server service user).  Picks up requests that ottoRequest.py has
 # acknowledged (status=1) and drives them through alignment setup
 # and workflow monitoring.
 #
 # Phase 1: new requests needing alignment setup - status=1 AND buildDir=''
 #          run ottoRequestAlign.sh to set up and launch the workflow
 # Phase 2: in-progress requests needing workflow monitoring
 #          run workflowMonitor.sh to poll Galaxy and install results
 #   0 pending, 1 notified, 2 in progress, 3 galaxy done, 4 tracks complete,
 #   5 ready to push, 6 push is done, 7 problems,
 #      8 final notification has been sent == process is complete
 
 set -eEu -o pipefail
-set -x
 
 export scriptDir=$(cd "$(dirname "$0")" && pwd)
 
+##############################################################################
+### singleton lock - only one instance at a time
+### Open lockPath on FD 9 for the lifetime of the shell, then take a
+### non-blocking exclusive lock.  Kernel releases the lock on exit
+### (normal, error, or kill -9), so no stale lock cleanup is needed.
+### Exit 0 silently if another instance holds the lock so cron doesn't
+### email on every overlapping tick.  PID is written to the file for
+### information only see the holder via:
+###   cat ottoRequestWatch.lock      (the PID)
+###   lsof ottoRequestWatch.lock     (the locking process)
+##############################################################################
+export lockPath="${scriptDir}/ottoRequestWatch.lock"
+# 9<> opens read+write without truncating, so a second instance that
+# comes along while we're running won't wipe our PID from the file
+# before its flock attempt fails.
+exec 9<>"${lockPath}"
+flock -n 9 || exit 0
+# we own the lock now safe to truncate and write our PID.  ': >file'
+# truncates via a separate FD; FD 9 keeps its position 0 from <>, so
+# the printf below starts writing at the beginning of the empty file.
+: >"${lockPath}"
+printf "%d\n" "$$" >&9
+##############################################################################
+
 ##############################################################################
 ### errors - set error status in the table
 function setErrorStatus() {
   id="${1}"
   hgsql -N -e \
       "UPDATE ottoRequest SET status=7 WHERE id=${id};" hgcentraltest
 }
 ##############################################################################
 
 ##############################################################################
 ### liftOverUrl - build the public download URL for an over.chain.gz file
 ###   args: srcDb dstDb
 ###   GenArk:      https://hgdownload.soe.ucsc.edu/hubs/<3>/<3>/<3>/<3>/<acc>/liftOver/<srcDb>To<DstDb>.over.chain.gz
 ###   UCSC native: https://hgdownload.soe.ucsc.edu/goldenPath/<srcDb>/liftOver/<srcDb>To<DstDb>.over.chain.gz
 ###   DstDb is dstDb with the first letter upper-cased (matches the