c54bbc1b2596adbb4bf4fcd278a531224c1d4624
hiram
  Mon Jun 8 15:23:59 2026 -0700
add checking for the files to go through the system refs #31811

diff --git src/hg/utils/otto/userRequests/asmRequestWatch.sh src/hg/utils/otto/userRequests/asmRequestWatch.sh
index f80129f6a69..c2635866144 100755
--- src/hg/utils/otto/userRequests/asmRequestWatch.sh
+++ src/hg/utils/otto/userRequests/asmRequestWatch.sh
@@ -100,30 +100,45 @@
 }	# function sendNotification()
 
 ##############################################################################
 ### accessionToPath - given an accession - expand to path name
 ##############################################################################
 function accessionToPath() {
   local acc="${1}"
   local gcX="${acc:0:3}"
   local d0="${acc:4:3}"
   local d1="${acc:7:3}"
   local d2="${acc:10:3}"
   local pathName="${gcX}/${d0}/${d1}/${d2}"
   printf "%s" "${pathName}"
 }	# function accessionToPath()
 
+##############################################################################
+### checkRemoteFile - via ssh command, check file exists on remote host
+##############################################################################
+function checkRemoteFile() {
+  local host="$1"
+  local filepath="$2"
+
+  if ssh -o ConnectTimeout=10 -o BatchMode=yes \
+             "$host" "test -f '$filepath'" </dev/null 2>/dev/null; then
+    return 0  # file exists
+  else
+    return 1  # file missing or SSH failed
+  fi
+}	# function checkRemoteFile()
+
 ############################################################################
 # phase 1: watch for new requests, detect when assembly build has started
 ############################################################################
 while IFS=$'\t' read -r reqId fromDb; do
   accPath=$(accessionToPath "${fromDb}")
   # if the trackData/ directory is present, the build is running
   shopt -s nullglob  # make globs expand to nothing if no matches
   trackDataDirs=(/hive/data/genomes/asmHubs/allBuild/${accPath}/${fromDb}_*/trackData)
   shopt -u nullglob  # restore default behavior
   # Check the results
   case ${#trackDataDirs[@]} in
     0)	# no directory seen yet, nothing happening
       ;;
     1)  # single directory seen - build has started - set the buildDir
       buildDir=$(dirname "$(realpath "${trackDataDirs[0]}")")
@@ -160,30 +175,125 @@
       /cluster/bin/x86_64/${hgSql} -N -e \
         "UPDATE ottoRequest SET status=3 WHERE id=${reqId};" ${centDb}
       ;;
     *)
       scriptName=$(basename "$0")
       printf "ERROR: %s: Multiple trackDb.txt files found for %s:\n" "${scriptName}" "${accPath}" 1>&2
       printf "  %s\n" "${trackDbFile[@]}" 1>&2
       setErrorStatus "${reqId}"
       ;;
   esac
 
 done < <(/cluster/bin/x86_64/${hgSql} -N -B -e \
   "SELECT id, fromDb FROM ottoRequest WHERE status = 2 AND requestType = 'assembly';" \
   ${centDb})
 
+############################################################################
+# phase 3: watch for the files to appear on hgwdev
+############################################################################
+while IFS=$'\t' read -r reqId fromDb; do
+  accPath=$(accessionToPath "${fromDb}")
+  # if the hub.txt file is present, the build is available on hgwdev
+  shopt -s nullglob  # make globs expand to nothing if no matches
+  hubTxt=(/gbdb/genark/${accPath}/${fromDb}/hub.txt)
+  shopt -u nullglob  # restore default behavior
+  # Check the results
+  case ${#hubTxt[@]} in
+    0)	# no hub.txt file seen yet, not done
+      ;;
+    1)  # single file seen - build is available on hgwdev
+      /cluster/bin/x86_64/${hgSql} -N -e \
+        "UPDATE ottoRequest SET status=4 WHERE id=${reqId};" ${centDb}
+      ;;
+    *)
+      scriptName=$(basename "$0")
+      printf "ERROR: %s: Multiple hub.txt files found for %s:\n" "${scriptName}" "${accPath}" 1>&2
+      printf "  %s\n" "${hubTxt[@]}" 1>&2
+      setErrorStatus "${reqId}"
+      ;;
+  esac
+
+done < <(/cluster/bin/x86_64/${hgSql} -N -B -e \
+  "SELECT id, fromDb FROM ottoRequest WHERE status = 3 AND requestType = 'assembly';" \
+  ${centDb})
+
+############################################################################
+# phase 4: watch for the files to appear on hgwbeta
+############################################################################
+while IFS=$'\t' read -r reqId fromDb; do
+  accPath=$(accessionToPath "${fromDb}")
+  # if the hub.txt file is present, the build is available on hgwdev
+  shopt -s nullglob  # make globs expand to nothing if no matches
+  hubTxt=(/gbdb/genark/${accPath}/${fromDb}/hub.txt)
+  shopt -u nullglob  # restore default behavior
+  # Check the results
+  case ${#hubTxt[@]} in
+    0)	# no hub.txt file seen, not supposed to happen in this state
+      printf "ERROR: %s status 4 /gbdb/genark/${accPath}/${fromDb}/hub.txt file is missing\n" 1>&2
+      setErrorStatus "${reqId}"
+      ;;
+    1)  # single file seen - build is available on hgwdev
+      if checkRemoteFile "qateam@hgwbeta" "${hubTxt[0]}"; then
+        /cluster/bin/x86_64/${hgSql} -N -e \
+          "UPDATE ottoRequest SET status=5 WHERE id=${reqId};" ${centDb}
+      fi
+      ;;
+    *)
+      scriptName=$(basename "$0")
+      printf "ERROR: %s: Multiple hub.txt files found for %s:\n" "${scriptName}" "${accPath}" 1>&2
+      printf "  %s\n" "${hubTxt[@]}" 1>&2
+      setErrorStatus "${reqId}"
+      ;;
+  esac
+
+done < <(/cluster/bin/x86_64/${hgSql} -N -B -e \
+  "SELECT id, fromDb FROM ottoRequest WHERE status = 4 AND requestType = 'assembly';" \
+  ${centDb})
+
+############################################################################
+# phase 5: watch for the files to appear on hgw2
+############################################################################
+while IFS=$'\t' read -r reqId fromDb; do
+  accPath=$(accessionToPath "${fromDb}")
+  # if the hub.txt file is present, the build is available on hgwdev
+  shopt -s nullglob  # make globs expand to nothing if no matches
+  hubTxt=(/gbdb/genark/${accPath}/${fromDb}/hub.txt)
+  shopt -u nullglob  # restore default behavior
+  # Check the results
+  case ${#hubTxt[@]} in
+    0)	# no hub.txt file seen, not supposed to happen in this state
+      printf "ERROR: %s status 5 /gbdb/genark/${accPath}/${fromDb}/hub.txt file is missing\n" 1>&2
+      setErrorStatus "${reqId}"
+      ;;
+    1)  # single file seen - build is available on hgwdev
+      if checkRemoteFile "qateam@hgw2" "${hubTxt[0]}"; then
+        /cluster/bin/x86_64/${hgSql} -N -e \
+          "UPDATE ottoRequest SET status=6 WHERE id=${reqId};" ${centDb}
+      fi
+      ;;
+    *)
+      scriptName=$(basename "$0")
+      printf "ERROR: %s: Multiple hub.txt files found for %s:\n" "${scriptName}" "${accPath}" 1>&2
+      printf "  %s\n" "${hubTxt[@]}" 1>&2
+      setErrorStatus "${reqId}"
+      ;;
+  esac
+
+done < <(/cluster/bin/x86_64/${hgSql} -N -B -e \
+  "SELECT id, fromDb FROM ottoRequest WHERE status = 5 AND requestType = 'assembly';" \
+  ${centDb})
+
 ############################################################################
 # check for phase 6: the assembly is complete and available on the RR
 #       this checking and setting status 6 is currently done manually,
 #       eventually this will become automatic.
 ############################################################################
 while IFS=$'\t' read -r reqId fromDb comment requestTime; do
 
   export gcX="${fromDb:0:3}"
   export d0="${fromDb:4:3}"
   export d1="${fromDb:7:3}"
   export d2="${fromDb:10:3}"
   export gbDbPath="/gbdb/genark/${gcX}/${d0}/${d1}/${d2}/${fromDb}/hub.txt"
   export hubTxt="https://genome.ucsc.edu/cgi-bin/hgTracks?genome=${fromDb}&hubUrl=${gbDbPath}"
 
   sendNotification "${reqId}" \