70b873ace3ffca0850aa635f7fc43c71cc7b09d5 hiram Fri May 1 14:50:19 2026 -0700 further silent the cron outputs and add a status indicator to the watch.cgi page refs #31811 diff --git src/hg/utils/otto/userRequests/ottoRequestWatch.sh src/hg/utils/otto/userRequests/ottoRequestWatch.sh index d5fe21e9d6c..c7b047e5e67 100755 --- src/hg/utils/otto/userRequests/ottoRequestWatch.sh +++ src/hg/utils/otto/userRequests/ottoRequestWatch.sh @@ -225,157 +225,176 @@ -path="${chainPath}" "${tDb}" "${qDb}"; then printf "ERROR: installLinks: hgAddLiftOverChain failed for %s -> %s\n" \ "${tDb}" "${qDb}" 1>&2 return 1 fi if ! "${HOME}/kent/src/hg/utils/automation/addQuickLift.py" \ "${tDb}" "${qDb}" "${quickPath}"; then printf "ERROR: installLinks: addQuickLift.py failed for %s -> %s\n" \ "${tDb}" "${qDb}" 1>&2 return 1 fi return 0 } ############################################################################## +############################################################################## +### refresh Galaxy queue status snapshot for ottoRequestView.cgi +############################################################################## +export galaxyStatusFile="/data/apache/trash/ottoRequestGalaxyStatus.json" +export profileJson="${HOME}/.planemo/profiles/vgp/planemo_profile_options.json" +gsTmp=$(mktemp "${galaxyStatusFile}.XXXXXX") +if timeout 45 "${scriptDir}/galaxyStatus.py" "${profileJson}" > "${gsTmp}" 2>/dev/null; then + mv "${gsTmp}" "${galaxyStatusFile}" +else + rm -f "${gsTmp}" +# printf "# WARNING: galaxyStatus.py failed, leaving stale snapshot\n" 1>&2 +fi +############################################################################## + ############################################################################ # phase 1: new requests needing alignment setup - status=1 AND buildDir='' ############################################################################ while read -r reqId; do - printf "# starting alignment for request %s\n" "${reqId}" 1>&2 - if "${scriptDir}/ottoRequestAlign.sh" "${reqId}"; then - printf "# alignment setup complete for request %s\n" "${reqId}" 1>&2 - else +# printf "# starting alignment for request %s\n" "${reqId}" 1>&2 + if ! "${scriptDir}/ottoRequestAlign.sh" "${reqId}"; then printf "# alignment setup FAILED for request %s\n" "${reqId}" 1>&2 setErrorStatus "${reqId}" fi done < <(/cluster/bin/x86_64/hgsql -N -B -e \ "SELECT id FROM ottoRequest WHERE status = 1 AND buildDir = '' AND requestType = 'liftOver';" \ hgcentraltest) ############################################################################ # phase 2: in-progress requests needing workflow monitoring ############################################################################ while IFS=$'\t' read -r reqId buildDir; do if [ ! -d "${buildDir}" ]; then printf "# WARNING: buildDir not found for request %s: %s\n" \ "${reqId}" "${buildDir}" 1>&2 continue fi # takes a while for the galaxy WF to start up, wait for this file to appear if [ ! -s "${buildDir}/pendingInvocationId.txt" ]; then continue fi # printf "# monitoring request %s: %s\n" "${reqId}" "${buildDir}" 1>&2 if "${scriptDir}/workflowMonitor.sh" "${reqId}" "${buildDir}"; then # workflowMonitor.sh exits 0 both when still running and when complete; # check for the success marker to distinguish if [ -s "${buildDir}/successInvocationId.txt" ]; then /cluster/bin/x86_64/hgsql -N -e \ "UPDATE ottoRequest SET status = 4, completeTime = NOW() \ WHERE id = ${reqId};" hgcentraltest - printf "# request %s completed successfully\n" "${reqId}" 1>&2 +# printf "# request %s completed successfully\n" "${reqId}" 1>&2 fi # else: still running, will check again next invocation else printf "# workflow error for request %s\n" "${reqId}" 1>&2 setErrorStatus "${reqId}" fi done < <(/cluster/bin/x86_64/hgsql -N -B -e \ "SELECT id, buildDir FROM ottoRequest \ WHERE status = 2 AND buildDir != '' AND requestType = 'liftOver';" hgcentraltest) ############################################################################ # phase 3: check for tracks done, setup symlinks set status=5 to indicate # ready to push ############################################################################ while IFS=$'\t' read -r reqId buildDir; do if [ ! -d "${buildDir}" ]; then printf "# WARNING: buildDir not found for request %s: %s\n" \ "${reqId}" "${buildDir}" 1>&2 continue fi source <(grep -E '^export (swapDir|targetDb|queryDb)=' "${buildDir}/kegAlign.sh") - echo $buildDir - echo $swapDir - echo targetDb $targetDb - echo queryDb $queryDb export trackData="$(dirname "${buildDir}")" export swapData="$(dirname "${swapDir}")" export workDir="$(basename "${buildDir}")" export swapWork="$(basename "${swapDir}")" export doTdb="`dirname ${trackData}`/doTrackDb.bash" export swapTdb="`dirname ${swapData}`/doTrackDb.bash" if [ ! -x "${doTdb}" ]; then printf "ERROR: can not find %s\n" "${doTdb}" 1>&2 setErrorStatus "${reqId}" continue fi if [ ! -x "${swapTdb}" ]; then printf "ERROR: can not find %s\n" "${swapTdb}" 1>&2 setErrorStatus "${reqId}" continue fi rm -f "${trackData}/lastz.${queryDb}" ln -s "${workDir}" "${trackData}/lastz.${queryDb}" - printf "%s\n" "${doTdb}" - ${doTdb} 1>&2 + if ! ${doTdb} 1>&2; then + printf "ERROR: %s failed\n" "${doTdb}" 1>&2 + setErrorStatus "${reqId}" + continue + fi rm -f "${swapData}/lastz.${targetDb}" ln -s "${swapWork}" "${swapData}/lastz.${targetDb}" - printf "%s\n" "${swapTdb}" - ${swapTdb} 1>&2 + if ! ${swapTdb} 1>&2; then + printf "ERROR: %s failed\n" "${swapTdb}" 1>&2 + setErrorStatus "${reqId}" + continue + fi # install liftOver and quickLift symlinks + register in hgcentraltest, # for both directions if ! installLinks "${targetDb}" "${queryDb}" "${buildDir}"; then setErrorStatus "${reqId}" continue fi if ! installLinks "${queryDb}" "${targetDb}" "${swapDir}"; then setErrorStatus "${reqId}" continue fi /cluster/bin/x86_64/hgsql -N -e \ "UPDATE ottoRequest SET status = 5 WHERE id=${reqId};" hgcentraltest done < <(/cluster/bin/x86_64/hgsql -N -B -e \ "SELECT id, buildDir FROM ottoRequest \ WHERE status = 4 AND buildDir != '' AND requestType = 'liftOver';" hgcentraltest) ############################################################################ # phase 4: check for push files is complete, send final notification # clean up galaxy workflow ############################################################################ -while IFS=$'\t' read -r reqId fromDb toDb buildDir; do +while IFS=$'\t' read -r reqId fromDb toDb comment requestTime buildDir; do # time to clean up the galaxy history and workflow to release the space if [ -s "${buildDir}/successInvocationId.txt" ]; then invocationId=$(cut -f2 "${buildDir}/successInvocationId.txt") - profileJson="${HOME}/.planemo/profiles/vgp/planemo_profile_options.json" if ! "${scriptDir}/galaxyCleanup.py" "${profileJson}" "${invocationId}"; then printf "# WARNING: galaxy cleanup failed for request %s\n" "${reqId}" 1>&2 fi fi if ! fromUrl="$(liftOverUrl "${fromDb}" "${toDb}")"; then setErrorStatus "${reqId}" continue fi if ! toUrl="$(liftOverUrl "${toDb}" "${fromDb}")"; then setErrorStatus "${reqId}" continue fi sendNotification "${reqId}" \ "from UCSC: liftOverRequest complete: ${fromDb}<->${toDb}" \ -"Your lift over request is complete. You can access the lift.over files at: +"Your lift over request is complete: + From: ${fromDb} + To: ${toDb} + comment: ${comment} +submitted: ${requestTime} + +The lift.over files are available at these links: ${fromUrl} ${toUrl} " /cluster/bin/x86_64/hgsql -N -e \ "UPDATE ottoRequest SET status=8, completeTime=now() WHERE id=${reqId};" hgcentraltest done < <(/cluster/bin/x86_64/hgsql -N -B -e \ - "SELECT id, fromDb, toDb, buildDir FROM ottoRequest \ + "SELECT id, fromDb, toDb, comment, requestTime, buildDir FROM ottoRequest \ WHERE status = 6 AND requestType = 'liftOver';" hgcentraltest)