5a34a4caa476493be5efb117dff50073d5637e20
braney
  Wed Sep 16 12:40:05 2026 -0700
doHgTablesTestRobot: fail when a run dies instead of reporting success, refs #38356

This is the robot that runs at the final build, against hgwbeta.  Its check
looked only for Total lines that carried errors.  hgTablesTest writes that line
after a run finishes, so a run that died partway through wrote no Total line at
all, the match came back empty, and the robot mailed "done successfully".  Every
log back to v490 is missing the summary line, so this gate had never once fired.

Give it the checks the preview2 robot already got in cf348a83d76: the exit
status of each of the two runs, a count of the Total lines against the number of
runs expected, the existing error-in-Total check, and a grep for crash
signatures over both the report log and a separate file holding the runs' own
output.  A missing summary is now itself a failure, and the mail says which of
those conditions tripped.

Also a ROBOT_MAILTO override, so re-running the robot to diagnose something does
not notify the QA list every time.

Tested against a stub hgTablesTest and a stub mail, four ways.  A clean run
still passes.  A run reporting errors in its Total line still fails.  A run that
dies partway, whether it exits nonzero or zero, used to pass and now fails.

diff --git src/utils/qa/weeklybld/doHgTablesTestRobot.csh src/utils/qa/weeklybld/doHgTablesTestRobot.csh
index 40094f6a538..b1e4627540c 100755
--- src/utils/qa/weeklybld/doHgTablesTestRobot.csh
+++ src/utils/qa/weeklybld/doHgTablesTestRobot.csh
@@ -1,35 +1,104 @@
 #!/bin/tcsh
 cd $WEEKLYBLD
 
 cd $BUILDDIR/v${BRANCHNN}_branch/kent/src/hg/hgTablesTest
 make 
 cd $WEEKLYBLD
 
 setenv HGDB_CONF /cluster/home/build/.hg.conf.beta
+
+# Run the binary the make above just produced.  userApp.mk installs to
+# $HOME/bin/$MACHTYPE, so name that path rather than a shared directory this
+# robot does not own.
+set hgTablesTest = $HOME/bin/$MACHTYPE/hgTablesTest
+if ( ! -x $hgTablesTest ) then
+    echo "ERROR: $hgTablesTest missing or not executable - did the make above fail?"
+    echo "$hgTablesTest missing or not executable; the make above probably failed." \
+      | mail -s "Errors in hgTablesTestRobot on $HOST" ${BUILDMEISTEREMAIL}
+    exit 1
+endif
+
 set log = v${BRANCHNN}.hgTables.log
+# Combined stdout/stderr of the two hgTablesTest runs.  Kept SEPARATE from $log:
+# hgTablesTest writes its structured report to $log itself and ALSO echoes the
+# summary to stdout, so merging the two streams into one file would double every
+# "Total:" line and defeat the completed-run count below.
+set runout = v${BRANCHNN}.hgTables.runout
+
+# Who hears about the result.  Defaults to the build meister plus QA; a
+# diagnostic re-run can narrow it with e.g.
+#   setenv ROBOT_MAILTO build@soe.ucsc.edu
+# so re-testing does not notify the QA list.
+set mailto = "${BUILDMEISTEREMAIL} browser-qa@soe.ucsc.edu"
+if ( $?ROBOT_MAILTO ) then
+    set mailto = "$ROBOT_MAILTO"
+endif
 
-echo "$HOME/bin/$MACHTYPE/hgTablesTest -db=hg38 https://hgwbeta.soe.ucsc.edu/cgi-bin/hgTables ./logs/$log" > ./logs/$log
-$HOME/bin/$MACHTYPE/hgTablesTest -appendLog -db=hg38 https://hgwbeta.soe.ucsc.edu/cgi-bin/hgTables ./logs/$log
+rm -f ./logs/$runout
+echo "$hgTablesTest -db=hg38 https://hgwbeta.soe.ucsc.edu/cgi-bin/hgTables ./logs/$log" > ./logs/$log
+$hgTablesTest -appendLog -db=hg38 https://hgwbeta.soe.ucsc.edu/cgi-bin/hgTables ./logs/$log >>& ./logs/$runout
+set st1 = $status
 echo "" >> ./logs/$log
-echo "$HOME/bin/$MACHTYPE/hgTablesTest -appendLog -org=Mouse -orgs=1  https://hgwbeta.soe.ucsc.edu/cgi-bin/hgTables ./logs/$log" >> ./logs/$log
-$HOME/bin/$MACHTYPE/hgTablesTest -appendLog -org=Mouse -orgs=1  https://hgwbeta.soe.ucsc.edu/cgi-bin/hgTables ./logs/$log
+echo "$hgTablesTest -appendLog -org=Mouse -orgs=1  https://hgwbeta.soe.ucsc.edu/cgi-bin/hgTables ./logs/$log" >> ./logs/$log
+$hgTablesTest -appendLog -org=Mouse -orgs=1  https://hgwbeta.soe.ucsc.edu/cgi-bin/hgTables ./logs/$log >>& ./logs/$runout
+set st2 = $status
 
 # creates hgTables.log - look for unusual errors
 
 #-- to check for errors:
-set res = `cat ./logs/$log | egrep "^[ \t]*Total" | egrep -v "0 soft errors,  0 hard errors"`
-set res2 = `cat ./logs/$log | awk '/^[ \t]*Total/ {print} /TablesTest/ {print}'`
+# hgTablesTest prints its "Total: N tests, N soft errors, N hard errors" line
+# from reportSummary() only after a run finishes.  When a run dies partway --
+# the carefulAlloc 500MB ceiling, a segfault, an errAbort it has no guard for --
+# that line never gets written.  The old check looked ONLY for Total lines that
+# carried errors, so a crash produced an empty match and was reported as
+# success.  So now a missing Total summary, or a nonzero exit from either run,
+# is itself a hard failure.  This mirrors preview2TablesTestRobot.csh, refs #38356.
+set expectedRuns = 2
+set nTotal = `egrep -c "^[ 	]*Total:" ./logs/$log`
+set res  = `egrep "^[ 	]*Total:" ./logs/$log | egrep -v "0 soft errors,  0 hard errors"`
+set res2 = `awk '/^[ \t]*Total/ {print} /TablesTest/ {print}' ./logs/$log`
+set crash = `cat ./logs/$log ./logs/$runout | egrep "carefulAlloc|Couldn.t |needLargeMem|needMem is null|Segmentation|Out of memory|Assertion|no form produced"`
 
+set problems = ""
+if ( "$st1" != "0" ) then
+    set problems = "$problems hg38-run-exited-$st1"
+endif
+if ( "$st2" != "0" ) then
+    set problems = "$problems mouse-run-exited-$st2"
+endif
+if ( "$nTotal" != "$expectedRuns" ) then
+    set problems = "$problems incomplete-only-$nTotal-of-$expectedRuns-runs-reached-their-Total-summary"
+endif
+if ( "$res" != "" ) then
+    set problems = "$problems errors-reported-in-Total-summary"
+endif
+if ( "$crash" != "" ) then
+    set problems = "$problems crash-signature-in-log"
+endif
 
-set wc = `echo "$res" | wc -w` 
-if ( "$wc" != "0" ) then
- echo "errs found:"
+if ( "$problems" != "" ) then
+    echo "errs found:$problems"
     echo "$res2"
- echo "$res2" | mail -s "Errors in hgTablesTestRobot on $HOST" ${BUILDMEISTEREMAIL} browser-qa@soe.ucsc.edu
+    ( echo "hgTablesTest robot FAILED for v${BRANCHNN} on $HOST" ; \
+      echo "" ; \
+      echo "problems:$problems" ; \
+      echo "" ; \
+      echo "hg38 run exit status : $st1" ; \
+      echo "Mouse run exit status: $st2" ; \
+      echo "completed runs       : $nTotal of $expectedRuns reached their Total summary" ; \
+      echo "" ; \
+      echo "crash signatures:" ; \
+      echo "$crash" ; \
+      echo "" ; \
+      echo "summary lines:" ; \
+      echo "$res2" ; \
+      echo "" ; \
+      echo "report log: $WEEKLYBLD/logs/$log" ; \
+      echo "run output: $WEEKLYBLD/logs/$runout" ) \
+      | mail -s "Errors in hgTablesTestRobot on $HOST" $mailto
     exit 1
 endif
 #
 echo Done.
-echo "$res2" | mail -s "v${BRANCHNN} hgTablesTest robot done successfully on $HOST." ${BUILDMEISTEREMAIL} browser-qa@soe.ucsc.edu
+echo "$res2" | mail -s "v${BRANCHNN} hgTablesTest robot done successfully on $HOST." $mailto
 exit 0
-