b761e0e7b483d9fbb2832820c138c0baab13a869
lrnassar
  Thu Jul 2 17:39:24 2026 -0700
Fix Total out of memory metric in errorLogKeywordSearch.py to count only genuine events. refs #37699

A single needMem OOM failure recurses in the errAbort/warn handler (which itself
allocates), re-emitting the "request size %llu bytes" format string many times per
crash. The old grep "needMem: Out of memory" counted every one of those frames,
inflating the weekly count ~60-175x (e.g. ~1070 reported vs ~18 genuine events for
the week of 20260614). Restrict the match to lines with a numeric request size so
only real allocation failures are counted.

diff --git src/utils/qa/errorLogKeywordSearch.py src/utils/qa/errorLogKeywordSearch.py
index 0fdf9fdd4a1..f1429f07089 100755
--- src/utils/qa/errorLogKeywordSearch.py
+++ src/utils/qa/errorLogKeywordSearch.py
@@ -62,31 +62,31 @@
                     continue
 
     return(user,latestLogs)
 
 
 def createDicOfSearchTerms():
     totalLinesInLog = dict(label='Total lines in logs', description='Total number of lines seen in the logs', value=[], searchKeyWord="wc -l")
     totalUniqueIPs = dict(label='Total unique IPs', description='Total number of unique IPs without port number, e.g. N.N.N and not N.N.N:NNN', value=[], searchKeyWord=r'grep "\[client" | cut -f4 -d "]" | cut -f3 -d " " | cut -f1 -d ":" | sort | uniq | wc -l')
     totalUniqueIPsSubnets = dict(label='Total unique IP subnets', description='Total number of unique IPs with only partial subnet, e.g. NNN.NNN and not NNN.NNN.N.NN', value=[], searchKeyWord=r'grep "\[client" | cut -f4 -d "]" | cut -f3 -d " " | cut -f1 -d ":" | cut -f1-2 -d "." | sort | uniq | wc -l')
     totalUniqueHgsids = dict(label='Total unique hgsIDs', description='Total number of unique hgsIDs', value=[], searchKeyWord=r"grep 'hgsid' | sed -n 's/.*[?&]hgsid=\([0-9A-Za-z_]*\).*/\1/p' | sort | uniq | wc -l")
     totalLoadedSessions = dict(label='Total loaded sessions', description='Total number of loaded sessions', value=[], searchKeyWord=' grep "CGI_TIME: hgTracks" | grep "/cgi-bin/hgSession?" | wc -l')
     totalSavedCTs = dict(label='Total saved CTs', description='Total number of saved custom tracks', value=[], searchKeyWord='grep "customTrack: saved" | wc -l')
     totalCTerrors = dict(label='Total CT errors', description='Total number of custom track load errors', value=[], searchKeyWord='grep "hgCustom load error" | wc -l')
     totalStackDumps = dict(label='Total stack dumps', description='Total number of stack dumps', value=[], searchKeyWord='grep "Stack dump" | wc -l')
     totalTryingToAllocate = dict(label='Total 500Mb allocate memory', description="Happens if code tries to allocate a chunk bigger than hard-wired limit of 500m. Could indicate naughty CGI", value=[], searchKeyWord='grep "needMem: trying to allocate" | wc -l')
-    totalOutOfMemory = dict(label='Total out of memory', description='Happens if malloc() fails because the OS native limits (or hg.conf maxMem limits)', value=[], searchKeyWord='grep "needMem: Out of memory" | wc -l')
+    totalOutOfMemory = dict(label='Total out of memory', description='Happens if malloc() fails because the OS native limits (or hg.conf maxMem limits). Matches only lines with a numeric request size, since a single failure recurses in the abort handler and re-emits the "request size %llu bytes" format string many times, inflating a plain grep', value=[], searchKeyWord=r'grep -E "needMem: Out of memory - request size [0-9]" | wc -l')
     totalHogExits = dict(label='hogExit', description='hogExit: Total number of people that hit the bottleneck', value=[], searchKeyWord='grep "hogExit" | wc -l')
     totalHgCollectionsExpire = dict(label='hgCollections', description='Total number of expired hgCollections', value=[], searchKeyWord='grep "Track Collections expire 48" | wc -l')
     totalWarnTimings = dict(label='warnTiming', description='warnTiming: Number of people that hit the warnSeconds hg.conf var. Warns them about image taking too long to load', value=[], searchKeyWord='grep "warnTiming" | wc -l')
 
     itemsToFind = [totalLinesInLog,totalUniqueIPs,totalUniqueIPsSubnets,totalUniqueHgsids,totalLoadedSessions,totalSavedCTs,totalCTerrors,totalHgCollectionsExpire,totalHogExits,totalStackDumps,totalTryingToAllocate,totalOutOfMemory,totalWarnTimings]
     return(itemsToFind)
 
 def searchForTermsInLogs():
     user,latestLogs = copyLatestLogs()
     itemsToFind = createDicOfSearchTerms()
 
     #n=0 ##### Uncomment these lines to see progress
     for log in latestLogs:
         #n+=1 ### Progress
         logPath = "zcat /hive/users/"+user+"/ErrorLogs/*"+log+" | "