4088b6e57be69de4f37e00beefb13969a34eb09e
braney
  Tue Aug 25 16:40:27 2026 -0700
trackDbCacheCleaner: do not abandon the run when one removal fails, refs #37551

A cache directory can be owned by any user who ran a command line utility,
not just by apache, so a removal can fail on permissions.  With
set -beEu that aborted the whole run part way through, leaving a half
cleaned cache and printing no summary at all.

Count the failures instead, warn about the first ten, and finish the rest
of the cleaning.  A run with any failure reports FAILED and exits 1, so a
cron notices, and the next run retries what was left behind.

diff --git src/product/scripts/trackDbCacheCleaner.sh src/product/scripts/trackDbCacheCleaner.sh
index 7ddc59213c7..c6780bccbdd 100755
--- src/product/scripts/trackDbCacheCleaner.sh
+++ src/product/scripts/trackDbCacheCleaner.sh
@@ -132,36 +132,50 @@
 if (used > newest[dir]) newest[dir] = used
 bytes[dir] += $3
 files[dir] += 1
 }
 END {
 for (dir in newest)
     if (newest[dir] < cutoff)
         printf "%d %d %s\n", files[dir], bytes[dir], dir
 }' "${scanList}" > "${expireList}"
 rm -f "${scanList}"
 
 export expireDirCount=`cat "${expireList}" | wc -l`
 export expireFileCount=`awk '{n += $1} END {printf "%d", n}' "${expireList}"`
 export expireByteCount=`awk '{n += $2} END {printf "%d", n}' "${expireList}"`
 
+# a cache directory can be owned by any user who ran a command line
+# utility, so a removal can fail on permissions.  Count those and carry on,
+# rather than abandoning the rest of the cleaning.
+export failedDirCount=0
+export warnLimit=10
+
 if [ "${dryRun}" -eq 1 ]; then
   awk '{printf "would remove %s (%d files, %d bytes)\n", $3, $1, $2}' "${expireList}"
 else
   while read fileCount byteCount dirName
   do
-    rm -fr "${dirName}"
+    if ! rm -fr "${dirName}" 2>/dev/null; then
+      failedDirCount=`echo "${failedDirCount}" | awk '{print $1 + 1}'`
+      if [ "${failedDirCount}" -le "${warnLimit}" ]; then
+        echo "# warning: could not remove ${dirName}" 1>&2
+      fi
+      if [ "${failedDirCount}" -eq "${warnLimit}" ]; then
+        echo "# warning: further removal failures not listed" 1>&2
+      fi
+    fi
   done < "${expireList}"
 fi
 
 # a directory left holding nothing at all has no files to judge it by,
 # so use the directory's own last use instead
 export emptyDirCount=0
 for dirName in `find "${cacheDir}" -mindepth 1 -maxdepth 1 -type d -empty 2>/dev/null || true`
 do
   dirUse=`stat -c '%X %Y' "${dirName}" 2>/dev/null | awk '{print ($1 > $2) ? $1 : $2}'`
   # empty and gone already, or filled in since the scan
   if [ -z "${dirUse}" ]; then
     continue
   fi
   if [ "${dirUse}" -lt "${cutoff}" ]; then
     emptyDirCount=`echo "${emptyDirCount}" | awk '{print $1 + 1}'`
@@ -169,16 +183,23 @@
       echo "would remove ${dirName} (empty)"
     else
       rmdir "${dirName}" 2>/dev/null || true
     fi
   fi
 done
 
 rm -f "${expireList}"
 
 export remainDirCount=`find "${cacheDir}" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l`
 
 LC_NUMERIC=en_US printf "# expired %'d directories, %'d files, %'d bytes\n" \
   "${expireDirCount}" "${expireFileCount}" "${expireByteCount}" 1>&2
 LC_NUMERIC=en_US printf "# expired %'d empty directories\n" "${emptyDirCount}" 1>&2
 LC_NUMERIC=en_US printf "# %'d directories remain in %s\n" "${remainDirCount}" "${cacheDir}" 1>&2
+
+if [ "${failedDirCount}" -gt 0 ]; then
+  LC_NUMERIC=en_US printf "# ERROR: %'d directories could not be removed\n" "${failedDirCount}" 1>&2
+  printf "# %s trackDbCacheCleaner.sh FAILED\n" "`date +%FT%T`" 1>&2
+  exit 1
+fi
+
 printf "# %s trackDbCacheCleaner.sh SUCCESS\n" "`date +%FT%T`" 1>&2