4f9164cb6142e7468325253a67c6ac8a6a3d1f91
braney
  Tue Aug 25 09:54:31 2026 -0700
ts: give each ticket sandbox its own udc cache, refs #37867

Every parked instance and the live sandbox shared trash/udcCache, so a cache
entry written by one instance was read by all of them. A sparse or bad entry
from one build reappeared under another, which breaks the freeze.

Each ticket now gets trash/udcCache/ts/NNNNN. It stays in the shared trash, so
the trash cleaner ages it out like any other udc cache, and it stays data, so
sync does not clear it. remove deletes it, since it sits inside the shared
trash and not in the sandbox directory. conf now sets it too, so an instance
frozen before this existed does not need a re-freeze.

diff --git src/utils/ts/ts src/utils/ts/ts
index 8ef2b7faa2e..3dca1c37e18 100755
--- src/utils/ts/ts
+++ src/utils/ts/ts
@@ -1,37 +1,40 @@
 #!/bin/bash
 #
 # ts - "ticket sandbox": park a UCSC Genome Browser Redmine ticket as its own
 # frozen browser instance on hgwdev.  refs #37867
 #
 # A per-ticket Apache instance, owned by the developer, bound to a loopback high
 # port, reached by an ssh tunnel (yours or a colleague's - all hgwdev users
 # share the loopback, so anyone with an hgwdev account can tunnel in).  The DB
 # and /gbdb are shared (hg.conf points at hgwdev MySQL; /gbdb is shared NFS) and
 # trash is shared with the live CGIs, so only the *code* is frozen: a full copy
 # of cgi-bin-$USER + htdocs-$USER plus a rewritten hg.conf.  Data is left
-# shared on purpose (freeze the code, share the data).
+# shared on purpose (freeze the code, share the data).  The one exception is the
+# udc cache: each ticket gets its own subtree under the shared trash, so one
+# instance cannot serve another a cache entry it wrote (see setUdcDir).
 #
 # The parked instances and the port registry live under $TS_ROOT, by default
 # $HOME/ticketSandboxes.  On hgwdev, point that at a large local pool (a freeze
 # is a few GB): mkdir /data/home/$USER/ticketSandboxes and symlink it from
 # $HOME, or set TS_ROOT.
 #
 # Subcommands (NNNNN = Redmine ticket number):
 #   create NNNNN [note]   freeze the live sandbox, start an httpd, register it
 #   sync   NNNNN          re-freeze a parked ticket to the current live sandbox
-#   conf   NNNNN          rewrite only httpd.conf, leaving the frozen code alone
+#   conf   NNNNN          rewrite httpd.conf and refresh the ts hg.conf settings,
+#                         leaving the frozen code alone
 #   start  NNNNN          start the ticket's httpd (e.g. after a reboot)
 #   stop   NNNNN          stop the ticket's httpd
 #   tunnel NNNNN          open an ssh tunnel and print the browser URL
 #   list                  show all parked tickets and their running status
 #   remove NNNNN          stop the httpd and delete the ticket sandbox
 #
 set -euo pipefail
 
 # --- configuration ----------------------------------------------------------
 ROOT="$(readlink -f "${TS_ROOT:-$HOME/ticketSandboxes}")"   # parked instances live here
 REG="$ROOT/ports.tsv"                        # ticket <TAB> port <TAB> created <TAB> note
 
 TS_USER="${USER:-$(id -un)}"                 # whose live sandbox we freeze
 LIVE_CGI=/usr/local/apache/cgi-bin-$TS_USER
 LIVE_HTDOCS=/usr/local/apache/htdocs-$TS_USER
@@ -114,55 +117,76 @@
         "$LIVE_HTDOCS/" "$dir/$HTDOCS_NAME/"
     # Rewritten, frozen hg.conf:
     #  - include ../cgi-bin/hg.conf -> absolute (avoid self-include loop; the
     #    shared base config is data-side and stays live by design)
     #  - browser.documentRoot -> this ticket's frozen htdocs-$USER
     sed -e 's#^[[:space:]]*include[[:space:]]\+\.\./cgi-bin/hg\.conf#include '"$BASE_HGCONF"'#' \
         -e 's#^[[:space:]]*browser\.documentRoot[[:space:]]*=.*#browser.documentRoot='"$dir"'/'"$HTDOCS_NAME"'#' \
         "$LIVE_CGI/hg.conf" > "$dir/cgi-bin/hg.conf"
     # Per-ticket marker: a densely tiled "RM NNNNN" watermark on the page
     # background, so it stays visible in the gaps around the content and it is
     # obvious which frozen instance you are looking at.  browser.style
     # (cart.c:2993) injects the stylesheet <link>; appended last so it wins.
     makeMarker "$tkt" "$dir"
     printf '\n# ts: per-ticket background marker (refs #37867)\nbrowser.style=/style/tsMarker.css\n' \
         >> "$dir/cgi-bin/hg.conf"
+    setUdcDir "$tkt" "$dir"
     echo "Freeze complete ($(du -sh "$dir/cgi-bin" "$dir/$HTDOCS_NAME" 2>/dev/null | awk '{print $1}' | paste -sd'+'))."
 }
 
 # --- generate the per-ticket background watermark image + stylesheet ---------
 # Small tile => the label repeats many times across the page, so at least some
 # copies land in the margins that the content does not cover.
 makeMarker() {
     local tkt="$1" dir="$2"
     local png="$dir/$HTDOCS_NAME/style/tsMarker.png"
     local css="$dir/$HTDOCS_NAME/style/tsMarker.css"
     if command -v convert >/dev/null 2>&1; then
         convert -size 150x70 xc:none -gravity center \
             -fill 'rgba(200,40,40,0.28)' -pointsize 17 -weight 700 \
             -annotate 0x0+0+0 "RM $tkt" "$png" 2>/dev/null || true
     fi
     cat > "$css" <<EOF
 /* ts marker for RM #$tkt - refs #37867 */
 body {
     background-image: url("/style/tsMarker.png") !important;
     background-repeat: repeat !important;
     background-attachment: fixed !important;
 }
 EOF
 }
 
+# --- give the ticket its own udc cache ---------------------------------------
+# udc caches remote bigData files by URL under ../trash/udcCache.  The live
+# sandbox and every parked ticket read and write the same tree, so a cache entry
+# written by one instance is read by all of them.  That breaks the freeze: a
+# sparse or bad entry from one build reappears under another.  Give each ticket
+# its own subtree.  It still lives in the shared trash, so the trash cleaner
+# ages it out like any other udc cache, and it is still data, not code: sync
+# does not clear it.  Appended to hg.conf, so it overrides the udc.cacheDir in
+# the included base config (the last assignment of a name wins).
+udcDir() { echo "$SHARED_TRASH/udcCache/ts/$1"; }
+
+setUdcDir() {
+    local tkt="$1" dir="$2"
+    mkdir -p "$(udcDir "$tkt")"
+    # freeze() writes hg.conf fresh, but conf() calls this on an existing one
+    grep -q '^# ts: per-ticket udc cache' "$dir/cgi-bin/hg.conf" && return 0
+    printf '\n# ts: per-ticket udc cache (refs #37867)\nudc.cacheDir=%s\n' \
+        "$(udcDir "$tkt")" >> "$dir/cgi-bin/hg.conf"
+}
+
 # --- render the per-ticket httpd.conf ----------------------------------------
 writeConf() {
     local tkt="$1" port="$2" dir; dir="$(tsDir "$1")"
     cat > "$dir/httpd.conf" <<EOF
 # Private loopback httpd for RM #$tkt - generated by ts, refs #37867
 ServerName localhost:$port
 ServerRoot "/etc/httpd"
 Listen 127.0.0.1:$port
 
 LoadModule mpm_prefork_module $MODDIR/mod_mpm_prefork.so
 LoadModule unixd_module $MODDIR/mod_unixd.so
 LoadModule authz_core_module $MODDIR/mod_authz_core.so
 LoadModule authz_host_module $MODDIR/mod_authz_host.so
 LoadModule mime_module $MODDIR/mod_mime.so
 LoadModule dir_module $MODDIR/mod_dir.so
@@ -269,30 +293,31 @@
     [[ -n "$running" ]] && startHttpd "$tkt" || echo "RM $tkt re-frozen (httpd was not running)."
 }
 
 # Rewrite httpd.conf from the current template without touching the frozen code.
 # "sync" also re-freezes, which is what you want after more work on the live
 # sandbox, but not when the freeze is the whole point of the park and only the
 # config template moved on.
 cmd_conf() {
     local tkt="$1"; validTkt "$tkt"
     local port; port="$(regPort "$tkt")"
     [[ -n "$port" ]] || die "RM $tkt not found in registry"
     [[ -d "$(tsDir "$tkt")" ]] || die "no sandbox directory for RM $tkt"
     local running=""; [[ -n "$(pidOf "$tkt")" ]] && running=1
     [[ -n "$running" ]] && stopHttpd "$tkt"
     writeConf "$tkt" "$port"
+    setUdcDir "$tkt" "$(tsDir "$tkt")"   # retrofit a sandbox frozen before this existed
     if [[ -n "$running" ]]; then
         startHttpd "$tkt"
     else
         echo "RM $tkt httpd.conf rewritten (httpd was not running)."
     fi
 }
 
 cmd_start()  { validTkt "$1"; startHttpd "$1"; }
 cmd_stop()   { validTkt "$1"; stopHttpd "$1"; }
 
 cmd_port() {   # print the port for a ticket (used by the laptop-side ts wrapper)
     local tkt="$1"; validTkt "$tkt"
     local port; port="$(regPort "$tkt")"
     [[ -n "$port" ]] || die "RM $tkt not found in registry"
     echo "$port"
@@ -322,30 +347,31 @@
     [[ -s "$REG" ]] || { echo "No parked tickets."; return 0; }
     printf '%-8s %-6s %-11s %-8s %s\n' RM PORT CREATED STATUS NOTE
     while IFS=$'\t' read -r tkt port created note; do
         [[ -z "$tkt" ]] && continue
         local status="stopped"; [[ -n "$(pidOf "$tkt")" ]] && status="running"
         printf '%-8s %-6s %-11s %-8s %s\n' "$tkt" "$port" "$created" "$status" "$note"
     done < "$REG"
 }
 
 cmd_remove() {
     local tkt="$1"; validTkt "$tkt"
     local port; port="$(regPort "$tkt")"
     [[ -n "$port" ]] || die "RM $tkt not found in registry"
     stopHttpd "$tkt" || true
     rm -rf "$(tsDir "$tkt")"     # trash is a symlink: removed, not followed
+    rm -rf "$(udcDir "$tkt")"    # the ticket's udc cache lives inside the shared trash
     # drop the registry row
     local tmp; tmp="$(mktemp)"
     awk -F'\t' -v r="$tkt" '$1!=r' "$REG" > "$tmp" && mv "$tmp" "$REG"
     echo "RM $tkt removed."
 }
 
 # --- dispatch ----------------------------------------------------------------
 [[ $# -ge 1 ]] || usage
 sub="$1"; shift || true
 case "$sub" in
     create) [[ $# -ge 1 ]] || usage; cmd_create "$@";;
     sync)   [[ $# -eq 1 ]] || usage; cmd_sync   "$1";;
     conf)   [[ $# -eq 1 ]] || usage; cmd_conf   "$1";;
     start)  [[ $# -eq 1 ]] || usage; cmd_start  "$1";;
     stop)   [[ $# -eq 1 ]] || usage; cmd_stop   "$1";;