4f9164cb6142e7468325253a67c6ac8a6a3d1f91 braney Tue Aug 25 09:54:31 2026 -0700 ts: give each ticket sandbox its own udc cache, refs #37867 Every parked instance and the live sandbox shared trash/udcCache, so a cache entry written by one instance was read by all of them. A sparse or bad entry from one build reappeared under another, which breaks the freeze. Each ticket now gets trash/udcCache/ts/NNNNN. It stays in the shared trash, so the trash cleaner ages it out like any other udc cache, and it stays data, so sync does not clear it. remove deletes it, since it sits inside the shared trash and not in the sandbox directory. conf now sets it too, so an instance frozen before this existed does not need a re-freeze. diff --git src/utils/ts/ts src/utils/ts/ts index 8ef2b7faa2e..3dca1c37e18 100755 --- src/utils/ts/ts +++ src/utils/ts/ts @@ -1,37 +1,40 @@ #!/bin/bash # # ts - "ticket sandbox": park a UCSC Genome Browser Redmine ticket as its own # frozen browser instance on hgwdev. refs #37867 # # A per-ticket Apache instance, owned by the developer, bound to a loopback high # port, reached by an ssh tunnel (yours or a colleague's - all hgwdev users # share the loopback, so anyone with an hgwdev account can tunnel in). The DB # and /gbdb are shared (hg.conf points at hgwdev MySQL; /gbdb is shared NFS) and # trash is shared with the live CGIs, so only the *code* is frozen: a full copy # of cgi-bin-$USER + htdocs-$USER plus a rewritten hg.conf. Data is left -# shared on purpose (freeze the code, share the data). +# shared on purpose (freeze the code, share the data). The one exception is the +# udc cache: each ticket gets its own subtree under the shared trash, so one +# instance cannot serve another a cache entry it wrote (see setUdcDir). # # The parked instances and the port registry live under $TS_ROOT, by default # $HOME/ticketSandboxes. On hgwdev, point that at a large local pool (a freeze # is a few GB): mkdir /data/home/$USER/ticketSandboxes and symlink it from # $HOME, or set TS_ROOT. # # Subcommands (NNNNN = Redmine ticket number): # create NNNNN [note] freeze the live sandbox, start an httpd, register it # sync NNNNN re-freeze a parked ticket to the current live sandbox -# conf NNNNN rewrite only httpd.conf, leaving the frozen code alone +# conf NNNNN rewrite httpd.conf and refresh the ts hg.conf settings, +# leaving the frozen code alone # start NNNNN start the ticket's httpd (e.g. after a reboot) # stop NNNNN stop the ticket's httpd # tunnel NNNNN open an ssh tunnel and print the browser URL # list show all parked tickets and their running status # remove NNNNN stop the httpd and delete the ticket sandbox # set -euo pipefail # --- configuration ---------------------------------------------------------- ROOT="$(readlink -f "${TS_ROOT:-$HOME/ticketSandboxes}")" # parked instances live here REG="$ROOT/ports.tsv" # ticket <TAB> port <TAB> created <TAB> note TS_USER="${USER:-$(id -un)}" # whose live sandbox we freeze LIVE_CGI=/usr/local/apache/cgi-bin-$TS_USER LIVE_HTDOCS=/usr/local/apache/htdocs-$TS_USER @@ -114,55 +117,76 @@ "$LIVE_HTDOCS/" "$dir/$HTDOCS_NAME/" # Rewritten, frozen hg.conf: # - include ../cgi-bin/hg.conf -> absolute (avoid self-include loop; the # shared base config is data-side and stays live by design) # - browser.documentRoot -> this ticket's frozen htdocs-$USER sed -e 's#^[[:space:]]*include[[:space:]]\+\.\./cgi-bin/hg\.conf#include '"$BASE_HGCONF"'#' \ -e 's#^[[:space:]]*browser\.documentRoot[[:space:]]*=.*#browser.documentRoot='"$dir"'/'"$HTDOCS_NAME"'#' \ "$LIVE_CGI/hg.conf" > "$dir/cgi-bin/hg.conf" # Per-ticket marker: a densely tiled "RM NNNNN" watermark on the page # background, so it stays visible in the gaps around the content and it is # obvious which frozen instance you are looking at. browser.style # (cart.c:2993) injects the stylesheet <link>; appended last so it wins. makeMarker "$tkt" "$dir" printf '\n# ts: per-ticket background marker (refs #37867)\nbrowser.style=/style/tsMarker.css\n' \ >> "$dir/cgi-bin/hg.conf" + setUdcDir "$tkt" "$dir" echo "Freeze complete ($(du -sh "$dir/cgi-bin" "$dir/$HTDOCS_NAME" 2>/dev/null | awk '{print $1}' | paste -sd'+'))." } # --- generate the per-ticket background watermark image + stylesheet --------- # Small tile => the label repeats many times across the page, so at least some # copies land in the margins that the content does not cover. makeMarker() { local tkt="$1" dir="$2" local png="$dir/$HTDOCS_NAME/style/tsMarker.png" local css="$dir/$HTDOCS_NAME/style/tsMarker.css" if command -v convert >/dev/null 2>&1; then convert -size 150x70 xc:none -gravity center \ -fill 'rgba(200,40,40,0.28)' -pointsize 17 -weight 700 \ -annotate 0x0+0+0 "RM $tkt" "$png" 2>/dev/null || true fi cat > "$css" <<EOF /* ts marker for RM #$tkt - refs #37867 */ body { background-image: url("/style/tsMarker.png") !important; background-repeat: repeat !important; background-attachment: fixed !important; } EOF } +# --- give the ticket its own udc cache --------------------------------------- +# udc caches remote bigData files by URL under ../trash/udcCache. The live +# sandbox and every parked ticket read and write the same tree, so a cache entry +# written by one instance is read by all of them. That breaks the freeze: a +# sparse or bad entry from one build reappears under another. Give each ticket +# its own subtree. It still lives in the shared trash, so the trash cleaner +# ages it out like any other udc cache, and it is still data, not code: sync +# does not clear it. Appended to hg.conf, so it overrides the udc.cacheDir in +# the included base config (the last assignment of a name wins). +udcDir() { echo "$SHARED_TRASH/udcCache/ts/$1"; } + +setUdcDir() { + local tkt="$1" dir="$2" + mkdir -p "$(udcDir "$tkt")" + # freeze() writes hg.conf fresh, but conf() calls this on an existing one + grep -q '^# ts: per-ticket udc cache' "$dir/cgi-bin/hg.conf" && return 0 + printf '\n# ts: per-ticket udc cache (refs #37867)\nudc.cacheDir=%s\n' \ + "$(udcDir "$tkt")" >> "$dir/cgi-bin/hg.conf" +} + # --- render the per-ticket httpd.conf ---------------------------------------- writeConf() { local tkt="$1" port="$2" dir; dir="$(tsDir "$1")" cat > "$dir/httpd.conf" <<EOF # Private loopback httpd for RM #$tkt - generated by ts, refs #37867 ServerName localhost:$port ServerRoot "/etc/httpd" Listen 127.0.0.1:$port LoadModule mpm_prefork_module $MODDIR/mod_mpm_prefork.so LoadModule unixd_module $MODDIR/mod_unixd.so LoadModule authz_core_module $MODDIR/mod_authz_core.so LoadModule authz_host_module $MODDIR/mod_authz_host.so LoadModule mime_module $MODDIR/mod_mime.so LoadModule dir_module $MODDIR/mod_dir.so @@ -269,30 +293,31 @@ [[ -n "$running" ]] && startHttpd "$tkt" || echo "RM $tkt re-frozen (httpd was not running)." } # Rewrite httpd.conf from the current template without touching the frozen code. # "sync" also re-freezes, which is what you want after more work on the live # sandbox, but not when the freeze is the whole point of the park and only the # config template moved on. cmd_conf() { local tkt="$1"; validTkt "$tkt" local port; port="$(regPort "$tkt")" [[ -n "$port" ]] || die "RM $tkt not found in registry" [[ -d "$(tsDir "$tkt")" ]] || die "no sandbox directory for RM $tkt" local running=""; [[ -n "$(pidOf "$tkt")" ]] && running=1 [[ -n "$running" ]] && stopHttpd "$tkt" writeConf "$tkt" "$port" + setUdcDir "$tkt" "$(tsDir "$tkt")" # retrofit a sandbox frozen before this existed if [[ -n "$running" ]]; then startHttpd "$tkt" else echo "RM $tkt httpd.conf rewritten (httpd was not running)." fi } cmd_start() { validTkt "$1"; startHttpd "$1"; } cmd_stop() { validTkt "$1"; stopHttpd "$1"; } cmd_port() { # print the port for a ticket (used by the laptop-side ts wrapper) local tkt="$1"; validTkt "$tkt" local port; port="$(regPort "$tkt")" [[ -n "$port" ]] || die "RM $tkt not found in registry" echo "$port" @@ -322,30 +347,31 @@ [[ -s "$REG" ]] || { echo "No parked tickets."; return 0; } printf '%-8s %-6s %-11s %-8s %s\n' RM PORT CREATED STATUS NOTE while IFS=$'\t' read -r tkt port created note; do [[ -z "$tkt" ]] && continue local status="stopped"; [[ -n "$(pidOf "$tkt")" ]] && status="running" printf '%-8s %-6s %-11s %-8s %s\n' "$tkt" "$port" "$created" "$status" "$note" done < "$REG" } cmd_remove() { local tkt="$1"; validTkt "$tkt" local port; port="$(regPort "$tkt")" [[ -n "$port" ]] || die "RM $tkt not found in registry" stopHttpd "$tkt" || true rm -rf "$(tsDir "$tkt")" # trash is a symlink: removed, not followed + rm -rf "$(udcDir "$tkt")" # the ticket's udc cache lives inside the shared trash # drop the registry row local tmp; tmp="$(mktemp)" awk -F'\t' -v r="$tkt" '$1!=r' "$REG" > "$tmp" && mv "$tmp" "$REG" echo "RM $tkt removed." } # --- dispatch ---------------------------------------------------------------- [[ $# -ge 1 ]] || usage sub="$1"; shift || true case "$sub" in create) [[ $# -ge 1 ]] || usage; cmd_create "$@";; sync) [[ $# -eq 1 ]] || usage; cmd_sync "$1";; conf) [[ $# -eq 1 ]] || usage; cmd_conf "$1";; start) [[ $# -eq 1 ]] || usage; cmd_start "$1";; stop) [[ $# -eq 1 ]] || usage; cmd_stop "$1";;