4ae50fa5255be82566187f8cdbf5591152d3cf34 braney Sat Aug 22 14:49:40 2026 -0700 ts: add the ticket-sandbox scripts, refs #37867 ts parks a Redmine ticket as its own frozen browser instance on hgwdev: a full copy of cgi-bin-$USER and htdocs-$USER served by a private loopback httpd, with the databases, /gbdb, and trash left shared. Generalized from the personal version: per-user sandbox paths, TS_ROOT for the instance directory, and existence checks in create. diff --git src/utils/ts/ts src/utils/ts/ts new file mode 100755 index 00000000000..02c9bd36f60 --- /dev/null +++ src/utils/ts/ts @@ -0,0 +1,328 @@ +#!/bin/bash +# +# ts - "ticket sandbox": park a UCSC Genome Browser Redmine ticket as its own +# frozen browser instance on hgwdev. refs #37867 +# +# A per-ticket Apache instance, owned by the developer, bound to a loopback high +# port, reached by an ssh tunnel (yours or a colleague's - all hgwdev users +# share the loopback, so anyone with an hgwdev account can tunnel in). The DB +# and /gbdb are shared (hg.conf points at hgwdev MySQL; /gbdb is shared NFS) and +# trash is shared with the live CGIs, so only the *code* is frozen: a full copy +# of cgi-bin-$USER + htdocs-$USER plus a rewritten hg.conf. Data is left +# shared on purpose (freeze the code, share the data). +# +# The parked instances and the port registry live under $TS_ROOT, by default +# $HOME/ticketSandboxes. On hgwdev, point that at a large local pool (a freeze +# is a few GB): mkdir /data/home/$USER/ticketSandboxes and symlink it from +# $HOME, or set TS_ROOT. +# +# Subcommands (NNNNN = Redmine ticket number): +# create NNNNN [note] freeze the live sandbox, start an httpd, register it +# sync NNNNN re-freeze a parked ticket to the current live sandbox +# start NNNNN start the ticket's httpd (e.g. after a reboot) +# stop NNNNN stop the ticket's httpd +# tunnel NNNNN open an ssh tunnel and print the browser URL +# list show all parked tickets and their running status +# remove NNNNN stop the httpd and delete the ticket sandbox +# +set -euo pipefail + +# --- configuration ---------------------------------------------------------- +ROOT="$(readlink -f "${TS_ROOT:-$HOME/ticketSandboxes}")" # parked instances live here +REG="$ROOT/ports.tsv" # ticket <TAB> port <TAB> created <TAB> note + +TS_USER="${USER:-$(id -un)}" # whose live sandbox we freeze +LIVE_CGI=/usr/local/apache/cgi-bin-$TS_USER +LIVE_HTDOCS=/usr/local/apache/htdocs-$TS_USER +HTDOCS_NAME="$(basename "$LIVE_HTDOCS")" # frozen copy keeps the live name +SHARED_HTDOCS=/usr/local/apache/htdocs # sibling ../htdocs the CGIs read for fonts etc. +SHARED_TRASH=/usr/local/apache/trash # trash is data: shared with the live CGIs +BASE_HGCONF=/usr/local/apache/cgi-bin/hg.conf + +HTTPD=/usr/sbin/httpd +MODDIR=modules # relative to ServerRoot /etc/httpd +PORT_BASE=48080 +HGWDEV="${HGWDEV:-hgwdev.gi.ucsc.edu}" + +# --- helpers ----------------------------------------------------------------- +die() { echo "ts: $*" >&2; exit 1; } + +usage() { # print the header comment block, minus the shebang + awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "${BASH_SOURCE[0]}" + exit 1 +} + +validTkt() { [[ "$1" =~ ^[0-9]+$ ]] || die "ticket must be numeric, got '$1'"; } + +tsDir() { echo "$ROOT/$1"; } + +regPort() { # regPort NNNNN -> port, or empty + [[ -f "$REG" ]] || return 0 + awk -F'\t' -v r="$1" '$1==r{print $2}' "$REG" +} + +portInUse() { # portInUse PORT -> 0 if listening + local ss; ss="$(command -v ss || echo /usr/sbin/ss)" + "$ss" -ltn 2>/dev/null | awk '{print $4}' | grep -qE "[:.]$1\$" +} + +nextPort() { + local p="$PORT_BASE" + while :; do + if [[ -f "$REG" ]] && awk -F'\t' -v p="$p" '$2==p{f=1} END{exit !f}' "$REG"; then + p=$((p+1)); continue + fi + portInUse "$p" && { p=$((p+1)); continue; } + echo "$p"; return 0 + done +} + +pidOf() { # pidOf NNNNN -> pid if httpd.pid exists and process alive + local dir; dir="$(tsDir "$1")" + local pf="$dir/httpd.pid" + [[ -f "$pf" ]] || return 0 + local pid; pid="$(cat "$pf" 2>/dev/null)" + [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && echo "$pid" +} + +# --- freeze the live sandbox into the ticket dir (used by create and sync) --- +freeze() { + local tkt="$1" dir; dir="$(tsDir "$tkt")" + echo "Freezing live sandbox into $dir ..." + mkdir -p "$dir/cgi-bin" "$dir/$HTDOCS_NAME" "$dir/logs" + # Mirror the live directory relationships so CWD-relative reads behave as on + # the live sandbox. The CGIs run with CWD = $dir/cgi-bin and read two htdocs: + # - DocumentRoot / browser.documentRoot -> $dir/htdocs-$USER (frozen web + # content: js, style, description pages) + # - ../htdocs (sibling of cgi-bin) -> the SHARED htdocs, for CGI-internal + # reads like urw-fonts. Freeze the code, not the shared static font data. + [[ -L "$dir/htdocs" || ! -e "$dir/htdocs" ]] || rm -rf "$dir/htdocs" + ln -sfn "$SHARED_HTDOCS" "$dir/htdocs" + # trash is data, shared with the live CGIs (custom tracks, uploaded files, + # sessions, cache): the CGIs write to ../trash relative to their CWD = + # $dir/cgi-bin, so point that at the live shared trash via a symlink. The + # shared trash is world-writable (0777, setgid apache), so the sandbox httpd + # running as the developer can write it. Freeze the code, share the data. + [[ -L "$dir/trash" || ! -e "$dir/trash" ]] || rm -rf "$dir/trash" # drop any old private trash dir + ln -sfn "$SHARED_TRASH" "$dir/trash" + # Full copy (real freeze, not hardlinks). Exclude the big old/ backup dir. + # hg.conf is generated fresh below, so exclude it from the copy. + rsync -a --delete --exclude 'old/' --exclude '/hg.conf' \ + "$LIVE_CGI/" "$dir/cgi-bin/" + rsync -a --delete \ + "$LIVE_HTDOCS/" "$dir/$HTDOCS_NAME/" + # Rewritten, frozen hg.conf: + # - include ../cgi-bin/hg.conf -> absolute (avoid self-include loop; the + # shared base config is data-side and stays live by design) + # - browser.documentRoot -> this ticket's frozen htdocs-$USER + sed -e 's#^[[:space:]]*include[[:space:]]\+\.\./cgi-bin/hg\.conf#include '"$BASE_HGCONF"'#' \ + -e 's#^[[:space:]]*browser\.documentRoot[[:space:]]*=.*#browser.documentRoot='"$dir"'/'"$HTDOCS_NAME"'#' \ + "$LIVE_CGI/hg.conf" > "$dir/cgi-bin/hg.conf" + # Per-ticket marker: a densely tiled "RM NNNNN" watermark on the page + # background, so it stays visible in the gaps around the content and it is + # obvious which frozen instance you are looking at. browser.style + # (cart.c:2993) injects the stylesheet <link>; appended last so it wins. + makeMarker "$tkt" "$dir" + printf '\n# ts: per-ticket background marker (refs #37867)\nbrowser.style=/style/tsMarker.css\n' \ + >> "$dir/cgi-bin/hg.conf" + echo "Freeze complete ($(du -sh "$dir/cgi-bin" "$dir/$HTDOCS_NAME" 2>/dev/null | awk '{print $1}' | paste -sd'+'))." +} + +# --- generate the per-ticket background watermark image + stylesheet --------- +# Small tile => the label repeats many times across the page, so at least some +# copies land in the margins that the content does not cover. +makeMarker() { + local tkt="$1" dir="$2" + local png="$dir/$HTDOCS_NAME/style/tsMarker.png" + local css="$dir/$HTDOCS_NAME/style/tsMarker.css" + if command -v convert >/dev/null 2>&1; then + convert -size 150x70 xc:none -gravity center \ + -fill 'rgba(200,40,40,0.28)' -pointsize 17 -weight 700 \ + -annotate 0x0+0+0 "RM $tkt" "$png" 2>/dev/null || true + fi + cat > "$css" <<EOF +/* ts marker for RM #$tkt - refs #37867 */ +body { + background-image: url("/style/tsMarker.png") !important; + background-repeat: repeat !important; + background-attachment: fixed !important; +} +EOF +} + +# --- render the per-ticket httpd.conf ---------------------------------------- +writeConf() { + local tkt="$1" port="$2" dir; dir="$(tsDir "$1")" + cat > "$dir/httpd.conf" <<EOF +# Private loopback httpd for RM #$tkt - generated by ts, refs #37867 +ServerName localhost:$port +ServerRoot "/etc/httpd" +Listen 127.0.0.1:$port + +LoadModule mpm_prefork_module $MODDIR/mod_mpm_prefork.so +LoadModule unixd_module $MODDIR/mod_unixd.so +LoadModule authz_core_module $MODDIR/mod_authz_core.so +LoadModule authz_host_module $MODDIR/mod_authz_host.so +LoadModule mime_module $MODDIR/mod_mime.so +LoadModule dir_module $MODDIR/mod_dir.so +LoadModule alias_module $MODDIR/mod_alias.so +LoadModule rewrite_module $MODDIR/mod_rewrite.so +LoadModule headers_module $MODDIR/mod_headers.so +LoadModule cgi_module $MODDIR/mod_cgi.so +LoadModule log_config_module $MODDIR/mod_log_config.so + +TypesConfig /etc/mime.types +DirectoryIndex index.html + +PidFile "$dir/httpd.pid" +ErrorLog "$dir/logs/error_log" +LogFormat "%h %l %u %t \"%r\" %>s %b" common +CustomLog "$dir/logs/access_log" common + +DocumentRoot "$dir/$HTDOCS_NAME" +ScriptAlias /cgi-bin/ "$dir/cgi-bin/" +Alias /trash/ "$SHARED_TRASH/" + +Header set Access-Control-Allow-Origin "*" + +<Directory "$dir/cgi-bin"> + AllowOverride None + Options +ExecCGI +FollowSymLinks + Require all granted +</Directory> +<Directory "$dir/$HTDOCS_NAME"> + AllowOverride None + Options +FollowSymLinks +Includes + Require all granted +</Directory> +<Directory "$SHARED_TRASH"> + AllowOverride None + Options +FollowSymLinks + Require all granted +</Directory> +EOF +} + +# --- lifecycle --------------------------------------------------------------- +startHttpd() { + local tkt="$1" dir; dir="$(tsDir "$1")" + [[ -f "$dir/httpd.conf" ]] || die "no httpd.conf for RM $tkt (create it first)" + if [[ -n "$(pidOf "$tkt")" ]]; then + echo "RM $tkt httpd already running (pid $(pidOf "$tkt"))."; return 0 + fi + "$HTTPD" -f "$dir/httpd.conf" -t >/dev/null # syntax check first + "$HTTPD" -f "$dir/httpd.conf" + sleep 1 + local pid; pid="$(pidOf "$tkt")" + [[ -n "$pid" ]] || { tail -5 "$dir/logs/error_log" >&2; die "httpd failed to start for RM $tkt"; } + echo "RM $tkt httpd started (pid $pid)." +} + +stopHttpd() { + local tkt="$1" dir; dir="$(tsDir "$1")" + local pid; pid="$(pidOf "$tkt")" + if [[ -z "$pid" ]]; then echo "RM $tkt httpd not running."; return 0; fi + kill "$pid" 2>/dev/null || true + sleep 1 + [[ -z "$(pidOf "$tkt")" ]] && echo "RM $tkt httpd stopped." || die "RM $tkt httpd did not stop (pid $pid)" +} + +# --- subcommands ------------------------------------------------------------- +cmd_create() { + local tkt="$1"; shift || true + local note="${*:-}" + validTkt "$tkt" + [[ -d "$LIVE_CGI" ]] || die "no live sandbox at $LIVE_CGI" + [[ -d "$LIVE_HTDOCS" ]] || die "no live htdocs at $LIVE_HTDOCS" + mkdir -p "$ROOT" + [[ -z "$(regPort "$tkt")" ]] || die "RM $tkt already exists (port $(regPort "$tkt")); use sync/remove" + local port; port="$(nextPort)" + freeze "$tkt" + writeConf "$tkt" "$port" + printf '%s\t%s\t%s\t%s\n' "$tkt" "$port" "$(date +%Y-%m-%d)" "$note" >> "$REG" + startHttpd "$tkt" + echo + echo "RM $tkt parked on 127.0.0.1:$port" + echo " on hgwdev: curl 'http://127.0.0.1:$port/cgi-bin/hgTracks?db=hg38'" + echo " you, remote: ts tunnel $tkt -> http://localhost:$port/cgi-bin/hgTracks" + echo " a colleague: ssh -N -L $port:localhost:$port <user>@$HGWDEV -> http://localhost:$port/cgi-bin/hgTracks" +} + +cmd_sync() { + local tkt="$1"; validTkt "$tkt" + local port; port="$(regPort "$tkt")" + [[ -n "$port" ]] || die "RM $tkt not found in registry" + local running=""; [[ -n "$(pidOf "$tkt")" ]] && running=1 + [[ -n "$running" ]] && stopHttpd "$tkt" + freeze "$tkt" + writeConf "$tkt" "$port" # regenerate in case template changed + [[ -n "$running" ]] && startHttpd "$tkt" || echo "RM $tkt re-frozen (httpd was not running)." +} + +cmd_start() { validTkt "$1"; startHttpd "$1"; } +cmd_stop() { validTkt "$1"; stopHttpd "$1"; } + +cmd_port() { # print the port for a ticket (used by the laptop-side ts wrapper) + local tkt="$1"; validTkt "$tkt" + local port; port="$(regPort "$tkt")" + [[ -n "$port" ]] || die "RM $tkt not found in registry" + echo "$port" +} + +cmd_tunnel() { + local tkt="$1"; validTkt "$tkt" + local port; port="$(regPort "$tkt")" + [[ -n "$port" ]] || die "RM $tkt not found in registry" + cat <<EOF +Opening ssh tunnel: localhost:$port -> $HGWDEV RM $tkt sandbox. +While this terminal stays open, point your browser at: + + http://localhost:$port/cgi-bin/hgTracks + +To let a colleague reach it from their laptop, have them run (with their own +hgwdev username) and then open the same URL: + + ssh -N -L $port:localhost:$port <user>@$HGWDEV + +Leave this window running. Ctrl-C closes the tunnel. +EOF + exec ssh -N -L "$port:localhost:$port" "$HGWDEV" +} + +cmd_list() { + [[ -s "$REG" ]] || { echo "No parked tickets."; return 0; } + printf '%-8s %-6s %-11s %-8s %s\n' RM PORT CREATED STATUS NOTE + while IFS=$'\t' read -r tkt port created note; do + [[ -z "$tkt" ]] && continue + local status="stopped"; [[ -n "$(pidOf "$tkt")" ]] && status="running" + printf '%-8s %-6s %-11s %-8s %s\n' "$tkt" "$port" "$created" "$status" "$note" + done < "$REG" +} + +cmd_remove() { + local tkt="$1"; validTkt "$tkt" + local port; port="$(regPort "$tkt")" + [[ -n "$port" ]] || die "RM $tkt not found in registry" + stopHttpd "$tkt" || true + rm -rf "$(tsDir "$tkt")" # trash is a symlink: removed, not followed + # drop the registry row + local tmp; tmp="$(mktemp)" + awk -F'\t' -v r="$tkt" '$1!=r' "$REG" > "$tmp" && mv "$tmp" "$REG" + echo "RM $tkt removed." +} + +# --- dispatch ---------------------------------------------------------------- +[[ $# -ge 1 ]] || usage +sub="$1"; shift || true +case "$sub" in + create) [[ $# -ge 1 ]] || usage; cmd_create "$@";; + sync) [[ $# -eq 1 ]] || usage; cmd_sync "$1";; + start) [[ $# -eq 1 ]] || usage; cmd_start "$1";; + stop) [[ $# -eq 1 ]] || usage; cmd_stop "$1";; + tunnel) [[ $# -eq 1 ]] || usage; cmd_tunnel "$1";; + port) [[ $# -eq 1 ]] || usage; cmd_port "$1";; + list) cmd_list;; + remove) [[ $# -eq 1 ]] || usage; cmd_remove "$1";; + -h|--help|help) usage;; + *) die "unknown subcommand '$sub' (try: create sync start stop tunnel list remove)";; +esac