bee41315e53da6fb5671f96302233c655906dfd5
max
  Wed Sep 9 06:56:49 2026 -0700
UniProt otto: rebuild the venv rather than relying on a system lxml

Correction to the previous commit: hgwdev has no system-wide lxml at all. The
import I tested was resolving to my own ~/.local/lib/python3.9/site-packages,
which cron never sees, because it runs the pipeline as otto.

So the environment is a virtualenv again, but a reproducible one. makeVenv.sh
deletes venv/ and rebuilds it from /usr/bin/python3, installs lxml, opens up the
permissions for otto, and then checks that lxml imports with an empty environment
so we know the venv stands on its own instead of borrowing from whoever ran it.
Built with --copies, so venv/bin/python is a real copy rather than a symlink that
would silently follow a system python upgrade while its compiled modules stayed
behind.

doUpdate.sh activates venv/ again and says to run makeVenv.sh if it is missing or
if the parser will not start. Verified: /usr/bin/python3 without per-user packages
cannot import lxml, the venv can, and after activation the parser runs and
converts real 2026_02 records.

Also shortened the README to how the pipeline is started and how it works, and
trimmed the history out of the code comments, leaving the ticket as the pointer.

refs #38300

diff --git src/hg/utils/otto/uniprot/doUpdate.sh src/hg/utils/otto/uniprot/doUpdate.sh
index 8b9f14220c8..67c7dbdaad7 100755
--- src/hg/utils/otto/uniprot/doUpdate.sh
+++ src/hg/utils/otto/uniprot/doUpdate.sh
@@ -1,47 +1,49 @@
 #!/bin/sh
 # configuration setup and cron wrapper for the doUniprot script
 
 cd /hive/data/outside/otto/uniprot || exit 1
 umask 002
 
 #echo WARNING: NOT DOWNLOADING
 #./doUniprot run --skipDownload
 
-# There is deliberately no virtualenv here anymore. uniprotToTab needs the lxml XML
-# parser, which on hgwdev comes from the system package python3-lxml and is upgraded
-# together with /usr/bin/python3. A virtualenv used to sit in venv/ instead, but its
-# python was only a symlink to /usr/bin/python3: when the system python moved from 3.6
-# to 3.9 the compiled lxml in the venv stopped loading, every monthly run died at the
-# parse step, and the tracks stayed on release 2024_06 for 19 months (redmine #38300).
-
 runLog=runLog.txt
 
 logRun() {
     echo "`date '+%Y-%m-%d %H:%M:%S'` $*" >> $runLog
 }
 
+# activate the python environment that has the lxml XML parser. Rebuild it with
+# ./makeVenv.sh if this fails.
+if [ ! -f venv/bin/activate ] ; then
+    logRun "PREFLIGHT-FAIL no venv"
+    echo "UniProt update did not start: venv/bin/activate is missing."
+    echo "Rebuild it with: cd /hive/data/outside/otto/uniprot && ./makeVenv.sh"
+    exit 1
+fi
+. venv/bin/activate
+
 # Do not spend 35 minutes downloading UniProt only to find out that the parser cannot
 # start. Run it with --help, which imports lxml and then exits, and stop here if that
 # fails. Invoked exactly the way doUniprot invokes it, so this tests the same python.
 if ! ./uniprotToTab --help > /dev/null 2>&1; then
     logRun "PREFLIGHT-FAIL uniprotToTab cannot start"
     echo "UniProt update did not start: ./uniprotToTab cannot be run."
     echo
-    echo "Almost certainly the lxml python module is missing. Check with:"
-    echo "    python3 -c 'import lxml.etree'"
-    echo "and see /hive/data/outside/otto/uniprot/README.txt for how to repair it."
+    echo "The lxml python module does not import. Rebuild the environment with:"
+    echo "    cd /hive/data/outside/otto/uniprot && ./makeVenv.sh"
     echo
     ./uniprotToTab --help 2>&1 | tail -20
     exit 1
 fi
 
 logRun "START"
 ./doUniprot run > lastRun.log 2>&1
 exitCode=$?
 logRun "END exit=$exitCode"
 
 if [ $exitCode -ne 0 ] ; then
     # lastRun.log is overwritten by the next run, so keep a copy. Without one, a
     # failure that nobody reads leaves no trace on disk at all.
     cp -f lastRun.log lastFail.log
     logRun "FAIL exit=$exitCode log=lastFail.log"