de23dd389f8e8e35c41880a85d74e9838a4df8e4
angie
  Thu Jun 18 15:11:49 2026 -0700
ssh to hgwdev for commands that need access to hgwdev-only directories, because these scripts run elsewhere now.  Tweak shell variables usherDir and friends to make it easier to try out different build directories.

diff --git src/hg/utils/otto/mpxv/buildTree.sh src/hg/utils/otto/mpxv/buildTree.sh
index aafd231ec2b..fab2118ce73 100755
--- src/hg/utils/otto/mpxv/buildTree.sh
+++ src/hg/utils/otto/mpxv/buildTree.sh
@@ -31,34 +31,34 @@
 
 # Minimum number of samples in tree to be sure the NCBI download wasn't incomplete
 minSamples_I=600
 minSamples_II=7000
 
 # Download archive storage directory
 archiveRoot_I=/hive/users/angie/publicTreesMpxvCladeI
 archiveRoot_II=/hive/users/angie/publicTreesHMPXV
 
 # Download archive URL directory
 downloadDir_I=UShER_MPXV_cladeI
 downloadDir_II=UShER_hMPXV
 
 mpxvNcbiDir=$mpxvDir/ncbi/ncbi.latest
 
-usherDir=~angie/github/usher
-usherSampled=$usherDir/build/usher-sampled
-usher=$usherDir/build/usher
-matUtils=$usherDir/build/matUtils
+usherDir=~angie/github/usher/build
+usherSampled=$usherDir/usher-sampled
+matUtils=$usherDir/matUtils
+matOptimize=$usherDir/matOptimize
 
 if [[ ! -d $mpxvDir/ncbi/ncbi.$today || ! -s $mpxvDir/ncbi/ncbi.$today/genbank.fa.xz ]]; then
     mkdir -p $mpxvDir/ncbi/ncbi.$today
     $mpxvScriptDir/getNcbiMpxv.sh >& $mpxvDir/ncbi/ncbi.$today/getNcbiMpxv.log
 fi
 
 buildDir=$mpxvDir/build/$today
 mkdir -p $buildDir
 cd $buildDir
 
 # This builds the whole tree from scratch!  Eventually we'll want to add only the new sequences
 # to yesterday's tree.
 echo '()' > emptyTree.nwk
 
 for clade in I II; do
@@ -108,35 +108,35 @@
     time cat <(twoBitToFa $ref2bit stdout) <(xzcat $mpxvNcbiDir/nextalign.$clade.fa.xz) \
     | faToVcf -maxDiff=$maxDiff -verbose=2 -excludeFile=exclude.ids -includeNoAltN stdin stdout \
         | vcfRenameAndPrune stdin renaming.$clade.tsv stdout \
         | vcfFilter -excludeVcf=$mask stdin \
         | pigz -p 8 \
         > all.masked.vcf.gz
 
     time $usherSampled -T 16 -A -e 10 \
         -t emptyTree.nwk \
         -v all.masked.vcf.gz \
         -o mpxv.clade$clade.$today.masked.preOpt.pb.gz \
         --optimization_radius 0 --batch_size_per_process 10 \
         > usher.addNew.log 2>usher-sampled.stderr
 
     # Optimize:
-    time ~angie/github/usher_branch/build/matOptimize \
-        -T 16 -r 20 -M 2 -S move_log.usher_branch \
+    time $matOptimize \
+        -T 16 -r 20 -M 2 -S move_log \
         -i mpxv.clade$clade.$today.masked.preOpt.pb.gz \
         -o mpxv.clade$clade.$today.masked.opt.pb.gz \
-        >& matOptimize.usher_branch.log
+        >& matOptimize.log
     # It crashes when I add
     #    -v all.masked.vcf.gz \
     # -- bug Cheng later.
 
     if [[ $clade == "II" ]]; then
         # Annotate root nodes for Nextstrain lineages.
         join -t$'\t' <(sort renaming.$clade.tsv) <(zcat $mpxvNcbiDir/nextclade.$clade.tsv.gz | cut -f 2,5 | sort) \
         | tawk '{print $3, $2;}' | sort > lineageToName
         $matUtils annotate -T 16 -i mpxv.clade$clade.$today.masked.opt.pb.gz -c lineageToName \
             -o mpxv.clade$clade.$today.masked.pb.gz \
             >& annotate.$clade.log
 
         # Make metadata that uses same names as tree and includes nextclade lineage assignments.
         echo -e "strain\tgenbank_accession\tdate\tcountry\tlocation\tlength\thost\tbioproject_accession\tbiosample_accession\tsra_accession\tauthors\tpublications\tNextstrain_lineage" \
             > mpxv.clade$clade.$today.metadata.tsv
@@ -179,34 +179,34 @@
         columns=genbank_accession,location,date,authors,Nextstrain_clade
     else
         columns=genbank_accession,location,date,authors,Nextstrain_lineage
     fi
     usher_to_taxonium --input mpxv.clade$clade.$today.masked.pb.gz \
         --metadata mpxv.clade$clade.$today.metadata.tsv.gz \
         --genbank $gbff \
         --columns $columns \
         --clade_types=pango \
         --output mpxv.clade$clade.$today.masked.taxonium.jsonl.gz \
         >& usher_to_taxonium.log
 
     # Update links to latest protobuf and metadata in /gbdb directories
     nc=$(basename $gbff .gbff)
     dir=/gbdb/wuhCor1/hgPhyloPlaceData/mpxv/$nc
-    mkdir -p $dir
-    ln -sf $(pwd)/mpxv.clade$clade.$today.masked.pb.gz $dir/mpxv.clade$clade.latest.pb.gz
-    ln -sf $(pwd)/mpxv.clade$clade.$today.metadata.tsv.gz $dir/mpxv.clade$clade.latest.metadata.tsv.gz
-    ln -sf $(pwd)/hgPhyloPlace.description.$clade.txt $dir/mpxv.clade$clade.latest.version.txt
+    ssh hgwdev mkdir -p $dir
+    ssh hgwdev ln -sf $(pwd)/mpxv.clade$clade.$today.masked.pb.gz $dir/mpxv.clade$clade.latest.pb.gz
+    ssh hgwdev ln -sf $(pwd)/mpxv.clade$clade.$today.metadata.tsv.gz $dir/mpxv.clade$clade.latest.metadata.tsv.gz
+    ssh hgwdev ln -sf $(pwd)/hgPhyloPlace.description.$clade.txt $dir/mpxv.clade$clade.latest.version.txt
 
     # Extract Newick and VCF for anyone who wants to download those instead of protobuf
     $matUtils extract -i mpxv.clade$clade.$today.masked.pb.gz \
         -t mpxv.clade$clade.$today.nwk \
         -v mpxv.clade$clade.$today.masked.vcf
     pigz -p 8 -f mpxv.clade$clade.$today.nwk mpxv.clade$clade.$today.masked.vcf
 
     # Link to public trees download directory hierarchy
     read y m d < <(echo $today | sed -re 's/-/ /g')
     archive=$archiveRoot/$y/$m/$d
     mkdir -p $archive
     if [[ $clade == "I" ]]; then
         ln -f $(pwd)/mpxv.clade$clade.$today.{nwk,masked.vcf,metadata.tsv,masked.taxonium.jsonl,masked.pb}.gz $archive/
         ln -f $(pwd)/hgPhyloPlace.description.$clade.txt $archive/mpxv.clade$clade.$today.version.txt
     else
@@ -215,35 +215,34 @@
         # in the name wouldn't be entirely accurate either.
         for f in mpxv.clade$clade.$today.{nwk,masked.vcf,metadata.tsv,masked.taxonium.jsonl,masked.pb}.gz; do
             downloadF=$(echo $f | sed -re 's/.cladeII//;')
             ln -f $(pwd)/$f $archive/$downloadF
         done
         ln -f $(pwd)/hgPhyloPlace.description.$clade.txt $archive/mpxv.$today.version.txt
     fi
 
     # Update 'latest' in $archiveRoot
     for f in $archive/mpxv*.$today.*; do
         latestF=$(echo $(basename $f) | sed -re 's/'$today'/latest/')
         ln -f $f $archiveRoot/$latestF
     done
 
     # Update hgdownload-test link for archive
-    mkdir -p /usr/local/apache/htdocs-hgdownload/hubs/$asmDir/$downloadDir/$y/$m
-    ln -sf $archive /usr/local/apache/htdocs-hgdownload/hubs/$asmDir/$downloadDir/$y/$m
+    ssh hgwdev ln -sf $archiveRoot/*.latest.* /data/apache/htdocs-hgdownload/hubs/$asmDir/$downloadDir/
     # rsync to hgdownload hubs dir
-    for h in hgdownload1 hgdownload3; do
-        if rsync -a -L --delete /usr/local/apache/htdocs-hgdownload/hubs/$asmDir/$downloadDir/* \
+    for h in hgdownload1 hgdownload2 hgdownload3; do
+        if ssh hgwdev rsync -a -L --delete /data/apache/htdocs-hgdownload/hubs/$asmDir/$downloadDir/* \
                  qateam@$h:/mirrordata/hubs/$asmDir/$downloadDir/; then
             true
         else
             echo ""
             echo "*** rsync to $h failed; disk full? ***"
             echo ""
         fi
     done
 
     if [[ $clade == "II" ]]; then
         set +o pipefail
         grep 'Could not' annotate.$clade.log | cat
         grep skipping annotate.$clade.log | cat
         set -o pipefail
     fi