e132560389050652178d61d4a6006f86bbe5ea4c
maximilianh
  Mon Apr 28 16:13:27 2025 +0200
GBIC: detecting mysql directory in case the user moved it, after current ML ticket. working around OSX weirdness with /gbdb. No redmine yet

diff --git src/product/installer/browserSetup.sh src/product/installer/browserSetup.sh
index 299ba142036..809f337ae63 100755
--- src/product/installer/browserSetup.sh
+++ src/product/installer/browserSetup.sh
@@ -78,46 +78,30 @@
 
 # udr binary URL
 UDRURL=http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/udr
 
 # rsync is a variable so it can be be set to use udr
 RSYNC=rsync
 
 # by default, everything is downloaded
 RSYNCOPTS=""
 # alternative?
 # --include='*/' --exclude='wgEncode*.bam' hgdownload.soe.ucsc.edu::gbdb/hg19/ ./  -h
 
 # a flagfile to indicate that the cgis were installed successfully
 COMPLETEFLAG=/usr/local/apache/cgiInstallComplete.flag
 
-# on OSX by default we download a pre-compiled package that includes Apache/Mysql/OpenSSL
-# change this to 1 to rebuild all of these locally from tarballs
-BUILDEXT=${BUILDEXT:-0}
-# On OSX, by default download the CGIs as a binary package
-# change this to 1 to rebuild CGIs locally from tarball
-BUILDKENT=${BUILDKENT:-0}
-
-# URL of a tarball with a the binaries of Apache/Mysql/Openssl
-BINPKGURL=http://hgwdev.gi.ucsc.edu/~max/gbInstall/mysqlApacheOSX_10.7.tgz
-# URL of tarball with the OSX CGI binaries
-CGIBINURL=http://hgwdev.gi.ucsc.edu/~max/gbInstall/kentCgi_OSX_10.7.tgz 
-# URL of tarball with a minimal Mysql data directory
-MYSQLDBURL=http://hgwdev.gi.ucsc.edu/~max/gbInstall/mysql56Data.tgz
-# mysql/apache startup script URL, currently only for OSX
-STARTSCRIPTURL=https://raw.githubusercontent.com/maximilianh/browserInstall/master/browserStartup.sh
-
 # the -t option allows to download only the genome databases, not hgFixed/proteome/go/uniProt
 # by default, this is off, so we download hgFixed and Co. 
 ONLYGENOMES=0
 
 # make apt-get fail on warnings
 APTERR="-o APT::Update::Error-Mode=any"
 # ---- END GLOBAL DEFAULT SETTINGS ----
 
 # ---- DEFAULT CONFIG FILES ------------------
 # We need to initialize hg.conf and apache.conf for the browser.
 # They are inline so we do not create a dependency on another file 
 # that has to be pushed to hgdownload when this script is updated
 
 # the read command always has an error exit code for here docs, so we deactivate 
 # exit on error for a moment
@@ -899,34 +883,51 @@
 
 function installOsxDevTools () 
 # make sure that the xcode command line tools are installed
 {
    # check for xcode
    if [ -f /usr/bin/xcode-select 2> /dev/null > /dev/null ]; then
        echo2 Found XCode
    else
        echo2
        echo2 'This installer has to compile the UCSC tools locally on OSX. We need clang. Starting installation.'
        xcode-select --install 2> /dev/null >/dev/null
    fi
 }
 
 # OSX specific setup of the installation
+# NOT USED ANYMORE, but kept here for reference and for future use one day
 function installOsx () 
 {
    installOsxDevTools
 
+   # on OSX by default we download a pre-compiled package that includes Apache/Mysql/OpenSSL
+   # change this to 1 to rebuild all of these locally from tarballs
+   BUILDEXT=${BUILDEXT:-0}
+   # On OSX, by default download the CGIs as a binary package
+   # change this to 1 to rebuild CGIs locally from tarball
+   BUILDKENT=${BUILDKENT:-0}
+   
+   # URL of a tarball with a the binaries of Apache/Mysql/Openssl
+   BINPKGURL=http://hgwdev.gi.ucsc.edu/~max/gbInstall/mysqlApacheOSX_10.7.tgz
+   # URL of tarball with the OSX CGI binaries
+   CGIBINURL=http://hgwdev.gi.ucsc.edu/~max/gbInstall/kentCgi_OSX_10.7.tgz 
+   # URL of tarball with a minimal Mysql data directory
+   MYSQLDBURL=http://hgwdev.gi.ucsc.edu/~max/gbInstall/mysql56Data.tgz
+   # mysql/apache startup script URL, currently only for OSX
+   STARTSCRIPTURL=https://raw.githubusercontent.com/maximilianh/browserInstall/master/browserStartup.sh
+
    # in case that it is running, try to stop Apple's personal web server, we need access to port 80
    # ignore any error messages
    #if [ -f /usr/sbin/apachectl ]; then
        #echo2 Stopping the Apple Personal Web Server
        #/usr/sbin/apachectl stop 2> /dev/null || true
        #launchctl unload -w /System/Library/LaunchDaemons/org.apache.httpd.plist 2> /dev/null || true
    #fi
 
    if [ -f ~/.my.cnf ]; then
        echo2
        echo2 ~/.my.cnf already exists. This will interfere with the installation.
        echo2 The file will be renamed to ~/.my.cnf.old now.
        mv ~/.my.cnf ~/.my.cnf.old
        waitKey
    fi
@@ -1479,76 +1480,79 @@
       exit 100
    fi
 }
 
 # set this machine for browser development: install required tools, clone the tree, build it
 function buildTree () 
 {
    if [[ "$DIST" == "OSX" ]]; then
        setupBuildOsx
    else
        setupBuildLinux
    fi
 
    if [ ! -e ~/kent ]; then
       echo2 Cloning kent repo into ~/kent using git with --depth=1
+      echo2 Branch is: \"beta\" = our current release, beta = testing
       waitKey
       cd ~
-      git clone https://github.com/ucscGenomeBrowser/kent.git --depth=1
+      git clone -b beta https://github.com/ucscGenomeBrowser/kent.git --depth=1
    fi
 
    echo2 Now building CGIs from ~/kent to /usr/local/apache/cgi-bin 
    echo2 Copying JS/HTML/CSS to /usr/local/apache/htdocs
    waitKey
    cd ~/kent/src
    make -j8 cgi-alpha
    make -j8 doc-alpha
 }
 
 # main function, installs the browser on Redhat/Debian and potentially even on OSX
 function installBrowser () 
 {
     if [ -f $COMPLETEFLAG ]; then
         echo2 error: the file $COMPLETEFLAG exists. It seems that you have installed the browser already.
         echo2 If you want to reset the Apache directory, you can run '"rm -rf /usr/local/apache"' and 
         echo2 then run this script again.
         exit 100
     fi
 
     echo '--------------------------------'
     echo UCSC Genome Browser installation
     echo '--------------------------------'
-    echo CPU type: $MACH, detected OS: $OS/$DIST, Version: $VERNUM, Release: $VER
+    echo CPU type: $MACH, detected OS: $OS/$DIST, Release: $VER, Version: $VERNUM 
     echo 
     echo This script will go through three steps:
     echo "1 - setup apache and mysql, open port 80, deactivate SELinux"
     echo "2 - copy CGI binaries into $CGIBINDIR, html files into $HTDOCDIR"
     echo "3 - optional: download genome assembly databases into mysql and /gbdb"
     echo
     echo This script will now install and configure MariaDB and Apache if they are not yet installed. 
     echo "Your distribution's package manager will be used for this."
     echo If MariaDB is not installed yet, it will be installed, secured and a root password defined.
     echo The MariaDB root password will be written into root\'s \~/.my.cnf
     echo
     echo This script will also deactivate SELinux if active and open port 80/http.
     waitKey
 
     # -----  OS - SPECIFIC part -----
     if [ ! -f $COMPLETEFLAG ]; then
        if [[ "$DIST" == "OSX" ]]; then
-          #installOsx <- not used anymore, but maybe one day
- 	  # on OSX, install clang, brew, kent and build the CGIs from scratch
+ 	  echo2 OSX: build the CGIs from scratch using clang, brew and git
+          buildTree
+       elif [[ "$MACH" == "aarch64" ]]; then
+          echo2 Linux, but ARM CPU: Need to build CGIs and htdocs locally from source using gcc, make and git
           buildTree
        elif [[ "$DIST" == "debian" ]]; then
           installDebian
        elif [[ "$DIST" == "redhat" ]]; then
           installRedhat
        fi
     fi
     # OS-specific mysql/apache installers can SET_MYSQL_ROOT to 1 to request that the root
     # mysql user password be changed
 
     # ---- END OS-SPECIFIC part -----
 
     mysqlAllowOldPasswords
 
     if [[ "${SET_MYSQL_ROOT}" == "1" ]]; then
@@ -1638,79 +1642,98 @@
        echo genome-euro seems to be closer
        echo modifying hg.conf to pull data from genome-euro instead of genome
        sed -i s/slow-db.host=genome-mysql.soe.ucsc.edu/slow-db.host=genome-euro-mysql.soe.ucsc.edu/ $CGIBINDIR/hg.conf
        sed -i "s#gbdbLoc2=http://hgdownload.soe.ucsc.edu/gbdb/#gbdbLoc2=http://hgdownload-euro.soe.ucsc.edu/gbdb/#" $CGIBINDIR/hg.conf
        HGDOWNLOAD=hgdownload-euro.soe.ucsc.edu
     else
        echo genome.ucsc.edu seems to be closer
        echo not modifying $CGIBINDIR/hg.conf
     fi
 
     # download the CGIs
     if [[ "$OS" == "OSX" ]]; then
         #setupCgiOsx
         echo2 Running on OSX, assuming that CGIs are already built into /usr/local/apache/cgi-bin
     elif [[ "$MACH" == "aarch64" ]]; then
-        echo2 Running on an ARM CPU: need to build CGIs and htdocs locally from source
-        buildTree
+        echo2 Running on an ARM CPU, assuming that CGIs are already built into /usr/local/apache/cgi-bin
     else
         # don't download RNAplot, it's a 32bit binary that won't work anywhere anymore but at UCSC
         # this means that hgGene cannot show RNA structures but that's not a big issue
         $RSYNC -avzP --exclude=RNAplot $HGDOWNLOAD::cgi-bin/ $CGIBINDIR/
         # now add the binaries for dot and RNAplot 
         $RSYNC -avzP $HGDOWNLOAD::genome/admin/exe/external.x86_64/RNAplot $CGIBINDIR/
         $RSYNC -avzP $HGDOWNLOAD::genome/admin/exe/external.x86_64/loader/dot_static $CGIBINDIR/loader/
     fi
 
     # download the html docs, exclude some big files on OSX
     # try to minimize storage for OSX, mostly laptops
     if [ "$OS" == "OSX" -o "$MACH" == "aarch64" ]; then
             #$RSYNC --delete -azP --exclude=training --exclude=ENCODE --exclude=encode --exclude=rosenbloom.pdf --exclude=pubs*.pdf --exclude=*.{bb,bam,bai,bw,gz,2bit} --exclude=goldenpath $HGDOWNLOAD::htdocs/ $HTDOCDIR/
             echo2 Not syncing htdocs folder, assuming that these were built from source.
             echo2 PDF and other large files only present at UCSC will be missing from htdocs.
             waitKey
     else
             rm -rf $APACHEDIR/htdocs/goldenpath
             $RSYNC -avzP --exclude ENCODE/**.pdf $HGDOWNLOAD::htdocs/ $HTDOCDIR/
     fi
     
     # assign all files just downloaded to a valid user. 
     # This also allows apache to write into the trash dir
     if [ "$OS" == "OSX" ]; then
-        echo2 OSX: Not chowning /usr/local/apache subdirectories
+        echo2 OSX: Not chowning /usr/local/apache subdirectories, as not running as root
     else
         chown -R $APACHEUSER:$APACHEUSER $CGIBINDIR $HTDOCDIR $TRASHDIR
     fi
     
     touch $COMPLETEFLAG
 
     echo2 Install complete. You should now be able to point your web browser to this machine
     echo2 and test your UCSC Genome Browser mirror. It will be too slow for practical use.
     echo2
     echo2 Notice that this mirror is still configured to use Mysql and data files loaded
     echo2 through the internet from UCSC. From most locations on the world, this is very slow.
     echo2 It also requires an open outgoing TCP port 3306 for Mysql to genome-mysql.soe.ucsc.edu/genome-euro-mysql.soe.ucsc.edu,
     echo2 and open TCP port 80 to hgdownload.soe.ucsc.edu/hgdownload-euro.soe.ucsc.edu.
     echo2
     echo2 To finish the installation, you need to download genome data to the local
     echo2 disk. To download a genome assembly and all its files now, call this script again with
     echo2 the parameters 'download "<assemblyName1> <assemblyName2> ..."', e.g. '"'bash $0 mirror mm10 hg19'"'
     echo2 
     showMyAddress
     exit 0
 }
 
+# mkdir /gbdb or do the weird things one has to do on OSX to make this directory
+function mkdirGbdb 
+{
+    if [[ "$OS" != "OSX" ]]; then 
+       mkdir -p $GBDBDIR
+       return
+    fi
+
+    sudo mkdir -p /usr/local/gbdb
+    sudo chmod a+rwx /usr/local/gbdb
+
+    # see https://apple.stackexchange.com/questions/388236/unable-to-create-folder-in-root-of-macintosh-hd
+    if [[ ! -f /etc/synthetic.conf ]] || grep -vq gbdb /etc/synthetic.conf; then
+         sudo /bin/sh -c 'echo "gbdb\tusr/local/gbdb" >> /etc/synthetic.conf'
+    fi
+    chmod 644  /etc/synthetic.conf
+    /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -t
+    echo 'This directory is /usr/local/gbdb, see /etc/synthetic.conf' >> /gbdb/README.txt
+}
+
 # GENOME DOWNLOAD: mysql and /gbdb
 function downloadGenomes
 {
     DBS=$*
     GENBANKTBLS=""
     if [ "$DBS" == "" ] ; then
         echo2 Argument error: the '"download"' command requires at least one assembly name, like hg19 or mm10.
         exit 100
     fi
 
     echo2
     echo2 Downloading databases $DBS plus hgFixed/proteome/go from the UCSC download server
     echo2
     echo2 Determining download file size... please wait...
 
@@ -1720,30 +1743,36 @@
             MYSQLDBS="$DBS proteome uniProt go hgFixed"
         else
             echo2 Downloading $DBS plus GenBank and RefSeq tables
             MYSQLDBS="$DBS"
             GENBANKTBLS="author cell description development gbCdnaInfo gbExtFile gbLoaded \
                          gbMiscDiff gbSeq gbWarn geneName imageClone keyword library \
                          mrnaClone organism productName refLink refSeqStatus \
                          refSeqSummary sex source tissue"
         fi
     else
         MYSQLDBS="$DBS"
     fi
 
     # rsync is doing globbing itself, so switch it off temporarily
     set -f
+
+    # On OSX the MariaDB datadir is not under /var. On Linux distros, the MariaDB directory may have been moved.
+    if [ ! -d $MYSQLDIR ]; then 
+        MYSQLDIR=`mysql -NBe 'SHOW Variables WHERE Variable_Name="datadir"' | cut -f2`
+    fi
+
     # use rsync to get total size of files in directories and sum the numbers up with awk
     for db in $MYSQLDBS; do
         rsync -avn $HGDOWNLOAD::mysql/$db/ $MYSQLDIR/$db/ $RSYNCOPTS | grep ^'total size' | cut -d' ' -f4 | tr -d ', ' 
     done | awk '{ sum += $1 } END { print "| Required space in '$MYSQLDIR':", sum/1000000000, "GB" }'
     
     if [ ! -z "$GENBANKTBLS" ]; then
         for tbl in $GENBANKTBLS; do
             rsync -avn $HGDOWNLOAD::mysql/hgFixed/${tbl}.* $MYSQLDIR/hgFixed/ $RSYNCOPTS | grep ^'total size' | cut -d' ' -f4 | tr -d ', '
         done | awk '{ sum += $1 } END { print "| Required space in '$MYSQLDIR'/hgFixed:", sum/1000000000, "GB" }'
     fi
 
     for db in $DBS; do
         rsync -avn $HGDOWNLOAD::gbdb/$db/ $GBDBDIR/$db/ $RSYNCOPTS | grep ^'total size' | cut -d' ' -f4 | tr -d ','
     done | awk '{ sum += $1 } END { print "| Required space in '$GBDBDIR':", sum/1000000000, "GB" }'
 
@@ -1778,34 +1807,35 @@
        chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/$db
     done
 
     if [ ! -z "$GENBANKTBLS" ]; then
         echo2 Downloading hgFixed tables
         for tbl in $GENBANKTBLS; do
             $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/${tbl}.* $MYSQLDIR/hgFixed/
         done
         chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/hgFixed
     fi
 
     echo2 Downloading hgFixed.refLink, required for all RefSeq tracks
     $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/refLink.* $MYSQLDIR/hgFixed/ 
     chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/hgFixed
 
+    mkdirGbdb
+
     # download /gbdb files
     for db in $DBS; do
        echo2 Downloading $GBDBDIR files for assembly $db
-       mkdir -p $GBDBDIR
        $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::gbdb/$db/ $GBDBDIR/$db/
        chown -R $APACHEUSER:$APACHEUSER $GBDBDIR/$db
     done
 
     set +f
 
     # Alexander Stuy reported that at FSU they had a few mysql databases with incorrect users on them
     chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/
     
     startMysql
 
     mysqlCheck
 
     hideSomeTracks