b65520b87b2f80ac7fd537e8c361223d0a9dc72d max Thu Apr 9 01:42:08 2026 -0700 fixing various gbic issues found by Theo/Jorge and Claude code review, no redmine diff --git src/product/installer/browserSetup.sh src/product/installer/browserSetup.sh index b40d41d3a79..8b2aa33961c 100755 --- src/product/installer/browserSetup.sh +++ src/product/installer/browserSetup.sh @@ -12,31 +12,31 @@ # set all locale settings to English # searching for certain strings in command output might fail, if locale is different export LANG=C exitHandler() { if [ "$1" == "100" -o "$1" == "0" ] ; then exit $1 # all fine, a specific error message has already been output fi # somehow this script exited with an unknown type of error code echo Exit error $1 occurred on line $2 echo The UCSC Genome Browser installation script exited with an error. echo Please contact us at genome-mirror@soe.ucsc.edu and send us an output log echo of the command prefixed with '"bash -x"', e.g. - echo 'bash -x browserSetup.sh install 2>&1 > install.log' + echo 'bash -x browserSetup.sh install > install.log 2>&1' } # only trap the exit, not the errors # see https://medium.com/@dirk.avery/the-bash-trap-trap-ce6083f36700 trap 'exitHandler $? $LINENO' EXIT # ---- GLOBAL DEFAULT SETTINGS ---- # Directory where CGI-BIN and htdocs are downloaded to. # this is the main runtime directory of the genome browser. It contains # CGI binaries, the config file (hg.conf), temporary ("trash") files, downloaded pieces of big files # ("udcCache") and various other runtime data needed for the browser APACHEDIR=/usr/local/apache # apache document root, for html documents @@ -180,31 +180,31 @@ # from/return email address used for system emails # NOEMAIL means that user accounts are not validated on this machine # by sending email to users who have just signed up. # This is set as we cannot be sure if sendmail is working from this host # If you know that email is working, please change this login.mailReturnAddr=NOEMAIL # title of host of browser, this text be shown in the user interface of # the login/sign up screens login.browserName=UCSC Genome Browser Mirror # base url of browser installed login.browserAddr=http://127.0.0.1 # signature written at the bottom of hgLogin system emails login.mailSignature=None # the browser login page by default uses https. This setting can be used to -# used to make it work over http (not recommended) +# make it work over http (not recommended) login.https=off # Credentials to access the local mysql server db.host=localhost db.user=readonly db.password=access db.socket=/var/run/mysqld/mysqld.sock # db.port=3306 # The locations of the directory that holds file-based data # (e.g. alignments, database images, indexed bigBed files etc) # By default, this mirror can load missing files from the hgdownload server at UCSC # To disable on-the-fly loading of files, comment out these lines, # the slow-db.* section below and the showTableCache statement. gbdbLoc1=/gbdb/ @@ -454,35 +454,36 @@ # download file to stdout, use either wget or curl function downloadFileQuiet () { url=$1 if which wget 2> /dev/null > /dev/null; then wget -q $1 -O - else curl -s $1 fi } function errorHandler () { + local errCode=$? startMysql # in case we stopped it echo2 Error: the UCSC Genome Browser installation script failed with an error echo2 You can run it again with '"bash -x '$0'"' to see what failed. echo2 You can then send us an email with the error message. - exit $? + exit $errCode } # three types of data can be remote: mysql data and gbdb data # SHOW TABLES results can be cached remotely. All three are # deactivated with the following: function goOffline () { # first make sure we do not have them commented out already $SEDINPLACE 's/^#slow-db\./slow-db\./g' $APACHEDIR/cgi-bin/hg.conf $SEDINPLACE 's/^#gbdbLoc1=/gbdbLoc1=/g' $APACHEDIR/cgi-bin/hg.conf $SEDINPLACE 's/^#gbdbLoc2=/gbdbLoc2=/g' $APACHEDIR/cgi-bin/hg.conf $SEDINPLACE 's/^#showTableCache=/showTableCache=/g' $APACHEDIR/cgi-bin/hg.conf # now comment them out $SEDINPLACE 's/^slow-db\./#slow-db\./g' $APACHEDIR/cgi-bin/hg.conf @@ -506,62 +507,62 @@ { echo2 echo2 Press any key to continue or CTRL-C to abort. read -n 1 echo2 } # set MYCNF to the path to my.cnf function setMYCNF () { if [ -f /etc/my.cnf.d/mariadb-server.cnf ] ; then # Centos 8 stream: This has to be first, because /etc/my.cnf exists on Centos 8 Stream, but it contains a mysqld section # by default and somehow the section doesn't seem to take effect since a specific [mariadb] section also exists in the mariadb-server.cnf. # As a result, we only modify the mariadb-server config file MYCNF=/etc/my.cnf.d/mariadb-server.cnf - elif [ -f /etc/mysql/mariadb.conf.d/*-server.cnf ] ; then + elif ls /etc/mysql/mariadb.conf.d/*-server.cnf 1>/dev/null 2>&1 ; then # Ubuntu with mariadb. Must come before etc/my.cnf - MYCNF=/etc/mysql/mariadb.conf.d/*-server.cnf + MYCNF=$(ls /etc/mysql/mariadb.conf.d/*-server.cnf | head -1) elif [ -f /etc/mysql/mysql.conf.d/mysqld.cnf ] ; then # Ubuntu 16, 18, 20 with mysqld MYCNF=/etc/mysql/mysql.conf.d/mysqld.cnf elif [ -f /etc/my.cnf ] ; then # generic Centos 6-8 MYCNF=/etc/my.cnf elif [ -f /etc/mysql/my.cnf ] ; then # generic Ubuntu 14 MYCNF=/etc/mysql/my.cnf elif [ -f /usr/local/etc/my.cnf ]; then # brew on x86 MYCNF=/usr/local/etc/my.cnf elif [ -f /opt/homebrew/etc/my.cnf ]; then # homebrew on ARMs MYCNF=/opt/homebrew/etc/my.cnf else echo Could not find my.cnf. Adapt 'setMYCNF()' in browserSetup.sh and/or contact us. exit 1 fi echo Found Mariadb config file: $MYCNF } function mysqlStrictModeOff () { # make sure that missing values in mysql insert statements do not trigger errors, #18368 = deactivate strict mode # This must happen before Mariadb is started or alternative Mariadb must be restarted after this has been done setMYCNF echo Deactivating MySQL strict mode -sed -Ei '/^.(mysqld|server).$/a sql_mode=' $MYCNF +sed -Ei '/^\[(mysqld|server)\]$/a sql_mode=' $MYCNF } function mysqlAllowOldPasswords # mysql >= 8 does not allow the old passwords anymore. But our client is still compiled # with the old, non-SHA256 encryption. So we must deactivate this new feature. # We do not support MySQL anymore, so this function could be removed { echo2 'Checking for Mysql version >= 8 (not officially supported by our software)' MYSQLMAJ=`mysql -e 'SHOW VARIABLES LIKE "version";' -NB | cut -f2 | cut -d. -f1` setMYCNF if [ "$MYSQLMAJ" -ge 8 ] ; then echo2 'Mysql/MariaDB >= 8 found, checking if default-authentication allows native passwords' if grep -q default-authentication $MYCNF; then echo2 "default-authentication already set in $MYCNF" @@ -634,31 +635,31 @@ echo2 Downloading Precompiled OSX CGI-BIN binaries echo2 cd $APACHEDIR downloadFile $CGIBINURL | tar xz chown -R $APACHEUSER:$APACHEUSER cgi-bin/ return fi echo2 echo2 Now downloading the UCSC source tree into $APACHEDIR/kent echo2 The compiled binaries will be installed into $APACHEDIR/cgi-bin echo2 HTML files will be copied into $APACHEDIR/htdocs waitKey export MYSQLINC=$APACHEDIR/ext/include - export MYSQLLIBS="/$APACHEDIR/ext/lib/libmysqlclient.a -lz -lc++" + export MYSQLLIBS="$APACHEDIR/ext/lib/libmysqlclient.a -lz -lc++" export SSLDIR=$APACHEDIR/ext/include export PNGLIB=$APACHEDIR/ext/lib/libpng.a # careful - PNGINCL is the only option that requires the -I prefix export PNGINCL=-I$APACHEDIR/ext/include export CGI_BIN=$APACHEDIR/cgi-bin export SAMTABIXDIR=$APACHEDIR/kent/samtabix export USE_SAMTABIX=1 export SCRIPTS=$APACHEDIR/util export BINDIR=$APACHEDIR/util export PATH=$BINDIR:$PATH mkdir -p $APACHEDIR/util cd $APACHEDIR # get the kent src tree @@ -755,31 +756,31 @@ } # generic redhat part of mysql and apache installation, also covers centos and rocky function installRedhat () { enableRhelRepos echo2 echo2 Installing EPEL, ghostscript, libpng waitKey # make sure we have and EPEL and ghostscript and rsync (not installed on vagrant boxes) # imagemagick is required for the session gallery yum -y update # Fedora doesn't have or need EPEL, however, it does not include chkconfig by default - yum -y chkconfig install ghostscript rsync ImageMagick R-core curl initscripts --allowerasing --nobest + yum install -y chkconfig install ghostscript rsync ImageMagick R-core curl initscripts --allowerasing --nobest # centos 7 does not provide libpng by default if ldconfig -p | grep libpng12.so > /dev/null; then echo2 libpng12 found else yum -y install libpng12 fi # install apache if not installed yet if [ ! -f /usr/sbin/httpd ]; then echo2 echo2 Installing Apache and making it start on boot waitKey yum -y install httpd chkconfig # start apache on boot @@ -865,31 +866,31 @@ yum install -y python3-mysqlclient python3 python3-devel mariadb-connector-c mariadb-common mariadb-connector-c-devel wget gcc fi # open port 80 in firewall if which firewall-cmd ; then echo2 Opening port HTTP/80 in firewall using the command firewall-cmd sudo firewall-cmd --zone=public --permanent --add-service=http sudo firewall-cmd --reload fi } function installOsxDevTools () # make sure that the xcode command line tools are installed { # check for xcode - if [ -f /usr/bin/xcode-select 2> /dev/null > /dev/null ]; then + if [ -f /usr/bin/xcode-select ]; then echo2 Found XCode else echo2 echo2 'This installer has to compile the UCSC tools locally on OSX. We need clang. Starting installation.' xcode-select --install 2> /dev/null >/dev/null fi } # OSX specific setup of the installation # NOT USED ANYMORE, but kept here for reference and for future use one day function installOsx () { installOsxDevTools # on OSX by default we download a pre-compiled package that includes Apache/Mysql/OpenSSL @@ -1279,31 +1280,31 @@ exit 100 fi } # check if a program exists in the PATH function commandExists () { type "$1" &> /dev/null ; } # DETECT AND DEACTIVATE SELINUX: if it exists and is active function disableSelinux () { # first check if the command exists on this system # then check if it comes back with a non-zero error code, which means it is enabled if commandExists selinuxenabled; then - if [ selinuxenabled ]; then + if selinuxenabled; then echo2 echo2 The Genome Browser requires that SELINUX is deactivated. echo2 Deactivating it now with "'setenforce 0'" and in /etc/sysconfig/selinux. waitKey # deactivate selinux until next reboot # On Redhat 7.5 setenforce 0 returns error code 1 if selinux is disabled # so we simply ignore the error code here. Possibly we should use 'setstatus | grep 'Current mode' # instead of the error code of selinuxenabled above. selinux set +e setenforce 0 set -e # permanently deactivate after next reboot if [ -f /etc/sysconfig/selinux ]; then sed -i 's/^SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux fi @@ -1447,31 +1448,31 @@ mkdir -p /usr/local/apache/trash sudo chmod a+rwx /usr/local/apache/trash # cgis use ../trash to find the trash directory, but we have a symlink here sudo ln -fs /usr/local/apache/trash /Library/WebServer/Documents/trash sudo ln -fs /usr/local/apache/trash /Library/WebServer/trash # cgis use ../htdocs to find the htdocs directory, but we have a symlink here sudo ln -fs /Library/WebServer/Documents /Library/WebServer/htdocs fi # switch on CGIs in Apple's Apache and restart it (OSX as a BSD does not understand \s, so need to use [[:space:]]) sudo sed -Ei '' 's/^[[:space:]]*#(LoadModule cgid_module)/\1/; s/^[[:space:]]*#(LoadModule cgi_module)/\1/' /etc/apache2/httpd.conf # allow server side includes sudo sed -Ei '' 's/^[[:space:]]*#(LoadModule include_module)/\1/' /etc/apache2/httpd.conf # activate server side includes and make them depend on the X flag sudo sed -Ei '' 's/Options FollowSymLinks Multiviews/Options FollowSymLinks Includes Multiviews/' /etc/apache2/httpd.conf - if grep -qv XBitHack /etc/apache2/httpd.conf ; then + if ! grep -q XBitHack /etc/apache2/httpd.conf ; then sudo sed -Ei '' '/Options FollowSymLinks Includes Multiviews/ a\ XBitHack on\ SSILegacyExprParser on ' /etc/apache2/httpd.conf fi sudo /usr/sbin/apachectl restart } # install gcc, make etc so we can build the tree on linux function setupBuildLinux () { echo2 Installing required linux packages from repositories: Git, GCC, G++, MariaDB-client-libs, uuid, etc waitKey if [[ "$DIST" == "debian" ]]; then debianInitApt @@ -1578,31 +1579,31 @@ # we need to check if we can access it. # so test if we can connect to the mysql server checkCanConnectMysql disableSelinux # May 2025: our udr binaries do not seem to work on some recent distros anymore (ubuntu22+24). Segfault at start. # For now, switching off udr downloads. When you change this, also add back the -u flag documentation # in the HELP_STR. Uncomment the following line to reactivate udr support. # checkDownloadUdr # CGI DOWNLOAD AND HGCENTRAL MYSQL DB SETUP if [ ! -f $COMPLETEFLAG ]; then # test if an apache file is already present - if [ -f "$APACHEDIR" ]; then + if [ -e "$APACHEDIR" -a ! -d "$APACHEDIR" ]; then echo2 error: please remove the file $APACHEDIR, then restart the script with "bash $0". exit 100 fi fi # check if /usr/local/apache is empty # on OSX, we had to create an empty htdocs, so skip this check there if [ -d "$APACHEDIR" ] ; then if [ -f "$APACHEDIR"/.madeByBrowserBuild ] ; then # leave a flag file so the script knows later that this directory did not exist and we made it # and it is safe to write into it rm -f "$APACHEDIR"/.madeByBrowserBuild else echo2 error: the directory $APACHEDIR already exists. echo2 This installer has to overwrite it, so please move it to a different name @@ -1728,31 +1729,31 @@ function mkdirGbdb { if [[ "$OS" != "OSX" ]]; then # On Linux, create /gbdb and let the apache user write to it mkdir -p $GBDBDIR # Why? hgConvert will download missing liftOver files on the fly and needs write # write access chown $APACHEUSER:$APACHEUSER $GBDBDIR return fi sudo mkdir -p /usr/local/gbdb sudo chmod a+rwx /usr/local/gbdb # see https://apple.stackexchange.com/questions/388236/unable-to-create-folder-in-root-of-macintosh-hd - if [[ ! -f /etc/synthetic.conf ]] || grep -vq gbdb /etc/synthetic.conf; then + if [[ ! -f /etc/synthetic.conf ]] || ! grep -q gbdb /etc/synthetic.conf; then sudo /bin/sh -c 'echo "gbdb\tusr/local/gbdb" >> /etc/synthetic.conf' fi chmod 644 /etc/synthetic.conf /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -t || true if ! test -L /gbdb; then echo "warning: apfs.util failed to create /gbdb" fi echo 'This directory is /usr/local/gbdb, see /etc/synthetic.conf' >> /gbdb/README.txt } # GENOME DOWNLOAD: mysql and /gbdb function downloadGenomes { DBS=$* GENBANKTBLS="" @@ -1846,31 +1847,33 @@ $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/refLink.* $MYSQLDIR/hgFixed/ mkdirGbdb # download /gbdb files for db in $DBS; do echo2 Downloading $GBDBDIR files for assembly $db $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::gbdb/$db/ $GBDBDIR/$db/ done set +f # Alexander Stuy reported that at FSU they had a few mysql databases with incorrect users on them if [[ "$OS" != "OSX" ]]; then chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/ + for db in $DBS; do chown -R $APACHEUSER:$APACHEUSER $GBDBDIR/$db + done fi startMysql mysqlCheck hideSomeTracks goOffline # modify hg.conf and remove all statements that use the UCSC download server echo2 echo2 Install complete. You should now be able to point your web browser to this machine echo2 and use your UCSC Genome Browser mirror. echo2 @@ -2058,59 +2061,59 @@ } function updateBlatServers () { echo2 Creating or updating the BLAT servers table downloadFile http://$HGDOWNLOAD/admin/hgcentral.sql | $MYSQL hgcentral # the blat servers don't have fully qualified dom names in the download data $MYSQL hgcentral -e 'UPDATE blatServers SET host=CONCAT(host,".soe.ucsc.edu");' # just in case that we ever add fully qualified dom names $MYSQL hgcentral -e 'UPDATE blatServers SET host=replace(host,".soe.ucsc.edu.soe.ucsc.edu", ".soe.ucsc.edu");' } function cgiUpdate () { # update the CGIs - $RSYNC -avzP --exclude=RNAplot --exclude=hg.conf --exclude=hg.conf.local --exclude=RNAplot $HGDOWNLOAD::cgi-bin/ $APACHEDIR/cgi-bin/ + $RSYNC -avzP --exclude=RNAplot --exclude=hg.conf --exclude=hg.conf.local $HGDOWNLOAD::cgi-bin/ $APACHEDIR/cgi-bin/ # update the html docs echo2 Updating Apache htdocs $RSYNC -avzP --exclude=*.{bb,bam,bai,bw,gz,2bit,bed} --exclude=ENCODE --exclude=trash $HGDOWNLOAD::htdocs/ $APACHEDIR/htdocs/ # assign all downloaded files to a valid user. chown -R $APACHEUSER:$APACHEUSER $APACHEDIR/* updateBlatServers } function updateBrowser { echo cgiUpdate DBS=$* # if none specified, update all if [ "$DBS" == "" ] ; then DBS=`ls $GBDBDIR/` fi # update gbdb echo updating GBDB: $DBS for db in $DBS; do echo2 syncing gbdb: $db rsync -avp $RSYNCOPTS $HGDOWNLOAD::gbdb/$db/ $GBDBDIR/$db/ done # update the mysql DBs stopMysql - DBS=`ls /var/lib/mysql/ | egrep -v '(Trash$)|(hgTemp)|(^ib_)|(^ibdata)|(^aria)|(^mysql)|(performance)|(.flag$)|(multi-master.info)|(sys)|(lost.found)|(hgcentral)'` + DBS=`ls $MYSQLDIR/ | egrep -v '(Trash$)|(hgTemp)|(^ib_)|(^ibdata)|(^aria)|(^mysql)|(performance)|(.flag$)|(multi-master.info)|(sys)|(lost.found)|(hgcentral)'` for db in $DBS; do echo2 syncing full mysql database: $db $RSYNC --update --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/$db/ $MYSQLDIR/$db/ done startMysql mysqlCheck hideSomeTracks echo2 update finished } function addTools { rsync -avP hgdownload.soe.ucsc.edu::genome/admin/exe/linux.x86_64/ /usr/local/bin/ @@ -2131,31 +2134,31 @@ # On Debian and OSX, sudo by default does not update the HOME variable (hence the -H option above) if [[ "${SUDO_USER:-}" != "" ]]; then export HOME=~root fi trap errorHandler ERR # OPTION PARSING # show help message if no argument is specified if [[ $# -eq 0 ]] ; then echo "$HELP_STR" exit 0 fi -while getopts ":beut:hof" opt; do +while getopts ":but:hof" opt; do case $opt in h) echo "$HELP_STR" exit 0 ;; b) function waitKey { echo } ;; #a) #HGDOWNLOAD=hgdownload-euro.soe.ucsc.edu #;; t) val=${OPTARG}