fa5cb52522742a54651d302464cb5deadfa7b7c7 max Mon May 18 02:54:46 2026 -0700 andy smith noticed that on the most recent ubuntu the mysql datadir was moved. Adding code now to auto-detect the datadir location and running the function after every use of MYSQLDIR. Except for OSX, but OSX without brew is not really supported anymore and needs work anyways. No redmine diff --git src/product/installer/browserSetup.sh src/product/installer/browserSetup.sh index 8952c05a8f4..17d7db5b46e 100755 --- src/product/installer/browserSetup.sh +++ src/product/installer/browserSetup.sh @@ -35,35 +35,37 @@ # Directory where CGI-BIN and htdocs are downloaded to. # this is the main runtime directory of the genome browser. It contains # CGI binaries, the config file (hg.conf), temporary ("trash") files, downloaded pieces of big files # ("udcCache") and various other runtime data needed for the browser APACHEDIR=/usr/local/apache # apache document root, for html documents HTDOCDIR=$APACHEDIR/htdocs # apache CGI-bin directory CGIBINDIR=$APACHEDIR/cgi-bin # directory for temporary files TRASHDIR=$APACHEDIR/trash # Mysql data directory for most genome annotation data -# Yes we only support mariaDB anymore, but the variables will keep their names +# Yes we only support mariaDB anymore, but the variables have 'mysql' in them. # Below please assume that mariadb is meant when mysql is written. # All user progress messages mention MariaDB, and no MySQL anymore. # (all non-mysql data is stored in /gbdb) -MYSQLDIR=/var/lib/mysql +# MYSQLDIR (the MariaDB datadir) is detected at runtime by setMysqlDir(): +# /var/lib/mysql on most distros, /var/lib/mariadb on recent Fedora/RHEL, +# falling back to `SHOW VARIABLES datadir` (e.g. OSX). # mysql admin binary, different path on OSX MYSQLADMIN=mysqladmin # mysql user account, different on OSX MYSQLUSER=mysql # mysql client command, will be adapted on OSX MYSQL=mysql # flag whether a mysql root password should be set # the root password is left empty on OSX, as mysql # there is not listening to a port SET_MYSQL_ROOT="0" @@ -565,31 +567,31 @@ if [ "$MYSQLMAJ" -ge 8 ] ; then echo2 'Mysql/MariaDB >= 8 found, checking if default-authentication allows native passwords' if grep -q default-authentication $MYCNF; then echo2 "default-authentication already set in $MYCNF" else echo2 Changing $MYCNF to allow native passwords and restarting Mysql echo '[mysqld]' >> $MYCNF echo 'default-authentication-plugin=mysql_native_password' >> $MYCNF stopMysql startMysql fi fi } -# some mariadb installers e.g. on redhat distros does not secure mariadb by default, so do this now +# some mariadb installers e.g. on redhat distros do not secure mariadb by default, so do this now # this is copied from Oracle's original script, on centos /usr/bin/mysql_secure_installation function secureMysql () { echo2 echo2 Securing the Mariadb install by removing the test user, restricting root echo2 logins to localhost and dropping the database named test. waitKey # do not parse .my.cnf for this, as we're sure that there is no root password yet # MYSQL2=`echo $MYSQL | sed -e 's/ / --no-defaults /'` # remove anonymous test users $MYSQL -e 'DELETE FROM mysql.user WHERE User="";' # remove remote root login $MYSQL -e "DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');" # removing test database $MYSQL -e "DROP DATABASE IF EXISTS test;" @@ -959,30 +961,31 @@ chmod u+w $APACHEDIR/ext/my.cnf fi echo '[mysqld]' > my.cnf echo "datadir = $APACHEDIR/mysqlData" >> my.cnf echo "default-storage-engine = myisam" >> my.cnf echo "default-tmp-storage-engine = myisam" >> my.cnf echo "skip-innodb" >> my.cnf echo "skip-networking" >> my.cnf echo "socket = $APACHEDIR/ext/mysql.socket" >> my.cnf echo '[client]' >> my.cnf echo "socket = $APACHEDIR/ext/mysql.socket" >> my.cnf cd $APACHEDIR # download minimal mysql db downloadFile $MYSQLDBURL | tar xz + MYSQLDIR=/var/lib/mysql # this makes little sense for OSX, adapt? chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR # configure apache echo2 Configuring Apache via files $APACHECONFDIR/httpd.conf and $APACHECONF echo "$APACHE_CONFIG_STR" > $APACHECONF # include browser config from apache config echo2 Appending browser config include line to $APACHECONFDIR/httpd.conf echo Include conf/001-browser.conf >> $APACHECONFDIR/httpd.conf # no need for document root, note BSD specific sed option -i # $SEDINPLACE 's/^DocumentRoot/#DocumentRoot/' $APACHECONFDIR/httpd.conf # server root provided on command line, note BSD sed -i option $SEDINPLACE 's/^ServerRoot/#ServerRoot/' $APACHECONFDIR/httpd.conf # need cgi and SSI - not needed anymore, are now compiled into apache $SEDINPLACE 's/^#LoadModule include_module/LoadModule include_module/' $APACHECONFDIR/httpd.conf $SEDINPLACE 's/^#LoadModule cgid_module/LoadModule cgid_module/' $APACHECONFDIR/httpd.conf @@ -1740,30 +1743,44 @@ sudo mkdir -p /usr/local/gbdb sudo chmod a+rwx /usr/local/gbdb # see https://apple.stackexchange.com/questions/388236/unable-to-create-folder-in-root-of-macintosh-hd if [[ ! -f /etc/synthetic.conf ]] || ! grep -q gbdb /etc/synthetic.conf; then sudo /bin/sh -c 'echo "gbdb\tusr/local/gbdb" >> /etc/synthetic.conf' fi chmod 644 /etc/synthetic.conf /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -t || true if ! test -L /gbdb; then echo "warning: apfs.util failed to create /gbdb" fi echo 'This directory is /usr/local/gbdb, see /etc/synthetic.conf' >> /gbdb/README.txt } +# Set MYSQLDIR to the MariaDB datadir. Default (and our recommendation) is +# /var/lib/mysql; recent Fedora/RHEL ship it as /var/lib/mariadb. Falls back +# to `SHOW VARIABLES datadir` (e.g. on OSX, or any non-standard install). +function setMysqlDir +{ + MYSQLDIR=/var/lib/mysql + if [ ! -d "$MYSQLDIR" ] && [ -d /var/lib/mariadb ]; then + MYSQLDIR=/var/lib/mariadb + fi + if [ ! -d "$MYSQLDIR" ]; then + MYSQLDIR=`mysql -NBe 'SHOW Variables WHERE Variable_Name="datadir"' | cut -f2` + fi +} + # GENOME DOWNLOAD: mysql and /gbdb function downloadGenomes { DBS=$* GENBANKTBLS="" if [ "$DBS" == "" ] ; then echo2 Argument error: the '"download"' command requires at least one assembly name, like hg19 or mm10. exit 100 fi echo2 echo2 Downloading databases $DBS plus hgFixed/proteome/go from the UCSC download server echo2 echo2 Determining download file size... please wait... @@ -1774,34 +1791,32 @@ else echo2 Downloading $DBS plus GenBank and RefSeq tables MYSQLDBS="$DBS" GENBANKTBLS="author cell description development gbCdnaInfo gbExtFile gbLoaded \ gbMiscDiff gbSeq gbWarn geneName imageClone keyword library \ mrnaClone organism productName refLink refSeqStatus \ refSeqSummary sex source tissue" fi else MYSQLDBS="$DBS" fi # rsync is doing globbing itself, so switch it off temporarily set -f - # On OSX the MariaDB datadir is not under /var. On Linux distros, the MariaDB directory may have been moved. - if [ ! -d $MYSQLDIR ]; then - MYSQLDIR=`mysql -NBe 'SHOW Variables WHERE Variable_Name="datadir"' | cut -f2` - fi + setMysqlDir + stopMysql # use rsync to get total size of files in directories and sum the numbers up with awk for db in $MYSQLDBS; do rsync -avn $HGDOWNLOAD::mysql/$db/ $MYSQLDIR/$db/ $RSYNCOPTS | grep ^'total size' | cut -d' ' -f4 | tr -d ', ' done | awk '{ sum += $1 } END { print "| Required space in '$MYSQLDIR':", sum/1000000000, "GB" }' if [ ! -z "$GENBANKTBLS" ]; then for tbl in $GENBANKTBLS; do rsync -avn $HGDOWNLOAD::mysql/hgFixed/${tbl}.* $MYSQLDIR/hgFixed/ $RSYNCOPTS | grep ^'total size' | cut -d' ' -f4 | tr -d ', ' done | awk '{ sum += $1 } END { print "| Required space in '$MYSQLDIR'/hgFixed:", sum/1000000000, "GB" }' fi for db in $DBS; do rsync -avn $HGDOWNLOAD::gbdb/$db/ $GBDBDIR/$db/ $RSYNCOPTS | grep ^'total size' | cut -d' ' -f4 | tr -d ',' done | awk '{ sum += $1 } END { print "| Required space in '$GBDBDIR':", sum/1000000000, "GB" }' @@ -1987,30 +2002,31 @@ # faster. This should be fast enough in the US West Coast area and maybe even on the East Coast. function downloadMinimal { DBS=$* if [ "$DBS" == "" ] ; then echo2 Argument error: the '"minimal"' command requires at least one assembly name, like hg19 or mm10. exit 100 fi echo2 echo2 Downloading minimal tables for databases $DBS # only these db tables are copied over by default minRsyncOpt="--include=cytoBand.* --include=chromInfo.* --include=cytoBandIdeo.* --include=kgColor.* --include=knownAttrs.* --include=knownGene.* --include=knownToTag.* --include=kgXref.* --include=ensemblLift.* --include=ucscToEnsembl.* --include=wgEncodeRegTfbsCells.* --include=encRegTfbsClusteredSources.* --include=tableList.* --include=refSeqStatus.* --include=wgEncodeRegTfbsCellsV3.* --include=extFile.* --include=trackDb.* --include=grp.* --include=ucscRetroInfo5.* --include=refLink.* --include=ucscRetroSeq5.* --include=ensemblLift.* --include=knownCanonical.* --include=gbExtFile.* --include=flyBase2004Xref --include=hgFindSpec.* --include=ncbiRefSeq*" + setMysqlDir stopMysql for db in $DBS; do echo2 Downloading Mysql files for mysql database $db $RSYNC $minRsyncOpt --exclude=* --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/$db/ $MYSQLDIR/$db/ chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/$db done echo2 Copying hgFixed.trackVersion, required for most tracks $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/trackVersion.* $MYSQLDIR/hgFixed/ echo2 Copying hgFixed.refLink, required for RefSeq tracks across all species $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/refLink.* $MYSQLDIR/hgFixed/ chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/hgFixed startMysql @@ -2088,31 +2104,33 @@ DBS=$* # if none specified, update all if [ "$DBS" == "" ] ; then DBS=`ls $GBDBDIR/` fi # update gbdb echo updating GBDB: $DBS for db in $DBS; do echo2 syncing gbdb: $db rsync -avp $RSYNCOPTS $HGDOWNLOAD::gbdb/$db/ $GBDBDIR/$db/ done # update the mysql DBs + setMysqlDir stopMysql + DBS=`ls $MYSQLDIR/ | egrep -v '(Trash$)|(hgTemp)|(^ib_)|(^ibdata)|(^aria)|(^mysql)|(performance)|(.flag$)|(multi-master.info)|(sys)|(lost.found)|(hgcentral)'` for db in $DBS; do echo2 syncing full mysql database: $db $RSYNC --update --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/$db/ $MYSQLDIR/$db/ done startMysql mysqlCheck hideSomeTracks echo2 update finished } function addTools {