6cbe994ed08fe34dc15e5128e20044c0086a7ba7 max Fri Apr 2 07:51:46 2021 -0700 update GBIC for Ubuntu 20, refs #27070 diff --git src/product/installer/browserSetup.sh src/product/installer/browserSetup.sh index 62a9a42..3a71188 100755 --- src/product/installer/browserSetup.sh +++ src/product/installer/browserSetup.sh @@ -347,30 +347,32 @@ minimal - download only a minimal set of tables. Missing tables are downloaded on-the-fly from UCSC. mirror - download a full assembly (also see the -t option below). After completion, no data is downloaded on-the-fly from UCSC. update - update the genome browser software and data, updates all tables of an assembly, like "mirror" cgiUpdate - update only the genome browser software, not the data. Not recommended, see documentation. clean - remove temporary files of the genome browser older than one day, but do not delete any uploaded custom tracks addTools - copy the UCSC User Tools, e.g. blat, featureBits, overlapSelect, bedToBigBed, pslCDnaFilter, twoBitToFa, gff3ToGenePred, bedSort, ... to /usr/local/bin This has to be run after the browser has been installed, other- wise these packages may be missing: libpng zlib libmysqlclient + mysql - Patch my.cnf and recreate Mysql users. This can fix + a broken Mysql server after an update to Mysql 8. parameters for 'minimal', 'mirror' and 'update': <assemblyList> - download Mysql + /gbdb files for a space-separated list of genomes examples: bash $0 install - install Genome Browser, do not download any genome assembly, switch to on-the-fly mode (see the -f option) bash $0 minimal hg19 - download only the minimal tables for the hg19 assembly bash $0 mirror hg19 mm9 - download hg19 and mm9, switch to offline mode (see the -o option) bash $0 -t noEncode mirror hg19 - install Genome Browser, download hg19 but no ENCODE tables and switch to offline mode (see the -o option) @@ -478,30 +480,51 @@ $SEDINPLACE 's/^#slow-db\./slow-db\./g' $APACHEDIR/cgi-bin/hg.conf $SEDINPLACE 's/^#gbdbLoc1=/gbdbLoc1=/g' $APACHEDIR/cgi-bin/hg.conf $SEDINPLACE 's/^#gbdbLoc2=/gbdbLoc2=/g' $APACHEDIR/cgi-bin/hg.conf $SEDINPLACE 's/^#showTableCache=/showTableCache=/g' $APACHEDIR/cgi-bin/hg.conf } # wait for a key press function waitKey () { echo2 echo2 Press any key to continue or CTRL-C to abort. read -n 1 echo2 } +function mysqlAllowOldPasswords +# mysql >= 8 does not allow the old passwords anymore. But our client is still compiled +# with the old, non-SHA256 encryption. So we must deactivate this new feature. +# What will MariaDB do? +{ +echo2 'Checking for Mysql version >= 8' +MYSQLMAJ=`mysql -e 'SHOW VARIABLES LIKE "version";' -NB | cut -f2 | cut -d. -f1` +if [ "$MYSQLMAJ" -ge 8 ] ; then + echo2 'Mysql >= 8 found, checking if default-authentication allows native passwords' + if grep -q default-authentication /etc/mysql/my.cnf; then + echo2 'default-authentication already set in /etc/mysql/my.cnf' + else + echo2 Changing /etc/mysql/my.cnf to allow native passwords and restarting Mysql + echo '[mysqld]' >> /etc/mysql/my.cnf + echo 'default-authentication-plugin=mysql_native_password' >> /etc/mysql/my.cnf + service mysql restart + fi +fi +} + + # oracle's mysql install e.g. on redhat distros does not secure mysql by default, so do this now # this is copied from Oracle's original script, on centos /usr/bin/mysql_secure_installation function secureMysql () { echo2 echo2 Securing the Mysql install by removing the test user, restricting root echo2 logins to localhost and dropping the database named test. waitKey # do not parse .my.cnf for this, as we're sure that there is no root password yet # MYSQL2=`echo $MYSQL | sed -e 's/ / --no-defaults /'` # remove anonymous test users $MYSQL -e 'DELETE FROM mysql.user WHERE User="";' # remove remote root login $MYSQL -e "DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');" # removing test database @@ -699,31 +722,31 @@ secureMysql SET_MYSQL_ROOT=1 else echo2 Mysql already installed fi # MySQL-python is required for hgGeneGraph # CentOS up to and including 7 default to python2, so MySQL-python is in the repos if yum list MySQL-python 2> /dev/null ; then yum -y install MySQL-python # Centos 8 defaults to python3 and it does not have a package MySQL-python anymore # So we install python2, the mysql libraries and fix up my_config.h manually # This is strange, but I was unable to find a different working solution. MariaDB does not have my_config.h else - yum install -y python2 mysql-devel python2-devel wget + yum install -y python2 mysql-devel python2-devel wget gcc if [ -f /usr/include/mysql/my_config.h ]; then echo my_config.h found else wget https://raw.githubusercontent.com/paulfitz/mysql-connector-c/master/include/my_config.h -P /usr/include/mysql/ fi pip2 install MySQL-python fi } # OSX specific setup of the installation function installOsx () { # check for xcode if [ -f /usr/bin/xcode-select 2> /dev/null > /dev/null ]; then @@ -858,31 +881,47 @@ function installDebian () { # update repos if [ ! -f /tmp/browserInstall.aptGetUpdateDone ]; then echo2 Running apt-get update apt-get update touch /tmp/browserInstall.aptGetUpdateDone fi echo2 Installing ghostscript and imagemagick waitKey # ghostscript for PDF export # imagemagick for the session gallery # r-base-core for the gtex tracks # python-mysqldb for hgGeneGraph - apt-get --no-install-recommends --assume-yes install ghostscript imagemagick wget rsync r-base-core python-mysqldb curl gsfonts + apt-get --no-install-recommends --assume-yes install ghostscript imagemagick wget rsync r-base-core curl gsfonts + # python-mysqldb has been removed in newer distros + if apt-cache policy python-mysqldb | grep "Candidate: .none." > /dev/null; then + echo2 The package python-mysqldb is not available anymore. Working around it + echo2 by installing python2 and MySQL-python with pip2 + apt-get install --assume-yes python2 libmysqlclient-dev python2-dev wget gcc + curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output /tmp/get-pip.py + python2 /tmp/get-pip.py + if [ -f /usr/include/mysql/my_config.h ]; then + echo my_config.h found + else + wget https://raw.githubusercontent.com/paulfitz/mysql-connector-c/master/include/my_config.h -P /usr/include/mysql/ + fi + pip2 install MySQL-python + else + apt-get --assume-yes install python-mysqldb + fi if [ ! -f $APACHECONF ]; then echo2 echo2 Now installing Apache2. echo2 "Apache's default config /etc/apache2/sites-enable/000-default will be" echo2 "deactivated. A new configuration $APACHECONF will be added and activated." echo2 The apache modules include, cgid and authz_core will be activated. waitKey # apache and mysql are absolutely required # ghostscript is required for PDF output apt-get --assume-yes install apache2 ghostscript # gmt is not required. install fails if /etc/apt/sources.list does not contain # a 'universe' repository mirror. Can be safely commented out. Only used @@ -1139,80 +1178,86 @@ # centos 7 seems to have another config file for this if [ -f /etc/selinux/config ]; then sed -i 's/^SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config fi fi fi } # setup the mysql databases for the genome browser and grant # user access rights function mysqlDbSetup () { # ------------------- # Mysql db setup # ------------------- + mysqlAllowOldPasswords + echo2 echo2 Creating Mysql databases customTrash, hgTemp and hgcentral waitKey $MYSQL -e 'CREATE DATABASE IF NOT EXISTS customTrash;' $MYSQL -e 'CREATE DATABASE IF NOT EXISTS hgcentral;' $MYSQL -e 'CREATE DATABASE IF NOT EXISTS hgTemp;' $MYSQL -e 'CREATE DATABASE IF NOT EXISTS hgFixed;' # empty db needed for gencode tracks updateBlatServers echo2 echo2 "Will now grant permissions to browser database access users:" echo2 "User: 'browser', password: 'genome' - full database access permissions" echo2 "User: 'readonly', password: 'access' - read only access for CGI binaries" echo2 "User: 'readwrite', password: 'update' - readwrite access for hgcentral DB" waitKey # Full access to all databases for the user 'browser' # This would be for browser developers that need read/write access # to all database tables. + $MYSQL -e "DROP USER IF EXISTS browser@localhost" $MYSQL -e "CREATE USER browser@localhost IDENTIFIED BY 'genome';" $MYSQL -e "GRANT SELECT, INSERT, UPDATE, DELETE, FILE, "\ "CREATE, DROP, ALTER, CREATE TEMPORARY TABLES on *.* TO browser@localhost; " # FILE permission for this user to all databases to allow DB table loading with # statements such as: "LOAD DATA INFILE file.tab" # For security details please read: # http://dev.mysql.com/doc/refman/5.1/en/load-data.html # http://dev.mysql.com/doc/refman/5.1/en/load-data-local.html $MYSQL -e "GRANT FILE on *.* TO browser@localhost;" # Read only access to genome databases for the browser CGI binaries + $MYSQL -e "DROP USER IF EXISTS readonly@localhost" $MYSQL -e "CREATE USER readonly@localhost IDENTIFIED BY 'access';" $MYSQL -e "GRANT SELECT, CREATE TEMPORARY TABLES on "\ "*.* TO readonly@localhost;" $MYSQL -e "GRANT SELECT, INSERT, CREATE TEMPORARY TABLES on hgTemp.* TO "\ "readonly@localhost;" # Readwrite access to hgcentral for browser CGI binaries to keep session state + $MYSQL -e "DROP USER IF EXISTS readwrite@localhost" $MYSQL -e "CREATE USER readwrite@localhost IDENTIFIED BY 'update';" $MYSQL -e "GRANT SELECT, INSERT, UPDATE, "\ "DELETE, CREATE, DROP, ALTER on hgcentral.* TO readwrite@localhost; " # create /gbdb and let the apache user write to it # hgConvert will download missing liftOver files on the fly and needs write # write access mkdir -p $GBDBDIR chown $APACHEUSER:$APACHEUSER $GBDBDIR # the custom track database needs it own user and permissions + $MYSQL -e "DROP USER IF EXISTS ctdbuser@localhost" $MYSQL -e "CREATE USER ctdbuser@localhost IDENTIFIED BY 'ctdbpassword';" $MYSQL -e "GRANT SELECT,INSERT,UPDATE,DELETE,CREATE,DROP,ALTER,INDEX "\ "on customTrash.* TO ctdbuser@localhost;" # removed these now for the new hgGateway page, Apr 2016 # by default hgGateway needs an empty hg19 database, will crash otherwise # $MYSQL -e 'CREATE DATABASE IF NOT EXISTS hg19' # mm9 needs an empty hg18 database $MYSQL -e 'CREATE DATABASE IF NOT EXISTS hg18' $MYSQL -e "FLUSH PRIVILEGES;" } # main function, installs the browser on Redhat/Debian and potentially even on OSX function installBrowser () @@ -1321,31 +1366,31 @@ # in OSX adapt the sockets # note that the sed -i syntax is different from linux echo2 Adapting mysql socket locations in $CGIBINDIR/hg.conf sockFile=$APACHEDIR/ext/mysql.socket $SEDINPLACE "s|^#?socket=.*|socket=$sockFile|" $CGIBINDIR/hg.conf $SEDINPLACE "s|^#?customTracks.socket.*|customTracks.socket=$sockFile|" $CGIBINDIR/hg.conf $SEDINPLACE "s|^#?db.socket.*|db.socket=$sockFile|" $CGIBINDIR/hg.conf $SEDINPLACE "s|^#?central.socket.*|central.socket=$sockFile|" $CGIBINDIR/hg.conf fi # check if UCSC or genome-euro MySQL server is closer echo comparing latency: genome.ucsc.edu Vs. genome-euro.ucsc.edu eurospeed=$( (time -p (for i in `seq 10`; do curl -sSI genome-euro.ucsc.edu > /dev/null; done )) 2>&1 | grep real | cut -d' ' -f2 ) ucscspeed=$( (time -p (for i in `seq 10`; do curl -sSI genome.ucsc.edu > /dev/null; done )) 2>&1 | grep real | cut -d' ' -f2 ) if [[ $(awk '{if ($1 <= $2) print 1;}' <<< "$eurospeed $ucscspeed") -eq 1 ]]; then echo genome-euro seems to be closer - echo modifying mirror to pull data from genome-euro instead of genome + echo modifying hg.conf to pull data from genome-euro instead of genome sed -i s/slow-db.host=genome-mysql.soe.ucsc.edu/slow-db.host=genome-euro-mysql.soe.ucsc.edu/ $CGIBINDIR/hg.conf sed -i "s#gbdbLoc2=http://hgdownload.soe.ucsc.edu/gbdb/#gbdbLoc2=http://hgdownload-euro.soe.ucsc.edu/gbdb/#" $CGIBINDIR/hg.conf HGDOWNLOAD=hgdownload-euro.soe.ucsc.edu else echo genome.ucsc.edu seems to be closer echo not modifying $CGIBINDIR/hg.conf fi # download the CGIs if [[ "$OS" == "OSX" ]]; then setupCgiOsx else # don't download RNAplot, it's a 32bit binary that won't work anywhere anymore but at UCSC # this means that hgGene cannot show RNA structures but that's not a big issue @@ -1356,38 +1401,38 @@ rm -rf $APACHEDIR/htdocs/goldenpath # try to minimize storage for OSX, mostly laptops if [ "$OS" == "OSX" ]; then $RSYNC --delete -azP --exclude=training --exclude=ENCODE --exclude=encode --exclude=rosenbloom.pdf --exclude=pubs*.pdf --exclude=*.{bb,bam,bai,bw,gz,2bit} --exclude=goldenpath $HGDOWNLOAD::htdocs/ $HTDOCDIR/ else $RSYNC -avzP --exclude ENCODE/**.pdf $HGDOWNLOAD::htdocs/ $HTDOCDIR/ fi # assign all files just downloaded to a valid user. # This also allows apache to write into the trash dir chown -R $APACHEUSER:$APACHEUSER $CGIBINDIR $HTDOCDIR $TRASHDIR touch $COMPLETEFLAG echo2 Install complete. You should now be able to point your web browser to this machine - echo2 and use your UCSC Genome Browser mirror. + echo2 and test your UCSC Genome Browser mirror. It will be too slow for practical use. echo2 echo2 Notice that this mirror is still configured to use Mysql and data files loaded echo2 through the internet from UCSC. From most locations on the world, this is very slow. echo2 It also requires an open outgoing TCP port 3306 for Mysql to genome-mysql.soe.ucsc.edu/genome-euro-mysql.soe.ucsc.edu, echo2 and open TCP port 80 to hgdownload.soe.ucsc.edu/hgdownload-euro.soe.ucsc.edu. echo2 - echo2 To speed up the installation, you need to download genome data to the local + echo2 To finish the installation, you need to download genome data to the local echo2 disk. To download a genome assembly and all its files now, call this script again with echo2 the parameters 'download "<assemblyName1> <assemblyName2> ..."', e.g. '"'bash $0 download mm10 hg19'"' echo2 showMyAddress } # GENOME DOWNLOAD: mysql and /gbdb function downloadGenomes { DBS=$* GENBANKTBLS="" if [ "$DBS" == "" ] ; then echo2 Argument error: the '"download"' command requires at least one assembly name, like hg19 or mm10. exit 1 fi @@ -1584,30 +1629,31 @@ $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/refLink.* $MYSQLDIR/hgFixed/ startMysql echo2 Hiding some tracks by default and removing some tracks from searches for db in $DBS; do echo $db for track in $hideTracks; do mysql $db -e 'UPDATE trackDb set visibility=0 WHERE tableName="'$track'"' done for track in $notSearchTables; do mysql $db -e 'DELETE from hgFindSpec WHERE searchTable="'$track'"' done done + echo2 echo2 The mirror should be functional now. It contains some basic assembly tables echo2 and will download missing data from the UCSC servers. This requires echo2 two open ports, outgoing, TCP, from this machine: echo2 - to genome-mysql.soe.ucsc.edu, port 3306, to load MySQL tables echo2 - to hgdownload.soe.ucsc.edu, port 80, to download non-MySQL data files echo2 - or the above two servers European counterparts: echo2 genome-euro-mysql.soe.ucsc.edu and hgdownload-euro.soe.ucsc.edu echo2 showMyAddress goOnline } function checkDownloadUdr () { @@ -1860,20 +1906,23 @@ elif [ "${1:-}" == "mirror" ]; then downloadGenomes ${@:2} # all arguments after the second one elif [ "${1:-}" == "cgiUpdate" ]; then cgiUpdate elif [ "${1:-}" == "update" ]; then updateBrowser ${@:2} # all arguments after the second one elif [ "${1:-}" == "clean" ]; then cleanTrash elif [ "${1:-}" == "addTools" ]; then addTools +elif [ "${1:-}" == "mysql" ]; then + mysqlDbSetup + else echo Unknown command: $1 echo "$HELP_STR" exit 1 fi