71e1738517ba4e9ff1b69a1cbe5668cd35dac5a0 maximilianh Tue Apr 29 18:02:45 2025 +0200 gbic: few changes for OSX, no redmine diff --git src/product/installer/browserSetup.sh src/product/installer/browserSetup.sh index 809f337ae63..2e34fa174ac 100755 --- src/product/installer/browserSetup.sh +++ src/product/installer/browserSetup.sh @@ -519,30 +519,33 @@ # by default and somehow the section doesn't seem to take effect since a specific [mariadb] section also exists in the mariadb-server.cnf. # As a result, we only modify the mariadb-server config file MYCNF=/etc/my.cnf.d/mariadb-server.cnf elif [ -f /etc/mysql/mariadb.conf.d/*-server.cnf ] ; then # Ubuntu with mariadb. Must come before etc/my.cnf MYCNF=/etc/mysql/mariadb.conf.d/*-server.cnf elif [ -f /etc/mysql/mysql.conf.d/mysqld.cnf ] ; then # Ubuntu 16, 18, 20 with mysqld MYCNF=/etc/mysql/mysql.conf.d/mysqld.cnf elif [ -f /etc/my.cnf ] ; then # generic Centos 6-8 MYCNF=/etc/my.cnf elif [ -f /etc/mysql/my.cnf ] ; then # generic Ubuntu 14 MYCNF=/etc/mysql/my.cnf + elif [ -f /usr/local/etc/my.cnf ]; then + # brew on x86 + MYCNF=/usr/local/etc/my.cnf elif [ -f /opt/homebrew/etc/my.cnf ]; then # homebrew on ARMs MYCNF=/opt/homebrew/etc/my.cnf else echo Could not find my.cnf. Adapt 'setMYCNF()' in browserSetup.sh and/or contact us. exit 1 fi echo Found Mariadb config file: $MYCNF } function mysqlStrictModeOff () { # make sure that missing values in mysql insert statements do not trigger errors, #18368 = deactivate strict mode # This must happen before Mariadb is started or alternative Mariadb must be restarted after this has been done setMYCNF @@ -600,32 +603,32 @@ if [ -f ~/.my.cnf ]; then echo2 echo2 Mysql is going to be installed, but the file ~/.my.cnf already exists echo2 The file will be renamed to .my.cnf.old so it will not interfere with the echo2 installation. waitKey mv ~/.my.cnf ~/.my.cnf.old fi } # print my various IP addresses to stdout function showMyAddress () { echo2 if [[ "$DIST" == "OSX" ]]; then - echo2 You can browse the genome at http://127.0.0.1:8080 - #/usr/bin/open http://127.0.0.1:8080 + echo2 You can browse the genome at http://localhost + #/usr/bin/open http://localhost else echo2 You can now access this server under one of these IP addresses: echo2 From same host: http://127.0.0.1 echo2 From same network: http://`ip route get 8.8.8.8 | awk 'NR==1 {print $NF}'` echo2 From the internet: http://`downloadFileQuiet http://icanhazip.com` fi echo2 } # On OSX, we have to compile everything locally # DEPRECATED. This can be used to build a native OSX app. But not supported right now. function setupCgiOsx () { if [[ "$BUILDKENT" == "0" ]]; then echo2 @@ -1391,36 +1394,30 @@ # http://dev.mysql.com/doc/refman/5.1/en/load-data-local.html $MYSQL -e "GRANT FILE on *.* TO browser@localhost;" # Read only access to genome databases for the browser CGI binaries $MYSQL -e "CREATE USER readonly@localhost IDENTIFIED BY 'access';" $MYSQL -e "GRANT SELECT, CREATE TEMPORARY TABLES on "\ "*.* TO readonly@localhost;" $MYSQL -e "GRANT SELECT, INSERT, CREATE TEMPORARY TABLES on hgTemp.* TO "\ "readonly@localhost;" # Readwrite access to hgcentral for browser CGI binaries to keep session state $MYSQL -e "CREATE USER readwrite@localhost IDENTIFIED BY 'update';" $MYSQL -e "GRANT SELECT, INSERT, UPDATE, "\ "DELETE, CREATE, DROP, ALTER on hgcentral.* TO readwrite@localhost; " - # create /gbdb and let the apache user write to it - # hgConvert will download missing liftOver files on the fly and needs write - # write access - sudo mkdir -p $GBDBDIR - sudo chown $APACHEUSER:$APACHEUSER $GBDBDIR - # the custom track database needs it own user and permissions $MYSQL -e "CREATE USER ctdbuser@localhost IDENTIFIED BY 'ctdbpassword';" $MYSQL -e "GRANT SELECT,INSERT,UPDATE,DELETE,CREATE,DROP,ALTER,INDEX "\ "on customTrash.* TO ctdbuser@localhost;" # removed these now for the new hgGateway page, Apr 2016 # by default hgGateway needs an empty hg19 database, will crash otherwise # $MYSQL -e 'CREATE DATABASE IF NOT EXISTS hg19' # mm9 needs an empty hg18 database $MYSQL -e 'CREATE DATABASE IF NOT EXISTS hg18' $MYSQL -e "FLUSH PRIVILEGES;" } @@ -1449,32 +1446,52 @@ if [ ! -e /usr/local/apache ]; then echo2 Creating /usr/local/apache to fill with symlinks later sudo mkdir -p /usr/local/apache sudo chmod a+rw /usr/local/apache fi if [ ! -e /usr/local/apache/cgi-bin ]; then echo2 Creating symlink /usr/local/apache/cgi-bin to /Library/WebServer/CGI-Executables sudo ln -s /Library/WebServer/CGI-Executables /usr/local/apache/cgi-bin fi if [ ! -e /usr/local/apache/htdocs ]; then echo2 Creating symlink /usr/local/apache/htdocs to /Library/WebServer/Documents sudo ln -s /Library/WebServer/Documents/ /usr/local/apache/htdocs fi - # switch on CGIs in Apple's Apache and restart it - sudo sed -i '' -E 's/^\s*#(LoadModule cgid_module)/\1/; s/^\s*#(LoadModule cgi_module)/\1/' /etc/apache2/httpd.conf + if [[ ! -d /usr/local/apache/trash ]]; then + mkdir -p /usr/local/apache/trash + sudo chmod a+rwx /usr/local/apache/trash + # cgis use ../trash to find the trash directory, but we have a symlink here + sudo ln -fs /usr/local/apache/trash /Library/WebServer/Documents/trash + sudo ln -fs /usr/local/apache/trash /Library/WebServer/trash + # cgis use ../htdocs to find the htdocs directory, but we have a symlink here + sudo ln -fs /Library/WebServer/Documents /Library/WebServer/htdocs + fi + + # switch on CGIs in Apple's Apache and restart it (OSX as a BSD does not understand \s, so need to use [[:space:]]) + sudo sed -Ei '' 's/^[[:space:]]*#(LoadModule cgid_module)/\1/; s/^[[:space:]]*#(LoadModule cgi_module)/\1/' /etc/apache2/httpd.conf + # allow server side includes + sudo sed -Ei '' 's/^[[:space:]]*#(LoadModule include_module)/\1/' /etc/apache2/httpd.conf + # activate server side includes and make them depend on the X flag + sudo sed -Ei '' 's/Options FollowSymLinks Multiviews/Options FollowSymLinks Includes Multiviews/' /etc/apache2/httpd.conf + if grep -qv XBitHack /etc/apache2/httpd.conf ; then + sudo sed -Ei '' '/Options FollowSymLinks Includes Multiviews/ a\ +XBitHack on\ +SSILegacyExprParser on +' /etc/apache2/httpd.conf + fi sudo /usr/sbin/apachectl restart } # install gcc, make etc so we can build the tree on linux function setupBuildLinux () { echo2 Installing required linux packages from repositories: Git, GCC, G++, Mysql-client-libs, uuid, etc waitKey if [[ "$DIST" == "debian" ]]; then apt-get --assume-yes $APTERR install make git gcc g++ libpng-dev libmysqlclient-dev uuid-dev libfreetype-dev libbz2-dev elif [[ "$DIST" == "redhat" ]]; then yum install -y git vim gcc gcc-c++ make libpng-devel libuuid-devel freetype-devel else echo Error: Cannot identify linux distribution exit 100 @@ -1629,58 +1646,59 @@ # note that the sed -i syntax is different from linux echo2 Adapting mysql socket locations in $CGIBINDIR/hg.conf sockFile=`mysql -NBe 'show variables like "socket"' | cut -f2` $SEDINPLACE "s|^#?socket=.*|socket=$sockFile|" $CGIBINDIR/hg.conf $SEDINPLACE "s|^#?customTracks.socket.*|customTracks.socket=$sockFile|" $CGIBINDIR/hg.conf $SEDINPLACE "s|^#?db.socket.*|db.socket=$sockFile|" $CGIBINDIR/hg.conf $SEDINPLACE "s|^#?central.socket.*|central.socket=$sockFile|" $CGIBINDIR/hg.conf fi # check if UCSC or genome-euro MySQL server is closer echo comparing latency: genome.ucsc.edu Vs. genome-euro.ucsc.edu eurospeed=$( (time -p (for i in `seq 10`; do curl -sSI genome-euro.ucsc.edu > /dev/null; done )) 2>&1 | grep real | cut -d' ' -f2 ) ucscspeed=$( (time -p (for i in `seq 10`; do curl -sSI genome.ucsc.edu > /dev/null; done )) 2>&1 | grep real | cut -d' ' -f2 ) if [[ $(awk '{if ($1 <= $2) print 1;}' <<< "$eurospeed $ucscspeed") -eq 1 ]]; then echo genome-euro seems to be closer echo modifying hg.conf to pull data from genome-euro instead of genome - sed -i s/slow-db.host=genome-mysql.soe.ucsc.edu/slow-db.host=genome-euro-mysql.soe.ucsc.edu/ $CGIBINDIR/hg.conf - sed -i "s#gbdbLoc2=http://hgdownload.soe.ucsc.edu/gbdb/#gbdbLoc2=http://hgdownload-euro.soe.ucsc.edu/gbdb/#" $CGIBINDIR/hg.conf + $SEDINPLACE s/slow-db.host=genome-mysql.soe.ucsc.edu/slow-db.host=genome-euro-mysql.soe.ucsc.edu/ $CGIBINDIR/hg.conf + $SEDINPLACE "s#gbdbLoc2=http://hgdownload.soe.ucsc.edu/gbdb/#gbdbLoc2=http://hgdownload-euro.soe.ucsc.edu/gbdb/#" $CGIBINDIR/hg.conf HGDOWNLOAD=hgdownload-euro.soe.ucsc.edu else echo genome.ucsc.edu seems to be closer echo not modifying $CGIBINDIR/hg.conf fi # download the CGIs if [[ "$OS" == "OSX" ]]; then #setupCgiOsx echo2 Running on OSX, assuming that CGIs are already built into /usr/local/apache/cgi-bin elif [[ "$MACH" == "aarch64" ]]; then echo2 Running on an ARM CPU, assuming that CGIs are already built into /usr/local/apache/cgi-bin else # don't download RNAplot, it's a 32bit binary that won't work anywhere anymore but at UCSC # this means that hgGene cannot show RNA structures but that's not a big issue $RSYNC -avzP --exclude=RNAplot $HGDOWNLOAD::cgi-bin/ $CGIBINDIR/ # now add the binaries for dot and RNAplot $RSYNC -avzP $HGDOWNLOAD::genome/admin/exe/external.x86_64/RNAplot $CGIBINDIR/ $RSYNC -avzP $HGDOWNLOAD::genome/admin/exe/external.x86_64/loader/dot_static $CGIBINDIR/loader/ fi # download the html docs, exclude some big files on OSX - # try to minimize storage for OSX, mostly laptops + # cannot do this when we built the tree outselves, as the .js versions will not match the C code if [ "$OS" == "OSX" -o "$MACH" == "aarch64" ]; then - #$RSYNC --delete -azP --exclude=training --exclude=ENCODE --exclude=encode --exclude=rosenbloom.pdf --exclude=pubs*.pdf --exclude=*.{bb,bam,bai,bw,gz,2bit} --exclude=goldenpath $HGDOWNLOAD::htdocs/ $HTDOCDIR/ - echo2 Not syncing htdocs folder, assuming that these were built from source. + # but we need the new font files + $RSYNC -azP hgdownload.soe.ucsc.edu::htdocs/urw-fonts/ /usr/local/apache/htdocs/urw-fonts/ + echo2 Not syncing most of htdocs folder, assuming that these were built from source. echo2 PDF and other large files only present at UCSC will be missing from htdocs. waitKey else rm -rf $APACHEDIR/htdocs/goldenpath $RSYNC -avzP --exclude ENCODE/**.pdf $HGDOWNLOAD::htdocs/ $HTDOCDIR/ fi # assign all files just downloaded to a valid user. # This also allows apache to write into the trash dir if [ "$OS" == "OSX" ]; then echo2 OSX: Not chowning /usr/local/apache subdirectories, as not running as root else chown -R $APACHEUSER:$APACHEUSER $CGIBINDIR $HTDOCDIR $TRASHDIR fi @@ -1694,43 +1712,50 @@ echo2 It also requires an open outgoing TCP port 3306 for Mysql to genome-mysql.soe.ucsc.edu/genome-euro-mysql.soe.ucsc.edu, echo2 and open TCP port 80 to hgdownload.soe.ucsc.edu/hgdownload-euro.soe.ucsc.edu. echo2 echo2 To finish the installation, you need to download genome data to the local echo2 disk. To download a genome assembly and all its files now, call this script again with echo2 the parameters 'download "<assemblyName1> <assemblyName2> ..."', e.g. '"'bash $0 mirror mm10 hg19'"' echo2 showMyAddress exit 0 } # mkdir /gbdb or do the weird things one has to do on OSX to make this directory function mkdirGbdb { if [[ "$OS" != "OSX" ]]; then + # On Linux, create /gbdb and let the apache user write to it mkdir -p $GBDBDIR + # Why? hgConvert will download missing liftOver files on the fly and needs write + # write access + chown $APACHEUSER:$APACHEUSER $GBDBDIR return fi sudo mkdir -p /usr/local/gbdb sudo chmod a+rwx /usr/local/gbdb # see https://apple.stackexchange.com/questions/388236/unable-to-create-folder-in-root-of-macintosh-hd if [[ ! -f /etc/synthetic.conf ]] || grep -vq gbdb /etc/synthetic.conf; then sudo /bin/sh -c 'echo "gbdb\tusr/local/gbdb" >> /etc/synthetic.conf' fi chmod 644 /etc/synthetic.conf - /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -t + /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -t || true + if ! test -L /gbdb; then + echo "warning: apfs.util failed to create /gbdb" + fi echo 'This directory is /usr/local/gbdb, see /etc/synthetic.conf' >> /gbdb/README.txt } # GENOME DOWNLOAD: mysql and /gbdb function downloadGenomes { DBS=$* GENBANKTBLS="" if [ "$DBS" == "" ] ; then echo2 Argument error: the '"download"' command requires at least one assembly name, like hg19 or mm10. exit 100 fi echo2 echo2 Downloading databases $DBS plus hgFixed/proteome/go from the UCSC download server @@ -1792,58 +1817,57 @@ echo2 symlink $GBDBDIR and $MYSQLDIR to the new locations. You might have to stop echo2 Mysql temporarily to do this. echo2 Example commands: echo2 " mv /gbdb /bigData/gbdb && ln -s /bigData/gbdb /gbdb" echo2 " mv /var/lib/mysql /bigData/mysql && ln -s /bigData/mysql /var/lib/mysql" echo2 echo2 You can interrupt this script with CTRL-C now, add more space and rerun the echo2 script later with the same parameters to start the download. echo2 waitKey # now do the actual download of mysql files for db in $MYSQLDBS; do echo2 Downloading Mysql files for mysql database $db $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/$db/ $MYSQLDIR/$db/ - chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/$db done if [ ! -z "$GENBANKTBLS" ]; then echo2 Downloading hgFixed tables for tbl in $GENBANKTBLS; do $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/${tbl}.* $MYSQLDIR/hgFixed/ done - chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/hgFixed fi echo2 Downloading hgFixed.refLink, required for all RefSeq tracks $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/refLink.* $MYSQLDIR/hgFixed/ - chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/hgFixed mkdirGbdb # download /gbdb files for db in $DBS; do echo2 Downloading $GBDBDIR files for assembly $db $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::gbdb/$db/ $GBDBDIR/$db/ - chown -R $APACHEUSER:$APACHEUSER $GBDBDIR/$db done set +f # Alexander Stuy reported that at FSU they had a few mysql databases with incorrect users on them + if [[ "$OS" != "OSX" ]]; then chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/ + chown -R $APACHEUSER:$APACHEUSER $GBDBDIR/$db + fi startMysql mysqlCheck hideSomeTracks goOffline # modify hg.conf and remove all statements that use the UCSC download server echo2 echo2 Install complete. You should now be able to point your web browser to this machine echo2 and use your UCSC Genome Browser mirror. echo2 showMyAddress @@ -1868,53 +1892,56 @@ waitKey } # stop the mysql database server, so we can write into its data directory function stopMysql { if [ -f /etc/init.d/mysql ]; then service mysql stop elif [ -f /etc/init.d/mysqld ]; then service mysqld stop elif [ -f /etc/init.d/mariadb ]; then service mariadb stop elif [ -f /usr/lib/systemd/system/mariadb.service ]; then # RHEL 7, etc use systemd instead of SysV systemctl stop mariadb - elif [ which brew > /dev/null ]; then + elif which brew > /dev/null ; then # homebrew on ARMs or X86 brew services stop mariadb elif [ -f /usr/lib/systemd/system/mysql.service ]; then # at least seen in Fedora 17 systemctl stop mysql else echo2 Could not find mysql nor mysqld file in /etc/init.d nor a systemd command. Please email genome-mirror@soe.ucsc.edu. fi } # start the mysql database server function startMysql { if [ -f /etc/init.d/mysql ]; then service mysql start elif [ -f /etc/init.d/mysqld ]; then service mysqld start elif [ -f /etc/init.d/mariadb ]; then service mariadb start elif [ -f /usr/lib/systemd/system/mariadb.service ]; then # RHEL 7, etc use systemd instead of SysV systemctl start mariadb + elif which brew > /dev/null ; then + echo2 Starting Mariadb using brew + brew services start mariadb elif [ -f /usr/lib/systemd/system/mysql.service ]; then # at least seen in Fedora 17 systemctl start mysql else echo2 Could not find mysql nor mysqld file in /etc/init.d nor a systemd command. Please email genome-mirror@soe.ucsc.edu. fi } function mysqlCheck # check all mysql tables. Rarely, some of them are in an unclosed state on the download server, this command will close them { echo2 Checking all mysql tables after the download to make sure that they are closed mysqlcheck --all-databases --auto-repair --quick --fast --silent }