e132560389050652178d61d4a6006f86bbe5ea4c maximilianh Mon Apr 28 16:13:27 2025 +0200 GBIC: detecting mysql directory in case the user moved it, after current ML ticket. working around OSX weirdness with /gbdb. No redmine yet diff --git src/product/installer/browserSetup.sh src/product/installer/browserSetup.sh index 299ba142036..809f337ae63 100755 --- src/product/installer/browserSetup.sh +++ src/product/installer/browserSetup.sh @@ -78,46 +78,30 @@ # udr binary URL UDRURL=http://hgdownload.soe.ucsc.edu/admin/exe/linux.x86_64/udr # rsync is a variable so it can be be set to use udr RSYNC=rsync # by default, everything is downloaded RSYNCOPTS="" # alternative? # --include='*/' --exclude='wgEncode*.bam' hgdownload.soe.ucsc.edu::gbdb/hg19/ ./ -h # a flagfile to indicate that the cgis were installed successfully COMPLETEFLAG=/usr/local/apache/cgiInstallComplete.flag -# on OSX by default we download a pre-compiled package that includes Apache/Mysql/OpenSSL -# change this to 1 to rebuild all of these locally from tarballs -BUILDEXT=${BUILDEXT:-0} -# On OSX, by default download the CGIs as a binary package -# change this to 1 to rebuild CGIs locally from tarball -BUILDKENT=${BUILDKENT:-0} - -# URL of a tarball with a the binaries of Apache/Mysql/Openssl -BINPKGURL=http://hgwdev.gi.ucsc.edu/~max/gbInstall/mysqlApacheOSX_10.7.tgz -# URL of tarball with the OSX CGI binaries -CGIBINURL=http://hgwdev.gi.ucsc.edu/~max/gbInstall/kentCgi_OSX_10.7.tgz -# URL of tarball with a minimal Mysql data directory -MYSQLDBURL=http://hgwdev.gi.ucsc.edu/~max/gbInstall/mysql56Data.tgz -# mysql/apache startup script URL, currently only for OSX -STARTSCRIPTURL=https://raw.githubusercontent.com/maximilianh/browserInstall/master/browserStartup.sh - # the -t option allows to download only the genome databases, not hgFixed/proteome/go/uniProt # by default, this is off, so we download hgFixed and Co. ONLYGENOMES=0 # make apt-get fail on warnings APTERR="-o APT::Update::Error-Mode=any" # ---- END GLOBAL DEFAULT SETTINGS ---- # ---- DEFAULT CONFIG FILES ------------------ # We need to initialize hg.conf and apache.conf for the browser. # They are inline so we do not create a dependency on another file # that has to be pushed to hgdownload when this script is updated # the read command always has an error exit code for here docs, so we deactivate # exit on error for a moment @@ -899,34 +883,51 @@ function installOsxDevTools () # make sure that the xcode command line tools are installed { # check for xcode if [ -f /usr/bin/xcode-select 2> /dev/null > /dev/null ]; then echo2 Found XCode else echo2 echo2 'This installer has to compile the UCSC tools locally on OSX. We need clang. Starting installation.' xcode-select --install 2> /dev/null >/dev/null fi } # OSX specific setup of the installation +# NOT USED ANYMORE, but kept here for reference and for future use one day function installOsx () { installOsxDevTools + # on OSX by default we download a pre-compiled package that includes Apache/Mysql/OpenSSL + # change this to 1 to rebuild all of these locally from tarballs + BUILDEXT=${BUILDEXT:-0} + # On OSX, by default download the CGIs as a binary package + # change this to 1 to rebuild CGIs locally from tarball + BUILDKENT=${BUILDKENT:-0} + + # URL of a tarball with a the binaries of Apache/Mysql/Openssl + BINPKGURL=http://hgwdev.gi.ucsc.edu/~max/gbInstall/mysqlApacheOSX_10.7.tgz + # URL of tarball with the OSX CGI binaries + CGIBINURL=http://hgwdev.gi.ucsc.edu/~max/gbInstall/kentCgi_OSX_10.7.tgz + # URL of tarball with a minimal Mysql data directory + MYSQLDBURL=http://hgwdev.gi.ucsc.edu/~max/gbInstall/mysql56Data.tgz + # mysql/apache startup script URL, currently only for OSX + STARTSCRIPTURL=https://raw.githubusercontent.com/maximilianh/browserInstall/master/browserStartup.sh + # in case that it is running, try to stop Apple's personal web server, we need access to port 80 # ignore any error messages #if [ -f /usr/sbin/apachectl ]; then #echo2 Stopping the Apple Personal Web Server #/usr/sbin/apachectl stop 2> /dev/null || true #launchctl unload -w /System/Library/LaunchDaemons/org.apache.httpd.plist 2> /dev/null || true #fi if [ -f ~/.my.cnf ]; then echo2 echo2 ~/.my.cnf already exists. This will interfere with the installation. echo2 The file will be renamed to ~/.my.cnf.old now. mv ~/.my.cnf ~/.my.cnf.old waitKey fi @@ -1479,76 +1480,79 @@ exit 100 fi } # set this machine for browser development: install required tools, clone the tree, build it function buildTree () { if [[ "$DIST" == "OSX" ]]; then setupBuildOsx else setupBuildLinux fi if [ ! -e ~/kent ]; then echo2 Cloning kent repo into ~/kent using git with --depth=1 + echo2 Branch is: \"beta\" = our current release, beta = testing waitKey cd ~ - git clone https://github.com/ucscGenomeBrowser/kent.git --depth=1 + git clone -b beta https://github.com/ucscGenomeBrowser/kent.git --depth=1 fi echo2 Now building CGIs from ~/kent to /usr/local/apache/cgi-bin echo2 Copying JS/HTML/CSS to /usr/local/apache/htdocs waitKey cd ~/kent/src make -j8 cgi-alpha make -j8 doc-alpha } # main function, installs the browser on Redhat/Debian and potentially even on OSX function installBrowser () { if [ -f $COMPLETEFLAG ]; then echo2 error: the file $COMPLETEFLAG exists. It seems that you have installed the browser already. echo2 If you want to reset the Apache directory, you can run '"rm -rf /usr/local/apache"' and echo2 then run this script again. exit 100 fi echo '--------------------------------' echo UCSC Genome Browser installation echo '--------------------------------' - echo CPU type: $MACH, detected OS: $OS/$DIST, Version: $VERNUM, Release: $VER + echo CPU type: $MACH, detected OS: $OS/$DIST, Release: $VER, Version: $VERNUM echo echo This script will go through three steps: echo "1 - setup apache and mysql, open port 80, deactivate SELinux" echo "2 - copy CGI binaries into $CGIBINDIR, html files into $HTDOCDIR" echo "3 - optional: download genome assembly databases into mysql and /gbdb" echo echo This script will now install and configure MariaDB and Apache if they are not yet installed. echo "Your distribution's package manager will be used for this." echo If MariaDB is not installed yet, it will be installed, secured and a root password defined. echo The MariaDB root password will be written into root\'s \~/.my.cnf echo echo This script will also deactivate SELinux if active and open port 80/http. waitKey # ----- OS - SPECIFIC part ----- if [ ! -f $COMPLETEFLAG ]; then if [[ "$DIST" == "OSX" ]]; then - #installOsx <- not used anymore, but maybe one day - # on OSX, install clang, brew, kent and build the CGIs from scratch + echo2 OSX: build the CGIs from scratch using clang, brew and git + buildTree + elif [[ "$MACH" == "aarch64" ]]; then + echo2 Linux, but ARM CPU: Need to build CGIs and htdocs locally from source using gcc, make and git buildTree elif [[ "$DIST" == "debian" ]]; then installDebian elif [[ "$DIST" == "redhat" ]]; then installRedhat fi fi # OS-specific mysql/apache installers can SET_MYSQL_ROOT to 1 to request that the root # mysql user password be changed # ---- END OS-SPECIFIC part ----- mysqlAllowOldPasswords if [[ "${SET_MYSQL_ROOT}" == "1" ]]; then @@ -1638,79 +1642,98 @@ echo genome-euro seems to be closer echo modifying hg.conf to pull data from genome-euro instead of genome sed -i s/slow-db.host=genome-mysql.soe.ucsc.edu/slow-db.host=genome-euro-mysql.soe.ucsc.edu/ $CGIBINDIR/hg.conf sed -i "s#gbdbLoc2=http://hgdownload.soe.ucsc.edu/gbdb/#gbdbLoc2=http://hgdownload-euro.soe.ucsc.edu/gbdb/#" $CGIBINDIR/hg.conf HGDOWNLOAD=hgdownload-euro.soe.ucsc.edu else echo genome.ucsc.edu seems to be closer echo not modifying $CGIBINDIR/hg.conf fi # download the CGIs if [[ "$OS" == "OSX" ]]; then #setupCgiOsx echo2 Running on OSX, assuming that CGIs are already built into /usr/local/apache/cgi-bin elif [[ "$MACH" == "aarch64" ]]; then - echo2 Running on an ARM CPU: need to build CGIs and htdocs locally from source - buildTree + echo2 Running on an ARM CPU, assuming that CGIs are already built into /usr/local/apache/cgi-bin else # don't download RNAplot, it's a 32bit binary that won't work anywhere anymore but at UCSC # this means that hgGene cannot show RNA structures but that's not a big issue $RSYNC -avzP --exclude=RNAplot $HGDOWNLOAD::cgi-bin/ $CGIBINDIR/ # now add the binaries for dot and RNAplot $RSYNC -avzP $HGDOWNLOAD::genome/admin/exe/external.x86_64/RNAplot $CGIBINDIR/ $RSYNC -avzP $HGDOWNLOAD::genome/admin/exe/external.x86_64/loader/dot_static $CGIBINDIR/loader/ fi # download the html docs, exclude some big files on OSX # try to minimize storage for OSX, mostly laptops if [ "$OS" == "OSX" -o "$MACH" == "aarch64" ]; then #$RSYNC --delete -azP --exclude=training --exclude=ENCODE --exclude=encode --exclude=rosenbloom.pdf --exclude=pubs*.pdf --exclude=*.{bb,bam,bai,bw,gz,2bit} --exclude=goldenpath $HGDOWNLOAD::htdocs/ $HTDOCDIR/ echo2 Not syncing htdocs folder, assuming that these were built from source. echo2 PDF and other large files only present at UCSC will be missing from htdocs. waitKey else rm -rf $APACHEDIR/htdocs/goldenpath $RSYNC -avzP --exclude ENCODE/**.pdf $HGDOWNLOAD::htdocs/ $HTDOCDIR/ fi # assign all files just downloaded to a valid user. # This also allows apache to write into the trash dir if [ "$OS" == "OSX" ]; then - echo2 OSX: Not chowning /usr/local/apache subdirectories + echo2 OSX: Not chowning /usr/local/apache subdirectories, as not running as root else chown -R $APACHEUSER:$APACHEUSER $CGIBINDIR $HTDOCDIR $TRASHDIR fi touch $COMPLETEFLAG echo2 Install complete. You should now be able to point your web browser to this machine echo2 and test your UCSC Genome Browser mirror. It will be too slow for practical use. echo2 echo2 Notice that this mirror is still configured to use Mysql and data files loaded echo2 through the internet from UCSC. From most locations on the world, this is very slow. echo2 It also requires an open outgoing TCP port 3306 for Mysql to genome-mysql.soe.ucsc.edu/genome-euro-mysql.soe.ucsc.edu, echo2 and open TCP port 80 to hgdownload.soe.ucsc.edu/hgdownload-euro.soe.ucsc.edu. echo2 echo2 To finish the installation, you need to download genome data to the local echo2 disk. To download a genome assembly and all its files now, call this script again with echo2 the parameters 'download "<assemblyName1> <assemblyName2> ..."', e.g. '"'bash $0 mirror mm10 hg19'"' echo2 showMyAddress exit 0 } +# mkdir /gbdb or do the weird things one has to do on OSX to make this directory +function mkdirGbdb +{ + if [[ "$OS" != "OSX" ]]; then + mkdir -p $GBDBDIR + return + fi + + sudo mkdir -p /usr/local/gbdb + sudo chmod a+rwx /usr/local/gbdb + + # see https://apple.stackexchange.com/questions/388236/unable-to-create-folder-in-root-of-macintosh-hd + if [[ ! -f /etc/synthetic.conf ]] || grep -vq gbdb /etc/synthetic.conf; then + sudo /bin/sh -c 'echo "gbdb\tusr/local/gbdb" >> /etc/synthetic.conf' + fi + chmod 644 /etc/synthetic.conf + /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -t + echo 'This directory is /usr/local/gbdb, see /etc/synthetic.conf' >> /gbdb/README.txt +} + # GENOME DOWNLOAD: mysql and /gbdb function downloadGenomes { DBS=$* GENBANKTBLS="" if [ "$DBS" == "" ] ; then echo2 Argument error: the '"download"' command requires at least one assembly name, like hg19 or mm10. exit 100 fi echo2 echo2 Downloading databases $DBS plus hgFixed/proteome/go from the UCSC download server echo2 echo2 Determining download file size... please wait... @@ -1720,30 +1743,36 @@ MYSQLDBS="$DBS proteome uniProt go hgFixed" else echo2 Downloading $DBS plus GenBank and RefSeq tables MYSQLDBS="$DBS" GENBANKTBLS="author cell description development gbCdnaInfo gbExtFile gbLoaded \ gbMiscDiff gbSeq gbWarn geneName imageClone keyword library \ mrnaClone organism productName refLink refSeqStatus \ refSeqSummary sex source tissue" fi else MYSQLDBS="$DBS" fi # rsync is doing globbing itself, so switch it off temporarily set -f + + # On OSX the MariaDB datadir is not under /var. On Linux distros, the MariaDB directory may have been moved. + if [ ! -d $MYSQLDIR ]; then + MYSQLDIR=`mysql -NBe 'SHOW Variables WHERE Variable_Name="datadir"' | cut -f2` + fi + # use rsync to get total size of files in directories and sum the numbers up with awk for db in $MYSQLDBS; do rsync -avn $HGDOWNLOAD::mysql/$db/ $MYSQLDIR/$db/ $RSYNCOPTS | grep ^'total size' | cut -d' ' -f4 | tr -d ', ' done | awk '{ sum += $1 } END { print "| Required space in '$MYSQLDIR':", sum/1000000000, "GB" }' if [ ! -z "$GENBANKTBLS" ]; then for tbl in $GENBANKTBLS; do rsync -avn $HGDOWNLOAD::mysql/hgFixed/${tbl}.* $MYSQLDIR/hgFixed/ $RSYNCOPTS | grep ^'total size' | cut -d' ' -f4 | tr -d ', ' done | awk '{ sum += $1 } END { print "| Required space in '$MYSQLDIR'/hgFixed:", sum/1000000000, "GB" }' fi for db in $DBS; do rsync -avn $HGDOWNLOAD::gbdb/$db/ $GBDBDIR/$db/ $RSYNCOPTS | grep ^'total size' | cut -d' ' -f4 | tr -d ',' done | awk '{ sum += $1 } END { print "| Required space in '$GBDBDIR':", sum/1000000000, "GB" }' @@ -1778,34 +1807,35 @@ chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/$db done if [ ! -z "$GENBANKTBLS" ]; then echo2 Downloading hgFixed tables for tbl in $GENBANKTBLS; do $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/${tbl}.* $MYSQLDIR/hgFixed/ done chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/hgFixed fi echo2 Downloading hgFixed.refLink, required for all RefSeq tracks $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/refLink.* $MYSQLDIR/hgFixed/ chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/hgFixed + mkdirGbdb + # download /gbdb files for db in $DBS; do echo2 Downloading $GBDBDIR files for assembly $db - mkdir -p $GBDBDIR $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::gbdb/$db/ $GBDBDIR/$db/ chown -R $APACHEUSER:$APACHEUSER $GBDBDIR/$db done set +f # Alexander Stuy reported that at FSU they had a few mysql databases with incorrect users on them chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/ startMysql mysqlCheck hideSomeTracks