03566d28af0d1630cec3eaae5dd278df3648f074 max Tue Sep 30 05:43:28 2025 -0700 removing hgdownload-euro from installer script, no ML ticket yet diff --git src/product/installer/browserSetup.sh src/product/installer/browserSetup.sh index 3ed3c805fa5..dc59e539798 100755 --- src/product/installer/browserSetup.sh +++ src/product/installer/browserSetup.sh @@ -390,32 +390,30 @@ assembly, switch to on-the-fly mode (see the -f option) bash $0 minimal hg19 - download only the minimal tables for the hg19 assembly bash $0 mirror hg19 mm9 - download hg19 and mm9, switch to offline mode (see the -o option) bash $0 -t noEncode mirror hg19 - install Genome Browser, download hg19 but no ENCODE tables and switch to offline mode (see the -o option) bash $0 update hg19 - update all data and all tables of the hg19 assembly (in total 7TB). Specifying no assemblies will update all assemblies. bash $0 cgiUpdate - update the Genome Browser CGI programs bash $0 clean - remove temporary files older than one day All options have to precede the command. options: - -a - use alternative download server at Univ Bielefeld, Germany - (used by default if faster ping time than to UCSC) -b - batch mode, do not prompt for key presses -t - For the "mirror" command: Track selection, requires a value. This option is only useful for Human/Mouse assemblies. Download only certain tracks, possible values: noEncode = do not download any tables with the wgEncode prefix, except Gencode genes, saves 4TB/6TB for hg19 bestEncode = our ENCODE recommendation, all summary tracks, saves 2TB/6TB for hg19 main = only Gencode genes and common SNPs, 5GB for hg19 -o - switch to offline-mode. Remove all statements from hg.conf that allow loading data on-the-fly from the UCSC download server. Requires that you have downloaded at least one assembly, using the '"download"' command, not the '"mirror"' command. -f - switch to on-the-fly mode. Change hg.conf to allow loading data through the internet, if it is not available locally. The default mode @@ -1663,34 +1661,34 @@ # in OSX adapt the sockets # note that the sed -i syntax is different from linux echo2 Adapting mysql socket locations in $CGIBINDIR/hg.conf sockFile=`mysql -NBe 'show variables like "socket"' | cut -f2` $SEDINPLACE "s|^#?socket=.*|socket=$sockFile|" $CGIBINDIR/hg.conf $SEDINPLACE "s|^#?customTracks.socket.*|customTracks.socket=$sockFile|" $CGIBINDIR/hg.conf $SEDINPLACE "s|^#?db.socket.*|db.socket=$sockFile|" $CGIBINDIR/hg.conf $SEDINPLACE "s|^#?central.socket.*|central.socket=$sockFile|" $CGIBINDIR/hg.conf fi # check if UCSC or genome-euro MySQL server is closer echo comparing latency: genome.ucsc.edu Vs. genome-euro.ucsc.edu eurospeed=$( (time -p (for i in `seq 10`; do curl -sSI genome-euro.ucsc.edu > /dev/null; done )) 2>&1 | grep real | cut -d' ' -f2 ) ucscspeed=$( (time -p (for i in `seq 10`; do curl -sSI genome.ucsc.edu > /dev/null; done )) 2>&1 | grep real | cut -d' ' -f2 ) if [[ $(awk '{if ($1 <= $2) print 1;}' <<< "$eurospeed $ucscspeed") -eq 1 ]]; then echo genome-euro seems to be closer - echo modifying hg.conf to pull data from genome-euro instead of genome + echo modifying hg.conf to pull MariaDB data from genome-euro-mysql instead of genome $SEDINPLACE s/slow-db.host=genome-mysql.soe.ucsc.edu/slow-db.host=genome-euro-mysql.soe.ucsc.edu/ $CGIBINDIR/hg.conf - $SEDINPLACE "s#gbdbLoc2=http://hgdownload.soe.ucsc.edu/gbdb/#gbdbLoc2=http://hgdownload-euro.soe.ucsc.edu/gbdb/#" $CGIBINDIR/hg.conf - HGDOWNLOAD=hgdownload-euro.soe.ucsc.edu + #$SEDINPLACE "s#gbdbLoc2=http://hgdownload.soe.ucsc.edu/gbdb/#gbdbLoc2=http://hgdownload-euro.soe.ucsc.edu/gbdb/#" $CGIBINDIR/hg.conf + #HGDOWNLOAD=hgdownload-euro.soe.ucsc.edu else echo genome.ucsc.edu seems to be closer echo not modifying $CGIBINDIR/hg.conf fi # download the CGIs if [[ "$OS" == "OSX" ]]; then #setupCgiOsx echo2 Running on OSX, assuming that CGIs are already built into /usr/local/apache/cgi-bin elif [[ "$MACH" == "aarch64" ]]; then echo2 Running on an ARM CPU, assuming that CGIs are already built into /usr/local/apache/cgi-bin else # don't download RNAplot, it's a 32bit binary that won't work anywhere anymore but at UCSC # this means that hgGene cannot show RNA structures but that's not a big issue $RSYNC -avzP --exclude=RNAplot $HGDOWNLOAD::cgi-bin/ $CGIBINDIR/ @@ -1716,31 +1714,31 @@ # This also allows apache to write into the trash dir if [ "$OS" == "OSX" ]; then echo2 OSX: Not chowning /usr/local/apache subdirectories, as not running as root else chown -R $APACHEUSER:$APACHEUSER $CGIBINDIR $HTDOCDIR $TRASHDIR fi touch $COMPLETEFLAG echo2 Install complete. You should now be able to point your web browser to this machine echo2 and test your UCSC Genome Browser mirror. It will be too slow for practical use. echo2 echo2 Notice that this mirror is still configured to use Mysql and data files loaded echo2 through the internet from UCSC. From most locations on the world, this is very slow. echo2 It also requires an open outgoing TCP port 3306 for Mysql to genome-mysql.soe.ucsc.edu/genome-euro-mysql.soe.ucsc.edu, - echo2 and open TCP port 80 to hgdownload.soe.ucsc.edu/hgdownload-euro.soe.ucsc.edu. + echo2 and open TCP port 80 to hgdownload.soe.ucsc.edu echo2 echo2 To finish the installation, you need to download genome data to the local echo2 disk. To download a genome assembly and all its files now, call this script again with echo2 the parameters 'download "<assemblyName1> <assemblyName2> ..."', e.g. '"'bash $0 mirror mm10 hg19'"' echo2 showMyAddress exit 0 } # mkdir /gbdb or do the weird things one has to do on OSX to make this directory function mkdirGbdb { if [[ "$OS" != "OSX" ]]; then # On Linux, create /gbdb and let the apache user write to it mkdir -p $GBDBDIR @@ -2024,32 +2022,31 @@ $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/refLink.* $MYSQLDIR/hgFixed/ chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/hgFixed startMysql mysqlCheck hideSomeTracks echo2 echo2 The mirror should be functional now. It contains some basic assembly tables echo2 and will download missing data from the UCSC servers. This requires echo2 two open ports, outgoing, TCP, from this machine: echo2 - to genome-mysql.soe.ucsc.edu, port 3306, to load MySQL tables echo2 - to hgdownload.soe.ucsc.edu, port 80, to download non-MySQL data files - echo2 - or the above two servers European counterparts: - echo2 genome-euro-mysql.soe.ucsc.edu and hgdownload-euro.soe.ucsc.edu + echo2 - or its counterpart: genome-euro-mysql.soe.ucsc.edu echo2 showMyAddress goOnline } function checkDownloadUdr () # download the faster downloader udr that is compatible with rsync and change the global variable RSYNC { if [[ "$OS" == "OSX" ]]; then return fi if [[ "$MACH" == "aarch64" ]]; then return fi @@ -2146,44 +2143,44 @@ # On Debian and OSX, sudo by default does not update the HOME variable (hence the -H option above) if [[ "${SUDO_USER:-}" != "" ]]; then export HOME=~root fi trap errorHandler ERR # OPTION PARSING # show help message if no argument is specified if [[ $# -eq 0 ]] ; then echo "$HELP_STR" exit 0 fi -while getopts ":baeut:hof" opt; do +while getopts ":beut:hof" opt; do case $opt in h) echo "$HELP_STR" exit 0 ;; b) function waitKey { echo } ;; - a) - HGDOWNLOAD=hgdownload-euro.soe.ucsc.edu - ;; + #a) + #HGDOWNLOAD=hgdownload-euro.soe.ucsc.edu + #;; t) val=${OPTARG} # need to include all subdirectories for include to work # need to exclude everything else for exclude to work if [[ "$val" == "bestEncode" ]]; then RSYNCOPTS="-m --include=wgEncodeGencode* --include=wgEncodeBroadHistone* --include=wgEncodeReg* --include=wgEncodeAwg* --include=wgEncode*Mapability* --include=*/ --exclude=wgEncode*" ONLYGENOMES=0 elif [[ "$val" == "noEncode" ]]; then RSYNCOPTS="-m --include=wgEncodeGencode* --include=*/ --exclude=wgEncode*" ONLYGENOMES=0 elif [[ "$val" == "main" ]]; then # gbCdnaInfo # SNP table selection explained in #17335 RSYNCOPTS="-m --include=grp.* --include=*gold* --include=augustusGene.* --include=chromInfo.* --include=cpgIslandExt.* --include=cpgIslandExtUnmasked.* --include=cytoBandIdeo.* --include=genscan.* --include=microsat.* --include=simpleRepeat.* --include=tableDescriptions.* --include=ucscToINSDC.* --include=windowmaskerSdust.* --include=gold.* --include=chromInfo.* --include=trackDb* --include=hgFindSpec.* --include=gap.* --include=*.2bit --include=html/description.html --include=refGene* --include=refLink.* --include=wgEncodeGencode* --include=snp146Common* --include=snp130* --include=snp142Common* --include=snp128* --include=gencode* --include=rmsk* --include=*/ --exclude=*" ONLYGENOMES=1 # do not download hgFixed,go,proteome etc