5adcf6bc2904690de7b7b30a83ec8a7a0996abe9 galt Tue Aug 21 00:01:25 2018 -0700 changing cse subdomain to soe diff --git src/browserbox/root/updateBrowser.sh src/browserbox/root/updateBrowser.sh index 53beca9..b8fe897 100755 --- src/browserbox/root/updateBrowser.sh +++ src/browserbox/root/updateBrowser.sh @@ -32,34 +32,34 @@ # In case something goes wrong it continues, this is intentional to # avoid a virtual machine that can not update itself anymore # rsync options: # l = preserve symlinks # t = preserve time # r = recurse # z = compress # v = verbose # h = human readable # u = skip if file is newer on receiver # We are not using the -z option anymore because it may cause # CPU overload on hgdownload RSYNCOPTS="-ltrvh" # rsync server for CGIs and html files -RSYNCSRC="rsync://hgdownload.cse.ucsc.edu" +RSYNCSRC="rsync://hgdownload.soe.ucsc.edu" RSYNCCGIBIN=cgi-bin RSYNCHTDOCS=htdocs -UPDATEFLAG=http://hgdownload.cse.ucsc.edu/gbib/lastUpdate +UPDATEFLAG=http://hgdownload.soe.ucsc.edu/gbib/lastUpdate LOGFILE=/var/log/gbibUpdates.log DEBUG=0 if [[ "$#" -ne 0 ]]; then DEBUG=1 fi # function to echo only if run with some arguments function echoDebug { if [[ DEBUG -eq "1" ]]; then echo $* fi } # make sure that apt-get never opens an interactive dialog export DEBIAN_FRONTEND=noninteractive @@ -302,31 +302,31 @@ rsync $RSYNCAPACHE $user@hgwdev.soe.ucsc.edu:/usr/local/apache/cgi-bin${cgiExt}/ /usr/local/apache/cgi-bin/ PUSHLOC=$user@hgwdev.soe.ucsc.edu:/usr/local/apache/htdocs/gbib/push/ # normal public updates from hgdownload are easier, not many excludes necessary else # update CGIs echo - Updating CGIs... rsync --delete -u $RSYNCOPTS $RSYNCSRC/$RSYNCCGIBIN /usr/local/apache/cgi-bin/ --exclude=hg.conf --exclude=hg.conf.local --exclude edw* --exclude *private --exclude hgNearData --exclude visiGeneData --exclude Neandertal echo - Updating HTML files... # not using -u because we had a case with a 0-byte html page that was # not updated anymore in #18337 rsync --delete $RSYNCOPTS $RSYNCSRC/$RSYNCHTDOCS/ /usr/local/apache/htdocs/ --include **/customTracks/*.html --exclude ENCODE/ --exclude *.bam --exclude *.bb --exclude */*.bw --exclude */*.gz --exclude favicon.ico --exclude folders --exclude ancestors --exclude admin --exclude goldenPath/customTracks --exclude images/mammalPsg --exclude style/gbib.css --exclude images/title.jpg --exclude images/homeIconSprite.png --exclude goldenPath/**.pdf --exclude training - PUSHLOC=hgdownload.cse.ucsc.edu::gbib/push/ + PUSHLOC=hgdownload.soe.ucsc.edu::gbib/push/ fi chown -R www-data.www-data /usr/local/apache/cgi-bin/* chown -R www-data.www-data /usr/local/apache/htdocs/ chmod -R a+r /usr/local/apache/htdocs # June 2017: add a basic set of hg38 files to the GBIB # This will add 1.2 GB to the size of the virtual disc but hg38 # is the default genome so probably should be included # Touching the files once is enough. Rsync will download them. mkdir -p /data/gbdb/hg38 mkdir -p /data/gbdb/hg38/targetDb/ mkdir -p /data/gbdb/hg38/html/ mkdir -p /data/mysql/hg38 @@ -338,31 +338,31 @@ # MySQL tables for i in chromInfo cytoBand cytoBandIdeo ensemblLift extFile grp gtexGene gtexGeneModel hgFindSpec kgColor kgXref knownCanonical knownGene knownToTag ncbiRefSeq ncbiRefSeqCurated ncbiRefSeqLink ncbiRefSeqOther ncbiRefSeqPredicted ncbiRefSeqPsl refGene tableList trackDb ucscToEnsembl ucscToINSDC xenoRefGene; do touch /data/mysql/hg38/$i.MYD; touch /data/mysql/hg38/$i.MYI; touch /data/mysql/hg38/$i.frm; done # adding tables that are required for gtex, which is now a default track, #19587 touch /data/mysql/hgFixed/gtexInfo.{MYI,MYD,frm} touch /data/mysql/hgFixed/gtexTissue.{MYI,MYD,frm} # -- END JUNE 2017 if [ "$1" != "hgwdev" ] ; then echo - Updating GBDB files... - rsync $RSYNCOPTS --existing rsync://hgdownload.cse.ucsc.edu/gbdb/ /data/gbdb/ + rsync $RSYNCOPTS --existing rsync://hgdownload.soe.ucsc.edu/gbdb/ /data/gbdb/ chown -R www-data.www-data /data/gbdb/ fi echo - Pulling other files # make sure we never overwrite the hg.conf.local file rsync $RSYNCOPTS $PUSHLOC / --exclude=hg.conf.local # July 2016: add the cram fetcher to root's crontab # this has to be done after the PUSHLOC directory has been copied over if grep -q fetchCramReference /var/spool/cron/crontabs/root; then true else crontab -l | awk '{print} END {print "\n# handle CRAM auto reference download\n*/1 * * * * /root/fetchCramReference.sh /data/cramCache/pending /data/cramCache/ /data/cramCache/error/\n"}' | crontab - fi # also create the directories for the cram files @@ -389,36 +389,36 @@ else echo Allowing LOAD DATA in MySQL and restart MySQL sed -i '/\[mysqld\]/a secure-file-priv = ""' /etc/mysql/my.cnf service mysql restart fi # we can now remove the old tables rm -f /data/mysql/hg19/refSeqStatus* if [ "$1" != "hgwdev" ] ; then echo updating MYSQL files - browser will not work during the MYSQL update # inspired by http://forums.mysql.com/read.php?35,45577,47063#msg-47063 # it doesn't work if I use two mysql invocations, as 'flush tables with read lock' # is only valid as long as the session is open # so I use the SYSTEM command - echo "FLUSH TABLES WITH READ LOCK; SYSTEM rsync $RSYNCOPTS --existing rsync://hgdownload.cse.ucsc.edu/mysql/ /data/mysql/; SYSTEM chown -R mysql.mysql /data/mysql/; UNLOCK TABLES;" | mysql + echo "FLUSH TABLES WITH READ LOCK; SYSTEM rsync $RSYNCOPTS --existing rsync://hgdownload.soe.ucsc.edu/mysql/ /data/mysql/; SYSTEM chown -R mysql.mysql /data/mysql/; UNLOCK TABLES;" | mysql echo updating hgcentral database, make sure to always overwrite - echo "FLUSH TABLES WITH READ LOCK; SYSTEM rsync -vrz --existing rsync://hgdownload.cse.ucsc.edu/mysql/hgcentral/ /data/mysql/hgcentral/; SYSTEM chown -R mysql.mysql /data/mysql/hgcentral; UNLOCK TABLES;" | mysql + echo "FLUSH TABLES WITH READ LOCK; SYSTEM rsync -vrz --existing rsync://hgdownload.soe.ucsc.edu/mysql/hgcentral/ /data/mysql/hgcentral/; SYSTEM chown -R mysql.mysql /data/mysql/hgcentral; UNLOCK TABLES;" | mysql # update blat servers - mysql hgcentral -e 'UPDATE blatServers SET host=CONCAT(host,".cse.ucsc.edu") WHERE host not like "%ucsc.edu"' + mysql hgcentral -e 'UPDATE blatServers SET host=CONCAT(host,".soe.ucsc.edu") WHERE host not like "%ucsc.edu"' # the box does not officially support the HAL right now, remove the ecoli hubs mysql hgcentral -e 'delete from hubPublic where hubUrl like "%nknguyen%"' fi echo - Adapting the menu cp /usr/local/apache/htdocs/inc/globalNavBar.inc /tmp/navbar.inc # remove mirrors and downloads menu sed -i '/<li class="menuparent" id="mirrors">/,/^<\/li>$/d' /tmp/navbar.inc sed -i '/<li class="menuparent" id="downloads">/,/^<\/li>$/d' /tmp/navbar.inc # adding the link to the mirror tracks tool sed -i '/hgLiftOver/a <li><a href="../cgi-bin/hgMirror">Mirror tracks</a></li>' /tmp/navbar.inc # add a link to the gbib shared data folder sed -i '/Track Hubs/a <li><a target="_blank" href="http:\/\/127.0.0.1:1234\/folders\/">GBiB Shared Data Folder<\/a><\/li>' /tmp/navbar.inc # adding a link to the GBIB help pages sed -i '/genomewiki/a <li><a href="../goldenPath/help/gbib.html">Help on GBiB</a></li>' /tmp/navbar.inc @@ -474,26 +474,26 @@ echo Creating /usr/local/apache/cgi-bin/hg.conf.local echo allowHgMirror=true > /usr/local/apache/cgi-bin/hg.conf.local fi # Sept 2017: check if genome-euro mysql server is closer if [ ! -f /usr/local/apache/trash/registration.txt ]; then # Mar 2018: check if we can connect to genome-euro before checking what is closest curl -sSI genome-euro.ucsc.edu 2>&1 > /dev/null if [[ $? -eq 0 ]]; then echo comparing latency: genome.ucsc.edu Vs. genome-euro.ucsc.edu euroSpeed=$( (time -p (for i in `seq 10`; do curl -sSI genome-euro.ucsc.edu > /dev/null; done )) 2>&1 | grep real | cut -d' ' -f2 ) ucscSpeed=$( (time -p (for i in `seq 10`; do curl -sSI genome.ucsc.edu > /dev/null; done )) 2>&1 | grep real | cut -d' ' -f2 ) if [[ $(awk '{if ($1 <= $2) print 1;}' <<< "$euroSpeed $ucscSpeed") -eq 1 ]]; then echo genome-euro seems to be closer echo modifying gbib to pull data from genome-euro instead of genome - sed -i s/slow-db.host=genome-mysql.cse.ucsc.edu/slow-db.host=genome-euro-mysql.soe.ucsc.edu/ /usr/local/apache/cgi-bin/hg.conf + sed -i s/slow-db.host=genome-mysql.soe.ucsc.edu/slow-db.host=genome-euro-mysql.soe.ucsc.edu/ /usr/local/apache/cgi-bin/hg.conf else echo genome.ucsc.edu seems to be closer echo not modifying /usr/local/apache/cgi-bin/hg.conf fi fi fi touch /root/lastUpdateTime.flag echo - GBiB update done cat /etc/issue | tr -s '\n'