879760b7b88eacf89dab7012af3a9b030ba351b9
max
  Fri Jun 9 12:22:23 2017 -0700
updating the updateBrowser script in the tree, refs #19587

diff --git src/browserbox/root/updateBrowser.sh src/browserbox/root/updateBrowser.sh
index e267452..b15de9e 100755
--- src/browserbox/root/updateBrowser.sh
+++ src/browserbox/root/updateBrowser.sh
@@ -28,31 +28,33 @@
 # - parameter "hgwdev": does not update itself, copies only the beta/alpha CGIs/htdocs from hgwdev
 # - parameter "notSelf": does not update itself and does not check flagfile
 
 # this script is not using the bash options pipefail or errabort.
 # In case something goes wrong it continues, this is intentional to 
 # avoid a virtual machine that can not update itself anymore
 
 # rsync options:
 # l = preserve symlinks
 # t = preserve time
 # r = recurse
 # z = compress
 # v = verbose
 # h = human readable
 # u = skip if file is newer on receiver
-RSYNCOPTS="-ltrzvh"
+# We are not using the -z option anymore because it may cause
+# CPU overload on hgdownload
+RSYNCOPTS="-ltrvh"
 # rsync server for CGIs and html files
 RSYNCSRC="rsync://hgdownload.cse.ucsc.edu"
 RSYNCCGIBIN=cgi-bin
 RSYNCHTDOCS=htdocs
 UPDATEFLAG=http://hgdownload.cse.ucsc.edu/gbib/lastUpdate
 LOGFILE=/var/log/gbibUpdates.log
 DEBUG=0
 if [[ "$#" -ne 0 ]]; then
     DEBUG=1
 fi
 
 # function to echo only if run with some arguments
 function echoDebug {
    if [[ DEBUG -eq "1" ]]; then
      echo $*
@@ -240,31 +242,31 @@
    apt-get update
    apt-get --no-install-recommends install -y python-mysqldb
    apt-get -y autoremove
 fi
 
 echo
 echo - Updating the genome browser software via rsync:
 
 # CGI-BIN and HTDOCS:
 # the parameter "hgwdev" copies over only the beta/alpha CGIs from hgwdev
 if [ "$1" == "hgwdev" ] ; then
     # note the missing -u option to RSYNC: in hgwdev mode, we want to overwrite everything.
     # On a development machine, the developer might have touched a file
     # for testing. We want to make sure that all local files are overwritten by the 
     # files on hgwdev
-    RSYNCOPTS="-ltrzvh"
+    RSYNCOPTS="-ltrvh"
     user=$2
     dirExt=$3
 
     if [ "$user" == "" ]; then
         echo arguments: updateBrowser hgwdev hgwdevUsername cgiDirectoryExtension 
         echo in alpha/beta mode you need to provide a username for the hgwdev login
         echo and a directory extension, the part after /usr/local/apache/cgi-bin-XXX
         echo The extension '"alpha"' is translated to '"no extension"'
         echo example '"updateBrowser hgwdev kent alpha"'
         echo example '"updateBrowser hgwdev hiram beta"'
         exit 5
     fi
 
     if [ "$dirExt" == "alpha" ] ; then
     	cgiExt=""
@@ -297,30 +299,51 @@
     echo - Updating CGIs...
     rsync --delete -u $RSYNCOPTS $RSYNCSRC/$RSYNCCGIBIN /usr/local/apache/cgi-bin/ --exclude=hg.conf --exclude=hg.conf.local --exclude edw* --exclude *private --exclude hgNearData --exclude visiGeneData --exclude Neandertal 
 
     echo - Updating HTML files...
     # not using -u because we had a case with a 0-byte html page that was 
     # not updated anymore in #18337
     rsync --delete $RSYNCOPTS $RSYNCSRC/$RSYNCHTDOCS/ /usr/local/apache/htdocs/ --include **/customTracks/*.html --exclude ENCODE/ --exclude *.bam --exclude *.bb --exclude */*.bw --exclude */*.gz --exclude favicon.ico --exclude folders --exclude ancestors --exclude admin --exclude goldenPath/customTracks --exclude images/mammalPsg --exclude style/gbib.css --exclude images/title.jpg --exclude images/homeIconSprite.png --exclude goldenPath/**.pdf --exclude training
 
     PUSHLOC=hgdownload.cse.ucsc.edu::gbib/push/
 fi
 
 chown -R www-data.www-data /usr/local/apache/cgi-bin/*
 chown -R www-data.www-data /usr/local/apache/htdocs/
 chmod -R a+r /usr/local/apache/htdocs
 
+# June 2017: add a basic set of hg38 files to the GBIB
+# This will add 1.2 GB to the size of the virtual disc but hg38
+# is the default genome so probably should be included
+# Touching the files once is enough. Rsync will download them.
+mkdir -p /data/gbdb/hg38
+mkdir -p /data/gbdb/hg38/targetDb/
+mkdir -p /data/mysql/hg38
+
+# GBDB files
+for i in hg38.2bit html/description.html knownGene.ix knownGene.ixx knownGene.bb targetDb/kgTargetSeq10.2bit targetDb/kgTargetSeq8.2bit targetDb/kgTargetSeq9.2bit trackDb.ix trackDb.ixx; do
+   touch /data/gbdb/hg38/$i;
+done
+
+# MySQL tables
+for i in chromInfo cytoBand cytoBandIdeo ensemblLift extFile grp gtexGene gtexGeneModel hgFindSpec kgColor kgXref knownCanonical knownGene knownToTag ncbiRefSeq ncbiRefSeqCurated ncbiRefSeqLink ncbiRefSeqOther ncbiRefSeqPredicted ncbiRefSeqPsl refGene tableList trackDb ucscToEnsembl ucscToINSDC xenoRefGene; do
+   touch /data/mysql/$i.MYD;
+   touch /data/mysql/$i.MYI;
+   touch /data/mysql/$i.frm;
+done
+# -- END JUNE 2017
+
 if [ "$1" != "hgwdev" ] ; then
   echo - Updating GBDB files...
   rsync $RSYNCOPTS --existing rsync://hgdownload.cse.ucsc.edu/gbdb/ /data/gbdb/
   chown -R www-data.www-data /data/gbdb/
 fi
 
 echo - Pulling other files
 # make sure we never overwrite the hg.conf.local file
 rsync $RSYNCOPTS $PUSHLOC / --exclude=hg.conf.local
 
 # July 2016: add the cram fetcher to root's crontab 
 # this has to be done after the PUSHLOC directory has been copied over
 if grep -q fetchCramReference /var/spool/cron/crontabs/root; then
         true
    else
@@ -331,30 +354,40 @@
 if [ ! -d /data/cramCache ]; then
     mkdir -p /data/cramCache/pending /data/cramCache/error
     chown -R www-data:www-data /data/cramCache
 fi
 # -- end July 2016
 
 # July 2016: genbank tables are now in hgFixed. By touching a few files, we make sure that at least refseqStatus 
 # is rsync'ed later, otherwise gbib is really slow, refs #17842
 touch /data/mysql/hgFixed/refSeqStatus.MYI /data/mysql/hgFixed/refSeqStatus.MYD /data/mysql/hgFixed/refSeqStatus.frm 
 touch /data/mysql/hgFixed/refLink.MYI /data/mysql/hgFixed/refLink.MYD /data/mysql/hgFixed/refLink.frm 
 
 # Jan 2017: hgVai does not work if /data/mysql/hg19/wgEncodeRegTfbsClusteredInputsV3 is not present, so force this 
 # table into the rsync, refs #18778
 touch /data/mysql/hg19/wgEncodeRegTfbsClusteredInputsV3.{frm,MRI,MYD}
 
+# Jun 2017: An Ubuntu security update in early 2017 deactivated LOAD DATA in Mysql
+# so we are switching it back on
+if grep -q secure-file-priv /etc/mysql/my.cnf; then
+    true
+else
+    echo Allowing LOAD DATA in MySQL again and restarat MySQL
+    sed -i '/\[mysqld\]/a secure-file-priv = ""' /etc/mysql/my.cnf
+    service mysql restart
+fi
+
 # we can now remove the old tables
 rm -f /data/mysql/hg19/refSeqStatus*
 
 if [ "$1" != "hgwdev" ] ; then
   echo updating MYSQL files - browser will not work during the MYSQL update
   # inspired by http://forums.mysql.com/read.php?35,45577,47063#msg-47063
   # it doesn't work if I use two mysql invocations, as 'flush tables with read lock'
   # is only valid as long as the session is open
   # so I use the SYSTEM command
   echo "FLUSH TABLES WITH READ LOCK; SYSTEM rsync $RSYNCOPTS --existing rsync://hgdownload.cse.ucsc.edu/mysql/ /data/mysql/; SYSTEM chown -R mysql.mysql /data/mysql/; UNLOCK TABLES;" | mysql
   
   echo updating hgcentral database, make sure to always overwrite
   echo "FLUSH TABLES WITH READ LOCK; SYSTEM rsync -vrz --existing rsync://hgdownload.cse.ucsc.edu/mysql/hgcentral/ /data/mysql/hgcentral/; SYSTEM chown -R mysql.mysql /data/mysql/hgcentral; UNLOCK TABLES;" | mysql
   # update blat servers
   mysql hgcentral -e 'UPDATE blatServers SET host=CONCAT(host,".cse.ucsc.edu") WHERE host not like "%ucsc.edu"'