1e43c3834a08e0bcc8d1833376d389e231e3d7cb max Wed Feb 3 06:28:54 2021 -0800 fixing a bug in the previous gbib solution that removed the symlinks, refs #26866 diff --git src/browserbox/root/updateBrowser.sh src/browserbox/root/updateBrowser.sh index e7676a1..1665054 100755 --- src/browserbox/root/updateBrowser.sh +++ src/browserbox/root/updateBrowser.sh @@ -295,44 +295,44 @@ # remove things that are on hgwdev beta directories but not necessary on the gbib if [ "$dirExt" == "beta" ] ; then RSYNCAPACHE="$RSYNCAPACHE --exclude favicon*.ico --exclude hg.conf* --exclude ENCODE --exclude *.gz --exclude *.bw --exclude *.bb --exclude *.bam --exclude goldenPath/**.pdf --exclude admin --exclude goldenPath/customTracks --exclude pubs --exclude ancestors --exclude training --exclude trash --exclude .htaccess --exclude htdocs --exclude Neandertal --exclude RNA-img --exclude ebola --exclude encodeDCC --exclude evoFold --exclude geneExtra --exclude js-public --exclude style-public --exclude hgNearData --exclude visiGeneData --exclude visiGene" fi rsync $RSYNCAPACHE $user@hgwdev.soe.ucsc.edu:/usr/local/apache/htdocs${htmlExt}/ /usr/local/apache/htdocs/ rsync $RSYNCAPACHE $user@hgwdev.soe.ucsc.edu:/usr/local/apache/cgi-bin${cgiExt}/ /usr/local/apache/cgi-bin/ PUSHLOC=$user@hgwdev.soe.ucsc.edu:/usr/local/apache/htdocs/gbib/push/ # normal public updates from hgdownload are easier, not many excludes necessary else # Feb 2021: cgi-bin and htdocs are now too big for the root partition. Moving both to the data partition. if [ ! -L /usr/local/apache/htdocs ] ; then - rsync -avp /usr/local/apache/htdocs/ /data/htdocs/ && rm -rf /usr/local/apache/htdocs && ln -s /usr/local/apache/htdocs /data/htdocs + rsync -avp /usr/local/apache/htdocs/ /data/htdocs/ && rm -rf /usr/local/apache/htdocs && ln -s /data/htdocs /usr/local/apache/htdocs fi if [ ! -L /usr/local/apache/cgi-bin ] ; then - rsync -avp /usr/local/apache/cgi-bin/ /data/cgi-bin/ && rm -rf /usr/local/apache/cgi-bin && ln -s /usr/local/apache/cgi-bin /data/cgi-bin + rsync -avp /usr/local/apache/cgi-bin/ /data/cgi-bin/ && rm -rf /usr/local/apache/cgi-bin && ln -s /data/cgi-bin /usr/local/apache/cgi-bin fi # update CGIs echo - Updating CGIs... - rsync --delete -u $RSYNCOPTS $RSYNCSRC/$RSYNCCGIBIN /usr/local/apache/cgi-bin/ --exclude=hg.conf --exclude=hg.conf.local --exclude edw* --exclude *private --exclude hgNearData --exclude visiGeneData --exclude Neandertal + rsync --delete -u $RSYNCOPTS $RSYNCSRC/$RSYNCCGIBIN /data/cgi-bin/ --exclude=hg.conf --exclude=hg.conf.local --exclude edw* --exclude *private --exclude hgNearData --exclude visiGeneData --exclude Neandertal echo - Updating HTML files... # not using -u because we had a case with a 0-byte html page that was # not updated anymore in #18337 - rsync --delete $RSYNCOPTS $RSYNCSRC/$RSYNCHTDOCS/ /usr/local/apache/htdocs/ --include **/customTracks/*.html --exclude ENCODE/ --exclude *.bam --exclude *.bb --exclude */*.bw --exclude */*.gz --exclude favicon.ico --exclude folders --exclude ancestors --exclude admin --exclude goldenPath/customTracks --exclude images/mammalPsg --exclude style/gbib.css --exclude images/title.jpg --exclude images/homeIconSprite.png --exclude goldenPath/**.pdf --exclude training + rsync --delete $RSYNCOPTS $RSYNCSRC/$RSYNCHTDOCS/ /data/htdocs/ --include **/customTracks/*.html --exclude ENCODE/ --exclude *.bam --exclude *.bb --exclude */*.bw --exclude */*.gz --exclude favicon.ico --exclude folders --exclude ancestors --exclude admin --exclude goldenPath/customTracks --exclude images/mammalPsg --exclude style/gbib.css --exclude images/title.jpg --exclude images/homeIconSprite.png --exclude goldenPath/**.pdf --exclude training PUSHLOC=hgdownload.soe.ucsc.edu::gbib/push/ fi chown -R www-data.www-data /usr/local/apache/cgi-bin/* chown -R www-data.www-data /usr/local/apache/htdocs/ chmod -R a+r /usr/local/apache/htdocs # June 2017: add a basic set of hg38 files to the GBIB # This will add 1.2 GB to the size of the virtual disc but hg38 # is the default genome so probably should be included # Touching the files once is enough. Rsync will download them. mkdir -p /data/gbdb/hg38 mkdir -p /data/gbdb/hg38/targetDb/ mkdir -p /data/gbdb/hg38/html/ @@ -345,45 +345,46 @@ # MySQL tables for i in chromInfo cytoBand cytoBandIdeo ensemblLift extFile grp gtexGene gtexGeneModel hgFindSpec kgColor kgXref knownCanonical knownGene knownToTag ncbiRefSeq ncbiRefSeqCurated ncbiRefSeqLink ncbiRefSeqOther ncbiRefSeqPredicted ncbiRefSeqPsl refGene tableList trackDb ucscToEnsembl ucscToINSDC xenoRefGene; do touch /data/mysql/hg38/$i.MYD; touch /data/mysql/hg38/$i.MYI; touch /data/mysql/hg38/$i.frm; done # adding tables that are required for gtex, which is now a default track, #19587 touch /data/mysql/hgFixed/gtexInfo.{MYI,MYD,frm} touch /data/mysql/hgFixed/gtexTissue.{MYI,MYD,frm} # -- END JUNE 2017 # Feb 2021, knownCds is required for knownGene display -for db in hg19 hg38 mm10; do +for db in hg38 mm10; do if [ -e "/data/mysql/$db" ] ; then touch /data/mysql/$db/knownCds.{MYI,MYD,frm} fi done if [ "$1" != "hgwdev" ] ; then echo - Updating GBDB files... rsync $RSYNCOPTS --existing rsync://hgdownload.soe.ucsc.edu/gbdb/ /data/gbdb/ chown -R www-data.www-data /data/gbdb/ fi echo - Pulling other files # make sure we never overwrite the hg.conf.local file -rsync $RSYNCOPTS $PUSHLOC / --exclude=hg.conf.local +# Feb 2021: requires -K now since /usr/local/apache/{cgi-bin,htdocs} are symlinks +rsync -ltrvhK --exclude=hg.conf.local $PUSHLOC / # July 2016: add the cram fetcher to root's crontab # this has to be done after the PUSHLOC directory has been copied over if grep -q fetchCramReference /var/spool/cron/crontabs/root; then true else crontab -l | awk '{print} END {print "\n# handle CRAM auto reference download\n*/1 * * * * /root/fetchCramReference.sh /data/cramCache/pending /data/cramCache/ /data/cramCache/error/\n"}' | crontab - fi # also create the directories for the cram files # cram also requires that the directories are writable by the apache user if [ ! -d /data/cramCache ]; then mkdir -p /data/cramCache/pending /data/cramCache/error chown -R www-data:www-data /data/cramCache fi # -- end July 2016