cb3778318684da37020f3a857d5d601e893b3d0b max Fri Nov 17 18:44:53 2023 -0800 applying patch received from alex Stuy, no redmine diff --git src/product/installer/browserSetup.sh src/product/installer/browserSetup.sh index 3f1bce9..1eefcc8 100644 --- src/product/installer/browserSetup.sh +++ src/product/installer/browserSetup.sh @@ -1624,32 +1624,39 @@ echo2 Downloading hgFixed.refLink, required for all RefSeq tracks $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/refLink.* $MYSQLDIR/hgFixed/ chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/hgFixed # download /gbdb files for db in $DBS; do echo2 Downloading $GBDBDIR files for assembly $db mkdir -p $GBDBDIR $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::gbdb/$db/ $GBDBDIR/$db/ chown -R $APACHEUSER:$APACHEUSER $GBDBDIR/$db done set +f + # Alexander Stuy reported that at FSU they had a few mysql databases with incorrect users on them + chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/ + + startMysql + mysqlCheck + hideSomeTracks + goOffline # modify hg.conf and remove all statements that use the UCSC download server echo2 echo2 Install complete. You should now be able to point your web browser to this machine echo2 and use your UCSC Genome Browser mirror. echo2 showMyAddress echo2 If you have not downloaded the human hg38 assembly and you get an error message echo2 'Could not connect to database' on the genome selection page, then modify echo2 the hg.conf file and change the organism, e.g. to Mouse if you downloaded mouse. echo2 with a command like "'nano /usr/local/apache/cgi-bin/hg.conf'" echo2 Search for Human in this file and replace it with the organism of the echo2 you downloaded. @@ -1698,79 +1705,85 @@ elif [ -f /usr/lib/systemd/system/mysql.service ]; then # at least seen in Fedora 17 systemctl start mysql else echo2 Could not find mysql nor mysqld file in /etc/init.d nor a systemd command. Please email genome-mirror@soe.ucsc.edu. fi } function mysqlCheck # check all mysql tables. Rarely, some of them are in an unclosed state on the download server, this command will close them { echo2 Checking all mysql tables after the download to make sure that they are closed mysqlcheck --all-databases --auto-repair --quick --fast --silent } +function hideSomeTracks +# hide the big tracks and the ones that we are not allowed to distribute +{ + # these tables are not used for searches by default. Searches are very slow. We focus on genes. + notSearchTables='wgEncodeGencodeBasicV19 wgEncodeGencodeCompV17 wgEncodeGencodeBasicV14 wgEncodeGencodeBasicV17 wgEncode GencodeCompV14 mgcFullMrna wgEncodeGencodeBasicV7 orfeomeMrna wgEncodeGencodePseudoGeneV14 wgEncodeGencodePseudoGeneV17 wgEncodeGencodePseudoGeneV19 wgEncodeGencodeCompV7 knownGeneOld6 geneReviews transMapAlnSplicedEst gbCdnaInfo oreganno vegaPseudoGene transMapAlnMRna ucscGenePfam qPcrPrimers transMapAlnUcscGenes transMapAlnRefSeq genscan bacEndPairs fosEndPairs' + + # these tracks are hidden by default + hideTracks='intronEst cons100way cons46way ucscRetroAli5 mrna omimGene2 omimAvSnp' + + echo2 Hiding some tracks by default and removing some tracks from searches + for db in $DBS; do + echo $db + for track in $hideTracks; do + mysql $db -e 'UPDATE trackDb set visibility=0 WHERE tableName="'$track'"' + done + + for track in $notSearchTables; do + mysql $db -e 'DELETE from hgFindSpec WHERE searchTable="'$track'"' + done + done +} + # only download a set of minimal mysql tables, to make a genome browser that is using the mysql failover mechanism # faster. This should be fast enough in the US West Coast area and maybe even on the East Coast. function downloadMinimal { DBS=$* if [ "$DBS" == "" ] ; then echo2 Argument error: the '"minimal"' command requires at least one assembly name, like hg19 or mm10. exit 100 fi echo2 echo2 Downloading minimal tables for databases $DBS # only these db tables are copied over by default minRsyncOpt="--include=cytoBand.* --include=chromInfo.* --include=cytoBandIdeo.* --include=kgColor.* --include=knownAttrs.* --include=knownGene.* --include=knownToTag.* --include=kgXref.* --include=ensemblLift.* --include=ucscToEnsembl.* --include=wgEncodeRegTfbsCells.* --include=encRegTfbsClusteredSources.* --include=tableList.* --include=refSeqStatus.* --include=wgEncodeRegTfbsCellsV3.* --include=extFile.* --include=trackDb.* --include=grp.* --include=ucscRetroInfo5.* --include=refLink.* --include=ucscRetroSeq5.* --include=ensemblLift.* --include=knownCanonical.* --include=gbExtFile.* --include=flyBase2004Xref --include=hgFindSpec.* --include=ncbiRefSeq*" - # these tables are not used for searches by default. Searches are very slow. We focus on genes. - notSearchTables='wgEncodeGencodeBasicV19 wgEncodeGencodeCompV17 wgEncodeGencodeBasicV14 wgEncodeGencodeBasicV17 wgEncode GencodeCompV14 mgcFullMrna wgEncodeGencodeBasicV7 orfeomeMrna wgEncodeGencodePseudoGeneV14 wgEncodeGencodePseudoGeneV17 wgEncodeGencodePseudoGeneV19 wgEncodeGencodeCompV7 knownGeneOld6 geneReviews transMapAlnSplicedEst gbCdnaInfo oreganno vegaPseudoGene transMapAlnMRna ucscGenePfam qPcrPrimers transMapAlnUcscGenes transMapAlnRefSeq genscan bacEndPairs fosEndPairs' - - # these tracks are hidden by default - hideTracks='intronEst cons100way cons46way ucscRetroAli5 mrna omimGene2 omimAvSnp' - stopMysql for db in $DBS; do echo2 Downloading Mysql files for mysql database $db $RSYNC $minRsyncOpt --exclude=* --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/$db/ $MYSQLDIR/$db/ chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/$db done echo2 Copying hgFixed.trackVersion, required for most tracks $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/trackVersion.* $MYSQLDIR/hgFixed/ echo2 Copying hgFixed.refLink, required for RefSeq tracks across all species $RSYNC --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/hgFixed/refLink.* $MYSQLDIR/hgFixed/ chown -R $MYSQLUSER:$MYSQLUSER $MYSQLDIR/hgFixed startMysql - echo2 Hiding some tracks by default and removing some tracks from searches - for db in $DBS; do - echo $db - for track in $hideTracks; do - mysql $db -e 'UPDATE trackDb set visibility=0 WHERE tableName="'$track'"' - done - - for track in $notSearchTables; do - mysql $db -e 'DELETE from hgFindSpec WHERE searchTable="'$track'"' - done - done + hideSomeTracks mysqlCheck echo2 echo2 The mirror should be functional now. It contains some basic assembly tables echo2 and will download missing data from the UCSC servers. This requires echo2 two open ports, outgoing, TCP, from this machine: echo2 - to genome-mysql.soe.ucsc.edu, port 3306, to load MySQL tables echo2 - to hgdownload.soe.ucsc.edu, port 80, to download non-MySQL data files echo2 - or the above two servers European counterparts: echo2 genome-euro-mysql.soe.ucsc.edu and hgdownload-euro.soe.ucsc.edu echo2 showMyAddress goOnline } @@ -1821,31 +1834,31 @@ DBS=$* # if none specified, update all if [ "$DBS" == "" ] ; then DBS=`ls $GBDBDIR/` fi # update gbdb echo updating GBDB: $DBS for db in $DBS; do echo2 syncing gbdb: $db rsync -avp $RSYNCOPTS $HGDOWNLOAD::gbdb/$db/ $GBDBDIR/$db/ done # update the mysql DBs stopMysql - DBS=`ls /var/lib/mysql/ | egrep -v '(Trash$)|(hgTemp)|(^ib_)|(^ibdata)|(^aria)|(^mysql)|(performance)|(.flag$)|(hgcentral)'` + DBS=`ls /var/lib/mysql/ | egrep -v '(Trash$)|(hgTemp)|(^ib_)|(^ibdata)|(^aria)|(^mysql)|(performance)|(.flag$)|(multi-master.info)|(sys)|(lost.found)|(hgcentral)'` for db in $DBS; do echo2 syncing full mysql database: $db $RSYNC --update --progress -avp $RSYNCOPTS $HGDOWNLOAD::mysql/$db/ $MYSQLDIR/$db/ done startMysql echo2 update finished } function addTools { rsync -avP hgdownload.soe.ucsc.edu::genome/admin/exe/linux.x86_64/ /usr/local/bin/ rm -rf /usr/local/bin/blat.tmp # in case an old one is still there mv /usr/local/bin/blat /usr/local/bin/blat.tmp # tools under the BLAT license are separated into their own directory mv /usr/local/bin/blat.tmp/* /usr/local/bin/ rmdir /usr/local/bin/blat.tmp