9d279d553cfcde698a092b45550c8a1d5341b62f hiram Mon Jun 15 10:40:15 2020 -0700 starting crispr run refs #25720 diff --git src/hg/makeDb/doc/panPan3/initialBuild.txt src/hg/makeDb/doc/panPan3/initialBuild.txt index 4a92332..8b351d4 100644 --- src/hg/makeDb/doc/panPan3/initialBuild.txt +++ src/hg/makeDb/doc/panPan3/initialBuild.txt @@ -807,138 +807,144 @@ # real 72m34.088s cat fb.panPan3.chainMm10Link.txt # 1017872526 bases of 2999027915 (33.940%) in intersection cat fb.panPan3.chainSynMm10Link.txt # 880983055 bases of 2999027915 (29.376%) in intersection time (doRecipBest.pl -load -workhorse=hgwdev panPan3 mm10 \ -buildDir=`pwd` -workhorse=hgwdev) > rbest.log 2>&1 & # real 237m38.959s cat fb.panPan3.chainRBest.Mm10.txt # 883663662 bases of 2999027915 (29.465%) in intersection ############################################################################# -# augustus gene track (TBD - 2019-11-20 - Hiram) +# augustus gene track (DONE - 2020-06-15 - Hiram) mkdir /hive/data/genomes/panPan3/bed/augustus cd /hive/data/genomes/panPan3/bed/augustus time (doAugustus.pl -buildDir=`pwd` -bigClusterHub=ku \ -species=human -dbHost=hgwdev \ -workhorse=hgwdev panPan3) > do.log 2>&1 +XXX - running - Mon Jun 15 10:21:59 PDT 2020 # real 139m55.244s cat fb.panPan3.augustusGene.txt # 55005426 bases of 2999027915 (1.834%) in intersection ######################################################################### # ncbiRefSeq (DONE - 2020-06-13 - Hiram) mkdir /hive/data/genomes/panPan3/bed/ncbiRefSeq.2020-06-13 cd /hive/data/genomes/panPan3/bed/ncbiRefSeq.2020-06-13 time (~/kent/src/hg/utils/automation/doNcbiRefSeq.pl -buildDir=`pwd` \ -bigClusterHub=ku -dbHost=hgwdev \ -fileServer=hgwdev -smallClusterHub=hgwdev -workhorse=hgwdev \ GCF_013052645.1_Mhudiblu_PPA_v0 panPan3) > do.log 2>&1 & -XXX - running - Sat Jun 13 10:10:09 PDT 2020 - # real 1m37.523s + # real 5m43.954s cat fb.ncbiRefSeq.panPan3.txt - # 74279781 bases of 2999027915 (2.477%) in intersection + # 91843495 bases of 3015350297 (3.046%) in intersection # add: include ../../refSeqComposite.ra alpha - # to the gorilla/panPan3/trackDb.ra to turn on the track in the browser + # to the bonobo/panPan3/trackDb.ra to turn on the track in the browser + # XXX TBD when genbank run is complete featureBits -enrichment panPan3 refGene ncbiRefSeq # refGene 0.006%, ncbiRefSeq 2.477%, both 0.006%, cover 99.87%, enrich 40.32x featureBits -enrichment panPan3 ncbiRefSeq refGene # ncbiRefSeq 2.477%, refGene 0.006%, both 0.006%, cover 0.25%, enrich 40.32x featureBits -enrichment panPan3 ncbiRefSeqCurated refGene # ncbiRefSeqCurated 0.007%, refGene 0.006%, both 0.006%, cover 94.29%, enrich 14956.14x featureBits -enrichment panPan3 refGene ncbiRefSeqCurated # refGene 0.006%, ncbiRefSeqCurated 0.007%, both 0.006%, cover 99.87%, enrich 14956.14x ######################################################################### -# LIFTOVER TO gorGor5 (TBD - 2019-11-20 - Hiram) +# LIFTOVER TO panPan2 (DONE - 2020-06-15 - Hiram) ssh hgwdev - mkdir /hive/data/genomes/panPan3/bed/blat.gorGor5.2019-11-20 - cd /hive/data/genomes/panPan3/bed/blat.gorGor5.2019-11-20 + mkdir /hive/data/genomes/panPan3/bed/blat.panPan2.2020-06-15 + cd /hive/data/genomes/panPan3/bed/blat.panPan2.2020-06-15 doSameSpeciesLiftOver.pl -verbose=2 \ -debug -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ -ooc=/hive/data/genomes/panPan3/jkStuff/panPan3.11.ooc \ - panPan3 gorGor5 + panPan3 panPan2 time (doSameSpeciesLiftOver.pl -verbose=2 \ -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ -ooc=/hive/data/genomes/panPan3/jkStuff/panPan3.11.ooc \ - panPan3 gorGor5) > doLiftOverToGorGor5.log 2>&1 + panPan3 panPan2) > doLiftOverToPanPan2.log 2>&1 +XXX - running - Mon Jun 15 10:25:21 PDT 2020 # real 936m35.524s - # see if the liftOver menus function in the browser from panPan3 to gorGor5 + # see if the liftOver menus function in the browser from panPan3 to panPan2 ######################################################################### -# LIFTOVER TO gorGor4 (TBD - 2019-11-20 - Hiram) +# LIFTOVER TO panPan1 (DONE - 2020-06-15 - Hiram) ssh hgwdev - mkdir /hive/data/genomes/panPan3/bed/blat.gorGor4.2019-11-20 - cd /hive/data/genomes/panPan3/bed/blat.gorGor4.2019-11-20 + mkdir /hive/data/genomes/panPan3/bed/blat.panPan1.2020-06-15 + cd /hive/data/genomes/panPan3/bed/blat.panPan1.2020-06-15 doSameSpeciesLiftOver.pl -verbose=2 \ -debug -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ -ooc=/hive/data/genomes/panPan3/jkStuff/panPan3.11.ooc \ - panPan3 gorGor4 + panPan3 panPan1 time (doSameSpeciesLiftOver.pl -verbose=2 \ -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ -ooc=/hive/data/genomes/panPan3/jkStuff/panPan3.11.ooc \ - panPan3 gorGor4) > doLiftOverToGorGor4.log 2>&1 + panPan3 panPan1) > doLiftOverToPanPan1.log 2>&1 +XXX - running - Mon Jun 15 10:26:22 PDT 2020 # real 654m46.645s - # see if the liftOver menus function in the browser from panPan3 to gorGor4 + # see if the liftOver menus function in the browser from panPan3 to panPan1 ######################################################################### # BLATSERVERS ENTRY (TBD - 2019-11-20 - Hiram) +XXX - requested - Mon Jun 15 10:31:11 PDT 2020 # After getting a blat server assigned by the Blat Server Gods, ssh hgwdev hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \ VALUES ("panPan3", "blat1c", "17914", "1", "0"); \ INSERT INTO blatServers (db, host, port, isTrans, canPcr) \ VALUES ("panPan3", "blat1c", "17915", "0", "1");' \ hgcentraltest # test it with some sequence ############################################################################ ## reset default position similar to gorGor5 found via blat of NR_046473.1 mRNA ## (TBD - 2019-11-20 - Hiram) # as found from the galGal5 to panPan3 liftOver ssh hgwdev hgsql -e 'update dbDb set defaultPos="chr14:81559118-81601404" where name="panPan3";' hgcentraltest ############################################################################## -# crispr whole genome (TBD - 2019-11-20 - Hiram) +# crispr whole genome (DONE - 2020-06-15 - Hiram) mkdir /hive/data/genomes/panPan3/bed/crisprAll cd /hive/data/genomes/panPan3/bed/crisprAll # the large shoulder argument will cause the entire genome to be scanned # this takes a while for a new genome to get the bwa indexing done time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 -stop=ranges \ - panPan3 ncbiRefSeq -shoulder=250000000 -tableName=crisprAll -fileServer=hgwdev \ - -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ + panPan3 ncbiRefSeq -shoulder=250000000 -tableName=crisprAll \ + -fileServer=hgwdev -buildDir=`pwd` -smallClusterHub=hgwdev \ + -bigClusterHub=ku \ -workhorse=hgwdev) > ranges.log 2>&1 +XXX - running - Mon Jun 15 10:32:36 PDT 2020 # real 72m58.740s time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 \ -continue=guides -stop=specScores panPan3 ncbiRefSeq \ -shoulder=250000000 -tableName=crisprAll -fileServer=hgwdev \ -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ -workhorse=hgwdev) > specScores.log 2>&1 # real 8m40.172s cat guides/run.time | sed -e 's/^/# /;' # Completed: 100 of 100 jobs # CPU time in finished jobs: 12309s 205.15m 3.42h 0.14d 0.000 y # IO & Wait Time: 290s 4.83m 0.08h 0.00d 0.000 y # Average job time: 126s 2.10m 0.03h 0.00d # Longest finished job: 380s 6.33m 0.11h 0.00d @@ -973,46 +979,47 @@ # CPU time in finished jobs: 13825593s 230426.55m 3840.44h 160.02d 0.438 y # IO & Wait Time: 172582s 2876.37m 47.94h 2.00d 0.005 y # Average job time: 501s 8.35m 0.14h 0.01d # Longest finished job: 20199s 336.65m 5.61h 0.23d # Submission to last job: 22274s 371.23m 6.19h 0.26d cat offTargets/run.time | sed -e 's/^/# /;' # Completed: 152056 of 152056 jobs # CPU time in finished jobs: 2009038s 33483.97m 558.07h 23.25d 0.064 y # IO & Wait Time: 2321685s 38694.75m 644.91h 26.87d 0.074 y # Average job time: 28s 0.47m 0.01h 0.00d # Longest finished job: 53s 0.88m 0.01h 0.00d # Submission to last job: 4266s 71.10m 1.19h 0.05d ######################################################################### -# all.joiner update, downloads and in pushQ - (WORKING - 2019-11-20 - Hiram) +# all.joiner update, downloads and in pushQ - (WORKING - 2020-06-15 - Hiram) cd $HOME/kent/src/hg/makeDb/schema # verify all the business is done for release ~/kent/src/hg/utils/automation/verifyBrowser.pl panPan3 # fixup all.joiner until this is a clean output joinerCheck -database=panPan3 -tableCoverage all.joiner joinerCheck -database=panPan3 -times all.joiner joinerCheck -database=panPan3 -keys all.joiner # when clean, check in: - git commit -m 'adding rules for panPan3 refs #24524' all.joiner + git commit -m 'adding rules for panPan3 refs #25720' all.joiner git push # run up a 'make alpha' in hg/hgTables to get this all.joiner file # into the hgwdev/genome-test system +XXX - ready - Mon Jun 15 10:35:45 PDT 2020 cd /hive/data/genomes/panPan3 time (~/kent/src/hg/utils/automation/makeDownloads.pl panPan3) > downloads.log 2>&1 # real 17m56.213s # now ready for pushQ entry mkdir /hive/data/genomes/panPan3/pushQ cd /hive/data/genomes/panPan3/pushQ time ($HOME/kent/src/hg/utils/automation/makePushQSql.pl -redmineList panPan3) > panPan3.pushQ.sql 2> stderr.out # real 15m52.548s # remove the tandemDups and gapOverlap from the file list: sed -i -e "/tandemDups/d" redmine.panPan3.table.list sed -i -e "/Tandem Dups/d" redmine.panPan3.releaseLog.txt sed -i -e "/gapOverlap/d" redmine.panPan3.table.list sed -i -e "/Gap Overlaps/d" redmine.panPan3.releaseLog.txt