86ae147ac68707dcb81a697e5e45f4619465097e hiram Tue Feb 16 08:52:47 2021 -0800 completed lastz/chain/net to mm10 mm39 hg38 for rn7 release refs #26682 diff --git src/hg/makeDb/doc/rn7/initialBuild.txt src/hg/makeDb/doc/rn7/initialBuild.txt index 4b51227..c917d54 100644 --- src/hg/makeDb/doc/rn7/initialBuild.txt +++ src/hg/makeDb/doc/rn7/initialBuild.txt @@ -618,545 +618,651 @@ # verify searches work in the position box git commit -m 'adding search rule for gold/assembly track refs #26682' \ trackDb.ra ########################################################################## # running repeat masker (DONE - 2020-02-03 - Hiram) # using new repeat masker version 4.1.0 mkdir /hive/data/genomes/rn7/bed/repeatMasker cd /hive/data/genomes/rn7/bed/repeatMasker time (doRepeatMasker.pl -buildDir=`pwd` \ -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ -smallClusterHub=hgwdev rn7) > do.log 2>&1 XXX - running - Wed Feb 3 16:29:03 PST 2021 - # real 402m37.300s + # real 397m24.001s cat faSize.rmsk.txt -# 2897824427 bases (38007443 N's 2859816984 real 1455718019 upper -# 1404098965 lower) in 964 sequences in 1 files -# Total size: mean 3006041.9 sd 20298858.9 min 285 (chrUn_NW_023264940v1) -# max 217961735 (chr1) median 42174 -# %48.45 masked total, %49.10 masked real +# 2647915728 bases (21334956 N's 2626580772 real 1457026804 upper +# 1169553968 lower) in 176 sequences in 1 files +# Total size: mean 15044975.7 sd 44833491.7 min 746 (chrUn_NW_023637828v1) +# max 260522016 (chr1) median 44754 +# %44.17 masked total, %44.53 masked real egrep -i "versi|relea" do.log # RepeatMasker,v 1.332 2017/04/17 19:01:11 rhubley Exp $ # CC Dfam_Consensus RELEASE 20181026; * # CC RepBase RELEASE 20181026; sed -e 's/^/# /;' versionInfo.txt # The repeat files provided for this assembly were generated using RepeatMasker. # Smit, AFA, Hubley, R & Green, P., # RepeatMasker Open-4.0. # 1996-2010 <http://www.repeatmasker.org>. # # VERSION: # RepeatMasker version development-$Id: RepeatMasker,v 1.332 2017/04/17 19:01:11 rhubley Exp $ # Search Engine: Crossmatch [ 1.090518 ] # Master RepeatMasker Database: /hive/data/staging/data/RepeatMasker181121/Libraries/RepeatMaskerLib.embl ( Complete Database: dc20181026-rb20181026 ) # # # RepeatMasker version development-$Id: RepeatMasker,v 1.332 2017/04/17 19:01:11 rhubley Exp $ # CC Dfam_Consensus RELEASE 20181026; * # CC RepBase RELEASE 20181026; * # # RepeatMasker engine: -engine crossmatch -s -# # RepeatMasker library options: -species 'Callithrix jacchus' +# # RepeatMasker library options: -species 'Rattus norvegicus' # # PARAMETERS: -# /hive/data/staging/data/RepeatMasker/RepeatMasker -engine crossmatch -s -align -species 'Callithrix jacchus' +# /hive/data/staging/data/RepeatMasker/RepeatMasker -engine crossmatch -s -align -species 'Rattus norvegicus' time featureBits -countGaps rn7 rmsk - # 1404097506 bases of 2897824427 (48.454%) in intersection - # real 0m23.891s + # 1169555321 bases of 2647915728 (44.169%) in intersection + # real 0m36.935s # why is it different than the faSize above ? # because rmsk masks out some N's as well as bases, the faSize count above # separates out the N's from the bases, it doesn't show lower case N's # faster way to get the same result on high contig count assemblies: time hgsql -N -e 'select genoName,genoStart,genoEnd from rmsk;' rn7 \ | bedSingleCover.pl stdin | ave -col=4 stdin | grep "^total" - # total 1404097506.000000 - # real 0m21.422s + # total 1169555321.000000 + # real 0m23.489s ########################################################################## # running simple repeat (DONE - 2020-02-03 - Hiram) mkdir /hive/data/genomes/rn7/bed/simpleRepeat cd /hive/data/genomes/rn7/bed/simpleRepeat time (doSimpleRepeat.pl -buildDir=`pwd` -bigClusterHub=ku \ -dbHost=hgwdev -workhorse=hgwdev -smallClusterHub=ku \ -trf409=6 rn7) > do.log 2>&1 & -XXX - running - Wed Feb 3 16:29:44 PST 2021 - # real 128m26.410s + # real 288m35.826s cat fb.simpleRepeat - # 106897704 bases of 2859817025 (3.738%) in intersection + # 94485470 bases of 2626580772 (3.597%) in intersection cd /hive/data/genomes/rn7 # if using the Window Masker result: cd /hive/data/genomes/rn7 # twoBitMask bed/windowMasker/rn7.cleanWMSdust.2bit \ # -add bed/simpleRepeat/trfMask.bed rn7.2bit # you can safely ignore the warning about fields >= 13 # add to rmsk after it is done: twoBitMask rn7.rmsk.2bit \ -add bed/simpleRepeat/trfMask.bed rn7.2bit # you can safely ignore the warning about fields >= 13 twoBitToFa rn7.2bit stdout | faSize stdin > faSize.rn7.2bit.txt cat faSize.rn7.2bit.txt -# 2897824427 bases (38007443 N's 2859816984 real 1454005411 upper -# 1405811573 lower) in 964 sequences in 1 files -# Total size: mean 3006041.9 sd 20298858.9 min 285 (chrUn_NW_023264940v1) -# max 217961735 (chr1) median 42174 -# %48.51 masked total, %49.16 masked real - +# 2647915728 bases (21334956 N's 2626580772 real 1452560391 upper +# 1174020381 lower) in 176 sequences in 1 files +# Total size: mean 15044975.7 sd 44833491.7 min 746 (chrUn_NW_023637828v1) +# max 260522016 (chr1) median 44754 +# %44.34 masked total, %44.70 masked real + # reset symlink rm /gbdb/rn7/rn7.2bit ln -s `pwd`/rn7.2bit /gbdb/rn7/rn7.2bit ######################################################################### -# CREATE MICROSAT TRACK (TBD - 2020-09-02 - Hiram) +# CREATE MICROSAT TRACK (DONE - 2021-02-03 - Hiram) ssh hgwdev mkdir /hive/data/genomes/rn7/bed/microsat cd /hive/data/genomes/rn7/bed/microsat awk '($5==2 || $5==3) && $6 >= 15 && $8 == 100 && $9 == 0 {printf("%s\t%s\t%s\t%dx%s\n", $1, $2, $3, $6, $16);}' \ ../simpleRepeat/simpleRepeat.bed > microsat.bed hgLoadBed rn7 microsat microsat.bed - # Read 41788 elements of size 4 from microsat.bed + # Read 244331 elements of size 4 from microsat.bed ########################################################################## -## WINDOWMASKER (TBD - 2020-09-02 - Hiram) +## WINDOWMASKER (DONE - 2021-02-03 - Hiram) mkdir /hive/data/genomes/rn7/bed/windowMasker cd /hive/data/genomes/rn7/bed/windowMasker time (doWindowMasker.pl -buildDir=`pwd` -workhorse=hgwdev \ -dbHost=hgwdev rn7) > do.log 2>&1 - # real 119m59.049s + # real 103m59.019s # Masking statistics cat faSize.rn7.cleanWMSdust.txt -# 2897824427 bases (38007443 N's 2859816984 real 1756905691 upper 1102911293 lower) in 964 sequences in 1 files -# Total size: mean 3006041.9 sd 20298858.9 min 285 (chrUn_NW_023264940v1) max 217961735 (chr1) median 42174 -# %38.06 masked total, %38.57 masked real +# 2647915728 bases (21334956 N's 2626580772 real 1677807978 upper +# 948772794 lower) in 176 sequences in 1 files +# Total size: mean 15044975.7 sd 44833491.7 min 746 (chrUn_NW_023637828v1) +# max 260522016 (chr1) median 44754 +# %35.83 masked total, %36.12 masked real ########################################################################## -# cpgIslands - (TBD - 2020-09-03 - Hiram) +# cpgIslands - (DONE - 2021-02-04 - Hiram) mkdir /hive/data/genomes/rn7/bed/cpgIslands cd /hive/data/genomes/rn7/bed/cpgIslands time (doCpgIslands.pl -dbHost=hgwdev -bigClusterHub=ku \ -workhorse=hgwdev -smallClusterHub=ku rn7) > do.log 2>&1 - # real 3m28.053s + # real 3m40.227s - cat fb.rn7.cpgIslandExt.txt - # 10571422 bases of 2654624157 (0.398%) in intersection + sed -e 's/^/ # /;' fb.rn7.cpgIslandExt.txt + # 10397605 bases of 2626580772 (0.396%) in intersection ############################################################################## -# genscan - (TBD - 2020-09-03 - Hiram) +# genscan - (DONE - 2021-02-04 - 02-15 - Hiram) mkdir /hive/data/genomes/rn7/bed/genscan cd /hive/data/genomes/rn7/bed/genscan time (doGenscan.pl -buildDir=`pwd` -workhorse=hgwdev -dbHost=hgwdev \ -bigClusterHub=ku rn7) > do.log 2>&1 # real 73m39.179s # two jobs broken: -./runGsBig2M.csh chr22 000 gtf/000/chr22.gtf pep/000/chr22.pep subopt/000/chr22.bed & -./runGsBig2M.csh chr4 000 gtf/000/chr4.gtf pep/000/chr4.pep subopt/000/chr4.bed +./runGsBig2M.csh chr2 000 gtf/000/chr2.gtf pep/000/chr2.pep subopt/000/chr2.bed & +./runGsBig2M.csh chr11 000 gtf/000/chr11.gtf pep/000/chr11.pep subopt/000/chr11.bed wait - # real 56m30.462s + # real 68m25.959s # continuing time (doGenscan.pl -buildDir=`pwd` -workhorse=hgwdev -dbHost=hgwdev \ -continue=makeBed -bigClusterHub=ku rn7) > makeBed.log 2>&1 - # real 1m1.807s + # real 0m43.231s - cat fb.rn7.genscan.txt - # 56416049 bases of 2859817025 (1.973%) in intersection + sed -e 's/^/ # /;' fb.rn7.genscan.txt + # 55066515 bases of 2626580772 (2.097%) in intersection - cat fb.rn7.genscanSubopt.txt - # 54809849 bases of 2859817025 (1.917%) in intersection + sed -e 's/^/ # /;' fb.rn7.genscanSubopt.txt + # 57355860 bases of 2626580772 (2.184%) in intersection ######################################################################### # ncbiGene (TBD - 2020-09-03 - Hiram) # don't need to do this on GCF/RefSeq assemblies, they have RefSeq genes mkdir /hive/data/genomes/rn7/bed/xenoRefGene cd /hive/data/genomes/rn7/bed/xenoRefGene time (~/kent/src/hg/utils/automation/doXenoRefGene.pl -buildDir=`pwd` \ -bigClusterHub=ku -workhorse=hgwdev -dbHost=hgwdev rn7) > do.log 2>&1 & # real 67m18.015s ######################################################################### -# Create kluster run files (TBD - 2020-09-03 - Hiram) +# Create kluster run files (DONE - 2021-02-04 - Hiram) # numerator is rn7 gapless bases "real" as reported by: featureBits -noRandom -noHap rn7 gap - # 36757259 bases of 2793460667 (1.316%) in intersection + # 19396258 bases of 2614093470 (0.742%) in intersection # ^^^ # denominator is hg19 gapless bases as reported by: # featureBits -noRandom -noHap hg19 gap # 234344806 bases of 2861349177 (8.190%) in intersection # 1024 is threshold used for human -repMatch: - calc \( 2793460667 / 2861349177 \) \* 1024 - # ( 2793460667 / 2861349177 ) * 1024 = 999.704526 + calc \( 2614093470 / 2861349177 \) \* 1024 + # ( 2614093470 / 2861349177 ) * 1024 = 935.513825 # ==> use -repMatch=1000 according to size scaled down from 1024 for human. cd /hive/data/genomes/rn7 time blat rn7.2bit \ /dev/null /dev/null -tileSize=11 -makeOoc=jkStuff/rn7.11.ooc \ -repMatch=1000 - # Wrote 34987 overused 11-mers to jkStuff/rn7.11.ooc - # real 0m29.438s - # calJac3 at repMatch=950 - # Wrote 32908 overused 11-mers to jkStuff/calJac3.11.ooc + # Wrote 27322 overused 11-mers to jkStuff/rn7.11.ooc + # real 0m26.195s + + # rn6 at repMatch=1000 + # Wrote 27021 overused 11-mers to jkStuff/rn6.11.ooc # survey sizes of all gaps: hgsql -N -e 'select size from gap;' rn7 | ave stdin | sed -e 's/^/# /;' # Q1 100.000000 -# median 24230.500000 -# Q3 55834.000000 -# average 100548.682540 -# min 13.000000 -# max 1000000.000000 -# count 378 -# total 38007402.000000 -# standard deviation 237103.349286 +# median 200.000000 +# Q3 22550.000000 +# average 36721.094664 +# min 10.000000 +# max 660122.000000 +# count 581 +# total 21334956.000000 +# standard deviation 95181.017236 + hgsql -N -e 'select size from gap;' rn7 | sort -n | uniq -c | sed -e 's/^/# /;' + # There are no non-bridged gaps on this genome # survey sizes of non-bridged gaps: hgsql -N -e 'select size from gap where bridge="no" order by size;' \ rn7 | sort | uniq -c | sort -k2,2n | sed -e 's/^/# /;' # 7 100 # 8 50000 # 22 1000000 # and survey the number bridged gaps over 50,000 bases: hgsql -N -e 'select size from gap where bridge="yes" and size > 49999;' \ rn7 | wc -l - # 86 + # 92 # forget the non-bridged of size 100, use 50,000 and allow bridged # use gap size of 50000 to construct a lift file: gapToLift -allowBridged -verbose=2 -minGap=50000 rn7 \ jkStuff/rn7.gaps.lft -bedFile=jkStuff/rn7.gaps.bed - wc -l jkStuff/rn7.gaps*. - # 1073 jkStuff/rn7.gaps.bed - # 1073 jkStuff/rn7.gaps.lft + wc -l jkStuff/rn7.gaps* + # 268 jkStuff/rn7.gaps.bed + # 268 jkStuff/rn7.gaps.lft # to see the gaps sizes used: bedInvert.pl chrom.sizes jkStuff/rn7.gaps.bed \ | cut -f4 | sort -n | uniq -c | less -######################################################################## -# lastz/chain/net swap human/hg38 (TBD - 2020-04-10 - Hiram) +############################################################################## +# lastz/chain/net swap human/hg38 (DONE - 2021-02-04 - Hiram) # original alignment - cd /hive/data/genomes/hg38/bed/lastzRn7.2020-08-03 + cd /hive/data/genomes/hg38/bed/lastzRn7.2021-02-04 + + sed -e 's/^/ # /;' fb.hg38.chainRn7Link.txt + # 958592205 bases of 3110768607 (30.815%) in intersection + sed -e 's/^/ # /;' fb.hg38.chainSynRn7Link.txt + # 904066852 bases of 3110768607 (29.062%) in intersection - cat fb.hg38.chainRn7Link.txt - # 2338614467 bases of 3110768607 (75.178%) in intersection - cat fb.hg38.chainSynRn7Link.txt - # 2261509186 bases of 3110768607 (72.699%) in intersection - cat fb.hg38.chainRBest.Rn7.txt - # 2146650109 bases of 3110768607 (69.007%) in intersection + time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ + hg38 rn7) > rbest.log 2>&1 & + # real 313m26.149s + + sed -e 's/^/ # /;' fb.hg38.chainRBest.Rn7.txt + # 883775977 bases of 3110768607 (28.410%) in intersection - # and for the swap: + # and the swap mkdir /hive/data/genomes/rn7/bed/blastz.hg38.swap cd /hive/data/genomes/rn7/bed/blastz.hg38.swap - time (doBlastzChainNet.pl -verbose=2 \ - /hive/data/genomes/hg38/bed/lastzRn7.2020-08-03/DEF \ - -swap -chainMinScore=3000 -chainLinearGap=medium \ - -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ - -syntenicNet) > swap.log 2>&1 - # real 105m30.938s + /hive/data/genomes/hg38/bed/lastzRn7.2021-02-04/DEF \ + -chainMinScore=3000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ + -swap -syntenicNet) > swap.log 2>&1 + # real 74m20.215s - cat fb.rn7.chainHg38Link.txt - # 2256668215 bases of 2859817025 (78.910%) in intersection - cat fb.rn7.chainSynHg38Link.txt - # 2183912419 bases of 2859817025 (76.365%) in intersection + sed -e 's/^/ # /;' fb.rn7.chainHg38Link.txt + # 928866703 bases of 2626580772 (35.364%) in intersection + + sed -e 's/^/ # /;' fb.rn7.chainSynHg38Link.txt + # 879484562 bases of 2626580772 (33.484%) in intersection time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ rn7 hg38) > rbest.log 2>&1 & - # real 467m22.590s + # real 373m6.610s - cat fb.rn7.chainRBest.Hg38.txt - # 2147250913 bases of 2859817025 (75.084%) in intersection + sed -e 's/^/ # /;' fb.rn7.chainRBest.Hg38.txt + # 885516265 bases of 2626580772 (33.714%) in intersection -############################################################################ -# lastz/chain/net swap mouse/mm10 (TBD - 2020-08-03 - Hiram) +############################################################################## +# lastz/chain/net swap mouse/mm39 (DONE - 2021-02-04 - Hiram) # original alignment - cd /hive/data/genomes/mm10/bed/lastzRn7.2020-08-03 + cd /hive/data/genomes/mm39/bed/lastzRn7.2021-02-04 - cat fb.mm10.chainRn7Link.txt - # 877278264 bases of 2652783500 (33.070%) in intersection - cat fb.mm10.chainSynRn7Link.txt - # 830868888 bases of 2652783500 (31.321%) in intersection - cat fb.mm10.chainRBest.Rn7.txt - # 835445771 bases of 2652783500 (31.493%) in intersection + sed -e 's/^/ # /;' fb.mm39.chainRn7Link.txt + # 1898735724 bases of 2654624157 (71.526%) in intersection + sed -e 's/^/ # /;' fb.mm39.chainSynRn7Link.txt + # 1787593557 bases of 2654624157 (67.339%) in intersection + + time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ + mm39 rn7) > rbest.log 2>&1 & + # real 588m38.122s + + sed -e 's/^/ # /;' fb.mm39.chainRBest.Rn7.txt + # 1754204799 bases of 2654624157 (66.081%) in intersection + + mkdir /hive/data/genomes/rn7/bed/blastz.mm39.swap + cd /hive/data/genomes/rn7/bed/blastz.mm39.swap + time (doBlastzChainNet.pl -verbose=2 \ + /hive/data/genomes/mm39/bed/lastzRn7.2021-02-04/DEF \ + -swap -syntenicNet \ + -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ + -chainMinScore=5000 -chainLinearGap=medium) > swap.log 2>&1 + # real 116m47.862s + + sed -e 's/^/ # /;' fb.rn7.chainMm39Link.txt + # 1855165978 bases of 2626580772 (70.630%) in intersection + sed -e 's/^/ # /;' fb.rn7.chainSynMm39Link.txt + # 1763550257 bases of 2626580772 (67.142%) in intersection + + time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ + rn7 mm39) > rbest.log 2>&1 + # real 605m42.354s + + sed -e 's/^/ # /;' fb.rn7.chainRBest.Mm39.txt + # 1754416686 bases of 2626580772 (66.795%) in intersection + +############################################################################## +# lastz/chain/net swap mouse/mm10 (DONE - 2021-02-15 - Hiram) + + # original alignment + cd /hive/data/genomes/mm10/bed/lastzRn7.2021-02-15 + + sed -e 's/^/ # /;' fb.mm10.chainRn7Link.txt + # 1896928045 bases of 2652783500 (71.507%) in intersection + sed -e 's/^/ # /;' fb.mm10.chainSynRn7Link.txt + # 1787142074 bases of 2652783500 (67.369%) in intersection + + time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ + mm10 rn7) > rbest.log 2>&1 & + # real 578m13.711s + + sed -e 's/^/ # /;' fb.mm10.chainRBest.Rn7.txt + # 1753198266 bases of 2652783500 (66.089%) in intersection mkdir /hive/data/genomes/rn7/bed/blastz.mm10.swap cd /hive/data/genomes/rn7/bed/blastz.mm10.swap - time (doBlastzChainNet.pl -verbose=2 \ - /hive/data/genomes/mm10/bed/lastzRn7.2020-08-03/DEF \ + /hive/data/genomes/mm10/bed/lastzRn7.2021-02-15/DEF \ -swap -syntenicNet \ -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ - -chainMinScore=3000 -chainLinearGap=medium) > swap.log 2>&1 & - # real 51m16.400s + -chainMinScore=5000 -chainLinearGap=medium) > swap.log 2>&1 + # real 112m36.899s - cat fb.rn7.chainMm10Link.txt - # 882506277 bases of 2859817025 (30.859%) in intersection - cat fb.rn7.chainSynMm10Link.txt - # 831171319 bases of 2859817025 (29.064%) in intersection + sed -e 's/^/ # /;' fb.rn7.chainMm10Link.txt + # 1853300495 bases of 2626580772 (70.559%) in intersection + sed -e 's/^/ # /;' fb.rn7.chainSynMm10Link.txt + # 1762899567 bases of 2626580772 (67.118%) in intersection - time (doRecipBest.pl -load -workhorse=hgwdev rn7 mm10 \ - -buildDir=`pwd` -workhorse=hgwdev) > rbest.log 2>&1 & - # real 308m53.845s + time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ + rn7 mm10) > rbest.log 2>&1 + # real 599m24.766s - cat fb.rn7.chainRBest.Mm10.txt - # 833789913 bases of 2859817025 (29.155%) in intersection + sed -e 's/^/ # /;' fb.rn7.chainRBest.Mm10.txt + # 1753558422 bases of 2626580772 (66.762%) in intersection ############################################################################## -# GENBANK AUTO UPDATE (TBD - 2020-09-03 - Hiram) +# GENBANK AUTO UPDATE (DONE - 2021-02-04 - Hiram) ssh hgwdev cd $HOME/kent/src/hg/makeDb/genbank git pull # /cluster/data/genbank/data/organism.lst shows: # organism mrnaCnt estCnt refSeqCnt - # Callithrix geoffroyi 7 0 0 - # Callithrix jacchus 1607 292992 221 - # Callithrix kuhlii 1 0 0 - # Callithrix penicillata 1 0 0 - # Callithrix pygmaea 5 0 0 - # Callithrix sp. 4 0 0 + # Rattus norvegicus 130246 1103640 17794 - # edit etc/genbank.conf to add rn7 just after mm10 + # edit etc/genbank.conf to add rn7 just before rn6 -# Marmoset - GCF_009663435.1 - Callithrix_jacchus_cj1700BN7.2 +# rn7 (rat GCF_015227675.2 mRatBN7.2) rn7.serverGenome = /hive/data/genomes/rn7/rn7.2bit rn7.ooc = /hive/data/genomes/rn7/jkStuff/rn7.11.ooc rn7.lift = /hive/data/genomes/rn7/jkStuff/rn7.gaps.lft -rn7.perChromTables = no rn7.refseq.mrna.native.pslCDnaFilter = ${finished.refseq.mrna.native.pslCDnaFilter} rn7.refseq.mrna.xeno.pslCDnaFilter = ${finished.refseq.mrna.xeno.pslCDnaFilter} rn7.genbank.mrna.native.pslCDnaFilter = ${finished.genbank.mrna.native.pslCDnaFilter} rn7.genbank.mrna.xeno.pslCDnaFilter = ${finished.genbank.mrna.xeno.pslCDnaFilter} rn7.genbank.est.native.pslCDnaFilter = ${finished.genbank.est.native.pslCDnaFilter} -rn7.genbank.est.xeno.pslCDnaFilter = ${finished.genbank.est.xeno.pslCDnaFilter} rn7.downloadDir = rn7 -rn7.refseq.mrna.native.load = yes rn7.refseq.mrna.xeno.load = yes rn7.refseq.mrna.xeno.loadDesc = yes rn7.genbank.mrna.xeno.load = yes +rn7.perChromTables = no +rn7.mgc = yes # rn7.upstreamGeneTbl = ensGene -# rn7.upstreamMaf = multiz13way /hive/data/genomes/rn7/bed/multiz13way/species.list +# rn7.upstreamMaf = multiz13way /hive/data/genomes/rn7/bed/multiz13way/species.list.txt # verify the files specified exist before checking in the file: grep ^rn7 etc/genbank.conf | grep hive | awk '{print $NF}' | xargs ls -og -# -rw-rw-r-- 1 755819249 Sep 3 10:51 /hive/data/genomes/rn7/rn7.2bit -# -rw-rw-r-- 1 139956 Sep 3 11:54 /hive/data/genomes/rn7/jkStuff/rn7.11.ooc -# -rw-rw-r-- 1 61169 Sep 3 12:42 /hive/data/genomes/rn7/jkStuff/rn7.gaps.lft +# -rw-rw-r-- 1 109296 Feb 4 09:28 /hive/data/genomes/rn7/jkStuff/rn7.11.ooc +# -rw-rw-r-- 1 13791 Feb 4 09:34 /hive/data/genomes/rn7/jkStuff/rn7.gaps.lft +# -rw-rw-r-- 1 692151553 Feb 4 09:20 /hive/data/genomes/rn7/rn7.2bit - git commit -m "Added rn7 rat; refs #26682" etc/genbank.conf + git commit -m "Added rn7 rat refs #26682" etc/genbank.conf git push # update /cluster/data/genbank/: make etc-update # enable daily alignment and update of hgwdev cd ~/kent/src/hg/makeDb/genbank git pull # add rn7 to: # etc/hgwdev.dbs etc/align.dbs git commit -m "Added rn7 - rat refs #26682" etc/hgwdev.dbs etc/align.dbs git push make etc-update # wait a few days for genbank magic to take place, the tracks will # appear ############################################################################# -# augustus gene track (TBD - 2020-09-04 - Hiram) +# augustus gene track (DONE - 2021-02-04 - Hiram) mkdir /hive/data/genomes/rn7/bed/augustus cd /hive/data/genomes/rn7/bed/augustus time (doAugustus.pl -buildDir=`pwd` -bigClusterHub=ku \ -species=human -dbHost=hgwdev \ -workhorse=hgwdev rn7) > do.log 2>&1 - # real 119m49.652s + # real 119m10.900s cat fb.rn7.augustusGene.txt - # 56033149 bases of 2859817025 (1.959%) in intersection + # 49646030 bases of 2626580772 (1.890%) in intersection ######################################################################### -# ncbiRefSeq (TBD - 2020-09-17 - Hiram) +# ncbiRefSeq (DONE - 2021-02-04 - Hiram) mkdir /hive/data/genomes/rn7/bed/ncbiRefSeq cd /hive/data/genomes/rn7/bed/ncbiRefSeq time (~/kent/src/hg/utils/automation/doNcbiRefSeq.pl -buildDir=`pwd` \ -bigClusterHub=ku -dbHost=hgwdev \ -fileServer=hgwdev -smallClusterHub=hgwdev -workhorse=hgwdev \ - GCF_009663435.1_Callithrix_jacchus_cj1700BN7.2 rn7) > do.log 2>&1 & + GCF_015227675.2_mRatBN7.2 rn7) > do.log 2>&1 & + # real 6m52.120s cat fb.ncbiRefSeq.rn7.txt - # 102754440 bases of 2859817025 (3.593%) in intersection + # 107770866 bases of 2626580772 (4.103%) in intersection # add: include ../../refSeqComposite.ra # to the rat/rn7/trackDb.ra to turn on the track in the browser - # XXX 2020-09-17 - ready for this after genbank runs +joinerCheck says: + + rn7.ncbiRefSeqLink.protAcc - hits 74754 of 74755 (99.999%) +Error: 1 of 74755 elements (0.001%) of rn7.ncbiRefSeqLink.protAcc are not in key ncbiRefSeqPepTable.name line 8640 of all.joiner +Example miss: NP_536324.1 + + + # XXX 2021-02-04 - ready for this after genbank runs featureBits -enrichment rn7 refGene ncbiRefSeq # refGene 0.402%, ncbiRefSeq 3.148%, both 0.402%, cover 99.90%, enrich 31.73x featureBits -enrichment rn7 ncbiRefSeq refGene # ncbiRefSeq 3.148%, refGene 0.402%, both 0.402%, cover 12.76%, enrich 31.73x featureBits -enrichment rn7 ncbiRefSeqCurated refGene # ncbiRefSeqCurated 0.401%, refGene 0.402%, both 0.400%, cover 99.66%, enrich 247.79x featureBits -enrichment rn7 refGene ncbiRefSeqCurated # refGene 0.402%, ncbiRefSeqCurated 0.401%, both 0.400%, cover 99.33%, enrich 247.79x ############################################################################## -# LIFTOVER TO calJac3 (TBD - 2020-09-04 - Hiram) +# LIFTOVER TO rn6 (DONE - 2021-02-04 - Hiram) ssh hgwdev - mkdir /hive/data/genomes/rn7/bed/blat.calJac3.2020-09-04 - cd /hive/data/genomes/rn7/bed/blat.calJac3.2020-09-04 + mkdir /hive/data/genomes/rn7/bed/blat.rn6.2021-02-04 + cd /hive/data/genomes/rn7/bed/blat.rn6.2021-02-04 doSameSpeciesLiftOver.pl -verbose=2 \ -debug -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ - -query2Bit=/hive/data/genomes/calJac3/calJac3.2bit \ - -querySizes=/hive/data/genomes/calJac3/chrom.sizes \ + -query2Bit=/hive/data/genomes/rn6/rn6.2bit \ + -querySizes=/hive/data/genomes/rn6/chrom.sizes \ -ooc=/hive/data/genomes/rn7/jkStuff/rn7.11.ooc \ - rn7 calJac3 + rn7 rn6 time (doSameSpeciesLiftOver.pl -verbose=2 \ -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ - -query2Bit=/hive/data/genomes/calJac3/calJac3.2bit \ - -querySizes=/hive/data/genomes/calJac3/chrom.sizes \ + -query2Bit=/hive/data/genomes/rn6/rn6.2bit \ + -querySizes=/hive/data/genomes/rn6/chrom.sizes \ -ooc=/hive/data/genomes/rn7/jkStuff/rn7.11.ooc \ - rn7 calJac3) > doLiftOverToRn7.log 2>&1 + rn7 rn6) > doLiftOverToRn7.log 2>&1 # real 248m51.413s - # see if the liftOver menus function in the browser from rn7 to calJac3 + # see if the liftOver menus function in the browser from rn7 to rn6 ############################################################################## -# BLATSERVERS ENTRY (TBD - 2020-09-04 - Hiram) +# BLATSERVERS ENTRY (DONE - 2021-02-04 - Hiram) # After getting a blat server assigned by the Blat Server Gods, ssh hgwdev hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \ - VALUES ("rn7", "blat1a", "17902", "1", "0"); \ + VALUES ("rn7", "blat1b", "17910", "1", "0"); \ INSERT INTO blatServers (db, host, port, isTrans, canPcr) \ - VALUES ("rn7", "blat1a", "17903", "0", "1");' \ + VALUES ("rn7", "blat1b", "17911", "0", "1");' \ hgcentraltest # test it with some sequence -############################################################################ -## reset default position to same as calJac3 via blat of the protein from there -## (TBD - 2020-09-17 - Hiram) +############################################################################## +## reset default position to same as rn6 via blat of the DNA from rn6 +## (DONE - 2021-02-04 - Hiram) ssh hgwdev - hgsql -e 'update dbDb set defaultPos="chr15:67061035-67077513" + hgsql -e 'update dbDb set defaultPos="chr1:79348972-79379997" where name="rn7";' hgcentraltest ############################################################################## # crispr whole genome (TBD - 2020-09-04 -> 2020-09-10 - Hiram) mkdir /hive/data/genomes/rn7/bed/crisprAll cd /hive/data/genomes/rn7/bed/crisprAll # need to have augustus genes done. This will not work with genscan # the large shoulder argument will cause the entire genome to be scanned # this takes a while for a new genome to get the bwa indexing done time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 -stop=ranges \ rn7 augustusGene -shoulder=250000000 -tableName=crisprAll \ -fileServer=hgwdev \ -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ -workhorse=hgwdev) >> ranges.log 2>&1 - # real 73m51.391s + # real 64m40.351s time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 \ -continue=guides -stop=load rn7 augustusGene \ -shoulder=250000000 -tableName=crisprAll -fileServer=hgwdev \ -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ -workhorse=hgwdev) > specScores.log 2>&1 # real 7968m4.344s + # broken specScores step due to power failure ku reset + # first part of specScores +# Completed: 925541 of 3066093 jobs +# Crashed: 503 jobs +# Other count: 1999108 jobs +# CPU time in finished jobs: 84291533s 1404858.88m 23414.31h 975.60d 2.673 y +# IO & Wait Time: 2560453s 42674.22m 711.24h 29.63d 0.081 y +# Average job time: 94s 1.56m 0.03h 0.00d +# Longest finished job: 709s 11.82m 0.20h 0.01d +# Submission to last job: 87660s 1461.00m 24.35h 1.01d + # second part of specScores +# Completed: 2140552 of 2140552 jobs +# CPU time in finished jobs: 190374829s 3172913.81m 52881.90h 2203.41d 6.037 y +# IO & Wait Time: 4108443s 68474.05m 1141.23h 47.55d 0.130 y +# Average job time: 91s 1.51m 0.03h 0.00d +# Longest finished job: 193s 3.22m 0.05h 0.00d +# Submission to last job: 208681s 3478.02m 57.97h 2.42d + + # and putting together the results + time find tmp/outGuides -type f | xargs cut -f3-6 > ../specScores.tab + + # real 806m23.857s + # user 4m0.906s + # sys 13m39.775s + + # continuing with effScores: + time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 \ + -continue=effScores -stop=load rn7 augustusGene \ + -shoulder=250000000 -tableName=crisprAll -fileServer=hgwdev \ + -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ + -workhorse=hgwdev) > effScores.log 2>&1 + # real 1964m18.261s + + cat guides/run.time | sed -e 's/^/# /;' # Completed: 100 of 100 jobs -# CPU time in finished jobs: 12157s 202.62m 3.38h 0.14d 0.000 y -# IO & Wait Time: 252s 4.20m 0.07h 0.00d 0.000 y -# Average job time: 124s 2.07m 0.03h 0.00d -# Longest finished job: 505s 8.42m 0.14h 0.01d -# Submission to last job: 517s 8.62m 0.14h 0.01d +# CPU time in finished jobs: 13041s 217.34m 3.62h 0.15d 0.000 y +# IO & Wait Time: 340s 5.67m 0.09h 0.00d 0.000 y +# Average job time: 134s 2.23m 0.04h 0.00d +# Longest finished job: 512s 8.53m 0.14h 0.01d +# Submission to last job: 513s 8.55m 0.14h 0.01d cat specScores/run.time | sed -e 's/^/# /;' -# Completed: 3085907 of 3085907 jobs -# CPU time in finished jobs: 294759806s 4912663.43m 81877.72h 3411.57d 9.347 y -# IO & Wait Time: 6938403s 115640.05m 1927.33h 80.31d 0.220 y -# Average job time: 98s 1.63m 0.03h 0.00d -# Longest finished job: 372s 6.20m 0.10h 0.00d -# Submission to last job: 324356s 5405.93m 90.10h 3.75d +# Completed: 2140552 of 2140552 jobs +# CPU time in finished jobs: 190374829s 3172913.81m 52881.90h 2203.41d 6.037 y +# IO & Wait Time: 4108443s 68474.05m 1141.23h 47.55d 0.130 y +# Average job time: 91s 1.51m 0.03h 0.00d +# Longest finished job: 193s 3.22m 0.05h 0.00d +# Submission to last job: 208681s 3478.02m 57.97h 2.42d grep "Number of specScores" specScores.log -# Number of specScores: 230996937 +# Number of specScores: 229203141 cat effScores/run.time | sed -e 's/^/# /;' -# Completed: 28837 of 28837 jobs -# CPU time in finished jobs: 14627399s 243789.98m 4063.17h 169.30d 0.464 y -# IO & Wait Time: 527816s 8796.94m 146.62h 6.11d 0.017 y -# Average job time: 526s 8.76m 0.15h 0.01d -# Longest finished job: 1466s 24.43m 0.41h 0.02d -# Submission to last job: 32001s 533.35m 8.89h 0.37d +# Completed: 27591 of 27591 jobs +# CPU time in finished jobs: 13780234s 229670.56m 3827.84h 159.49d 0.437 y +# IO & Wait Time: 93812s 1563.54m 26.06h 1.09d 0.003 y +# Average job time: 503s 8.38m 0.14h 0.01d +# Longest finished job: 52764s 879.40m 14.66h 0.61d +# Submission to last job: 70493s 1174.88m 19.58h 0.82d cat offTargets/run.time | sed -e 's/^/# /;' -# Completed: 154296 of 154296 jobs -# CPU time in finished jobs: 2640432s 44007.20m 733.45h 30.56d 0.084 y -# IO & Wait Time: 986281s 16438.02m 273.97h 11.42d 0.031 y -# Average job time: 24s 0.39m 0.01h 0.00d -# Longest finished job: 49s 0.82m 0.01h 0.00d -# Submission to last job: 12302s 205.03m 3.42h 0.14d +# Completed: 153305 of 153305 jobs +# CPU time in finished jobs: 2423705s 40395.08m 673.25h 28.05d 0.077 y +# IO & Wait Time: 1680214s 28003.57m 466.73h 19.45d 0.053 y +# Average job time: 27s 0.45m 0.01h 0.00d +# Longest finished job: 60s 1.00m 0.02h 0.00d +# Submission to last job: 5880s 98.00m 1.63h 0.07d bigBedInfo crispr.bb | sed -e 's/^/# /;' # version: 4 # fieldCount: 22 # hasHeaderExtension: yes # isCompressed: yes # isSwapped: 0 # extraIndexCount: 0 -# itemCount: 287,101,022 -# primaryDataSize: 12,714,904,685 -# primaryIndexSize: 18,017,884 +# itemCount: 274,741,534 +# primaryDataSize: 12,314,717,925 +# primaryIndexSize: 17,246,852 # zoomLevels: 10 -# chromCount: 34 -# basesCovered: 2,272,744,456 -# meanDepth (of bases covered): 2.905440 +# chromCount: 31 +# basesCovered: 2,157,342,037 +# meanDepth (of bases covered): 2.929093 # minDepth: 1.000000 -# maxDepth: 29.000000 -# std of depth: 1.956870 +# maxDepth: 33.000000 +# std of depth: 1.944611 ######################################################################### # all.joiner update, downloads and in pushQ - (WORKING - 2019-11-20 - Hiram) + # had incorrect orderKey specified in beginning rn6.config.ra + # correct to 18019 by looking at the output of: +# hgsql -e 'select name,organism,orderKey from dbDb order by orderKey;' \ +# hgcentraltest | less +# oryCun1 Rabbit 18010 +# regenRn1 Rat 18020 +# regenRn0 Rat 18021 +# rn6 Rat 18031 +# rn7 Rat 18032 +# rn5 Rat 18032 +# rn4 Rat 18033 +# rn3 Rat 18034 +# rn2 Rat 18035 +# tauEry1 Red crested turaco 18360 + + hgsql -e 'update dbDb set orderKey=18019 where name="rn7";' hgcentraltest + + cd $HOME/kent/src/hg/makeDb/schema # verify all the business is done for release ~/kent/src/hg/utils/automation/verifyBrowser.pl rn7 # 66 tables in database rn7 - Dog, Canis lupus familiaris # verified 55 tables in database rn7, 11 extra tables, 14 optional tables # chainNetRBestHg38 3 optional tables # chainNetRBestMm10 3 optional tables # chainNetSynHg38 3 optional tables # chainNetSynMm10 3 optional tables # gapOverlap 1 optional tables # tandemDups 1 optional tables # 1 chainCanFam3 - extra table # 2 chainCanFam3Link - extra table # 3 chainRBestCanFam3 - extra table # 4 chainRBestCanFam3Link - extra table