e8368a2af6ff218c1ead44cd1fe83da7127ad7a1 hiram Wed Mar 31 14:26:31 2021 -0700 genbank done browser complete in pushQ refs #24693 diff --git src/hg/makeDb/doc/xenTro10/initialBuild.txt src/hg/makeDb/doc/xenTro10/initialBuild.txt index 0fd8f75..ed7590c 100644 --- src/hg/makeDb/doc/xenTro10/initialBuild.txt +++ src/hg/makeDb/doc/xenTro10/initialBuild.txt @@ -1,23 +1,23 @@ # for emacs: -*- mode: sh; -*- # This file describes browser build for the xenTro10 # GCF_000004195.4 UCB_Xtro_10.0 # Can use existing photograph (otherwise find one before starting here) -GCF_000004195.4_UCB_Xtro_10.0 +# GCF_000004195.4_UCB_Xtro_10.0 ######################################################################### # Initial steps, reuse existing photograph (DONE - 2021-02-22 - Hiram) # To start this initialBuild.txt document, from a previous assembly document: mkdir ~/kent/src/hg/makeDb/doc/xenTro10 cd ~/kent/src/hg/makeDb/doc/xenTro10 sed -e 's/rn7/xenTro10/g; s/Rn7/XenTro10/g; s/DONE/TBD/g;' \ ../rn7/initialBuild.txt > initialBuild.txt mkdir -p /hive/data/genomes/xenTro10/refseq cd /hive/data/genomes/xenTro10 @@ -438,103 +438,96 @@ # scaffolds are not gaps between 'scaffolds', they are gaps that have # known distances due to paired end matching, this is typically what # a 'contig' gap would be. ############################################################################## # cpgIslands on UNMASKED sequence (DONE - 2021-02-22 - Hiram) mkdir /hive/data/genomes/xenTro10/bed/cpgIslandsUnmasked cd /hive/data/genomes/xenTro10/bed/cpgIslandsUnmasked time (doCpgIslands.pl -dbHost=hgwdev -bigClusterHub=ku -buildDir=`pwd` \ -tableName=cpgIslandExtUnmasked \ -maskedSeq=/hive/data/genomes/xenTro10/xenTro10.unmasked.2bit \ -workhorse=hgwdev -smallClusterHub=ku xenTro10) > do.log 2>&1 # real 2m30.637s - cat fb.xenTro10.cpgIslandExtUnmasked.txt | sed -e 's/^/ # /;' + sed -e 's/^/ # /;' fb.xenTro10.cpgIslandExtUnmasked.txt # 19528674 bases of 1448461978 (1.348%) in intersection ############################################################################# # cytoBandIdeo - (DONE - 2021-02-22 - Hiram) mkdir /hive/data/genomes/xenTro10/bed/cytoBand cd /hive/data/genomes/xenTro10/bed/cytoBand makeCytoBandIdeo.csh xenTro10 ############################################################################# # run up idKeys files for chromAlias/ncbiRefSeq (DONE - 2021-02-22 - Hiram) mkdir /hive/data/genomes/xenTro10/bed/idKeys cd /hive/data/genomes/xenTro10/bed/idKeys time (doIdKeys.pl \ -twoBit=/hive/data/genomes/xenTro10/xenTro10.unmasked.2bit \ -buildDir=`pwd` xenTro10) > do.log 2>&1 & # real 0m28.960s cat xenTro10.keySignature.txt # b7d209c6d0405acdf866c641f2b10929 ############################################################################# # gapOverlap (DONE - 2020-02-22 - Hiram) mkdir /hive/data/genomes/xenTro10/bed/gapOverlap cd /hive/data/genomes/xenTro10/bed/gapOverlap time (doGapOverlap.pl \ -twoBit=/hive/data/genomes/xenTro10/xenTro10.unmasked.2bit xenTro10 ) \ > do.log 2>&1 & -XXX - running - Mon Feb 22 13:00:00 PST 2021 - # real 0m54.302s + # real 67m49.060s - # there were not very many gaps, it only had to do one job and blat - # found nothing. + # there were 4 items found # this result does not exist: - cat fb.xenTro10.gapOverlap.txt - # 608 bases of 2728222451 (0.000%) in intersection - - # manually finish off since it quit in the load step - doGapOverlap.pl -continue=cleanup \ - -twoBit=/hive/data/genomes/xenTro10/xenTro10.unmasked.2bit xenTro10 + sed -e 's/^/ # /;' fb.xenTro10.gapOverlap.txt + # 3896 bases of 1451301209 (0.000%) in intersection ############################################################################# # tandemDups (DONE - 2020-02-22 - Hiram) mkdir /hive/data/genomes/xenTro10/bed/tandemDups cd /hive/data/genomes/xenTro10/bed/tandemDups time (~/kent/src/hg/utils/automation/doTandemDup.pl \ -twoBit=/hive/data/genomes/xenTro10/xenTro10.unmasked.2bit xenTro10) \ > do.log 2>&1 & -XXX - running - Mon Feb 22 13:05:48 PST 2021 - # real 193m21.761s + # real 384m13.927s - cat fb.xenTro10.tandemDups.txt - # 80358205 bases of 2897824427 (2.773%) in intersection + sed -e 's/^/ # /;' fb.xenTro10.tandemDups.txt + # 107198001 bases of 1451301209 (7.386%) in intersection bigBedInfo xenTro10.tandemDups.bb | sed -e 's/^/# /;' # version: 4 # fieldCount: 13 # hasHeaderExtension: yes # isCompressed: yes # isSwapped: 0 # extraIndexCount: 0 -# itemCount: 1,402,773 -# primaryDataSize: 36,657,311 -# primaryIndexSize: 119,132 -# zoomLevels: 9 -# chromCount: 894 -# basesCovered: 1,457,658,879 -# meanDepth (of bases covered): 8.428920 +# itemCount: 1,594,381 +# primaryDataSize: 40,401,533 +# primaryIndexSize: 111,984 +# zoomLevels: 10 +# chromCount: 133 +# basesCovered: 610,044,450 +# meanDepth (of bases covered): 8.832028 # minDepth: 1.000000 -# maxDepth: 344.000000 -# std of depth: 18.274027 +# maxDepth: 252.000000 +# std of depth: 14.581624 ######################################################################### # ucscToINSDC and ucscToRefSeq table/track (DONE - 2021-02-22 - Hiram) # construct idKeys for the refseq and genbank sequence mkdir /hive/data/genomes/xenTro10/refseq/idKeys cd /hive/data/genomes/xenTro10/refseq/idKeys faToTwoBit ../G*10.0_genomic.fna.gz xenTro10.refseq.2bit time (doIdKeys.pl -buildDir=`pwd` \ -twoBit=`pwd`/xenTro10.refseq.2bit refseqXenTro10) > do.log 2>&1 & # real 0m26.841s sed -e 's/^/ # /;' refseqXenTro10.keySignature.txt # b7d209c6d0405acdf866c641f2b10929 mkdir /hive/data/genomes/xenTro10/genbank @@ -701,749 +694,768 @@ # verify searches work in the position box git commit -m 'adding gold table assembly track search rule refs #24693' \ trackDb.ra # verify in the browser the searches for the 'contig' names will function ########################################################################## # running repeat masker (DONE - 2020-02-22 - Hiram) # using new repeat masker version 4.1.0 mkdir /hive/data/genomes/xenTro10/bed/repeatMasker cd /hive/data/genomes/xenTro10/bed/repeatMasker time (doRepeatMasker.pl -buildDir=`pwd` \ -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ -smallClusterHub=hgwdev xenTro10) > do.log 2>&1 -XXX - running - Mon Feb 22 12:46:57 PST 2021 - # real 397m24.001s + # real 351m3.424s cat faSize.rmsk.txt -# 2647915728 bases (21334956 N's 2626580772 real 1457026804 upper -# 1169553968 lower) in 176 sequences in 1 files -# Total size: mean 15044975.7 sd 44833491.7 min 746 (chrUn_NW_023637828v1) -# max 260522016 (chr1) median 44754 -# %44.17 masked total, %44.53 masked real +# 1451301209 bases (2839231 N's 1448461978 real 940693203 upper +# 507768775 lower) in 167 sequences in 1 files +# Total size: mean 8690426.4 sd 36088840.6 min 582 (chrUn_NW_022279502v1) +# max 217471166 (chr1) median 4995 +# %34.99 masked total, %35.06 masked real egrep -i "versi|relea" do.log # RepeatMasker,v 1.332 2017/04/17 19:01:11 rhubley Exp $ # CC Dfam_Consensus RELEASE 20181026; * # CC RepBase RELEASE 20181026; sed -e 's/^/# /;' versionInfo.txt # The repeat files provided for this assembly were generated using RepeatMasker. # Smit, AFA, Hubley, R & Green, P., # RepeatMasker Open-4.0. # 1996-2010 <http://www.repeatmasker.org>. # # VERSION: # RepeatMasker version development-$Id: RepeatMasker,v 1.332 2017/04/17 19:01:11 rhubley Exp $ # Search Engine: Crossmatch [ 1.090518 ] # Master RepeatMasker Database: /hive/data/staging/data/RepeatMasker181121/Libraries/RepeatMaskerLib.embl ( Complete Database: dc20181026-rb20181026 ) # # # RepeatMasker version development-$Id: RepeatMasker,v 1.332 2017/04/17 19:01:11 rhubley Exp $ # CC Dfam_Consensus RELEASE 20181026; * # CC RepBase RELEASE 20181026; * # # RepeatMasker engine: -engine crossmatch -s -# # RepeatMasker library options: -species 'Rattus norvegicus' +# # RepeatMasker library options: -species 'Xenopus tropicalis' # # PARAMETERS: -# /hive/data/staging/data/RepeatMasker/RepeatMasker -engine crossmatch -s -align -species 'Rattus norvegicus' +# /hive/data/staging/data/RepeatMasker/RepeatMasker -engine crossmatch -s -align -species 'Xenopus tropicalis' time featureBits -countGaps xenTro10 rmsk - # 1169555321 bases of 2647915728 (44.169%) in intersection - # real 0m36.935s + # 507769814 bases of 1451301209 (34.987%) in intersection + # real 0m18.468s # why is it different than the faSize above ? # because rmsk masks out some N's as well as bases, the faSize count above # separates out the N's from the bases, it doesn't show lower case N's # faster way to get the same result on high contig count assemblies: time hgsql -N -e 'select genoName,genoStart,genoEnd from rmsk;' xenTro10 \ | bedSingleCover.pl stdin | ave -col=4 stdin | grep "^total" - # total 1169555321.000000 - # real 0m23.489s + # total 507769814.000000 + # real 0m10.569s ########################################################################## # running simple repeat (DONE - 2020-02-22 - Hiram) mkdir /hive/data/genomes/xenTro10/bed/simpleRepeat cd /hive/data/genomes/xenTro10/bed/simpleRepeat # a bit smaller trf409 option 4 instead of the usual 6 # it refers to the expected maximum TR length in millions time (doSimpleRepeat.pl -buildDir=`pwd` -bigClusterHub=ku \ -dbHost=hgwdev -workhorse=hgwdev -smallClusterHub=ku \ -trf409=4 xenTro10) > do.log 2>&1 & -XXX - running - Mon Feb 22 12:50:54 PST 2021 - # real 288m35.826s + # real 113m16.113s - cat fb.simpleRepeat - # 94485470 bases of 2626580772 (3.597%) in intersection + sed -e 's/^/ # /;' fb.simpleRepeat + # 190634579 bases of 1448461978 (13.161%) in intersection cd /hive/data/genomes/xenTro10 # if using the Window Masker result: cd /hive/data/genomes/xenTro10 # twoBitMask bed/windowMasker/xenTro10.cleanWMSdust.2bit \ # -add bed/simpleRepeat/trfMask.bed xenTro10.2bit # you can safely ignore the warning about fields >= 13 # add to rmsk after it is done: twoBitMask xenTro10.rmsk.2bit \ -add bed/simpleRepeat/trfMask.bed xenTro10.2bit # you can safely ignore the warning about fields >= 13 twoBitToFa xenTro10.2bit stdout | faSize stdin > faSize.xenTro10.2bit.txt cat faSize.xenTro10.2bit.txt -# 2647915728 bases (21334956 N's 2626580772 real 1452560391 upper -# 1174020381 lower) in 176 sequences in 1 files -# Total size: mean 15044975.7 sd 44833491.7 min 746 (chrUn_NW_023637828v1) -# max 260522016 (chr1) median 44754 -# %44.34 masked total, %44.70 masked real +# 1451301209 bases (2839231 N's 1448461978 real 938798043 upper +# 509663935 lower) in 167 sequences in 1 files +# Total size: mean 8690426.4 sd 36088840.6 min 582 (chrUn_NW_022279502v1) +# max 217471166 (chr1) median 4995 +# %35.12 masked total, %35.19 masked real # reset symlink rm /gbdb/xenTro10/xenTro10.2bit ln -s `pwd`/xenTro10.2bit /gbdb/xenTro10/xenTro10.2bit ######################################################################### -# CREATE MICROSAT TRACK (TBD - 2021-02-03 - Hiram) +# CREATE MICROSAT TRACK (DONE - 2021-02-22 - Hiram) ssh hgwdev mkdir /hive/data/genomes/xenTro10/bed/microsat cd /hive/data/genomes/xenTro10/bed/microsat awk '($5==2 || $5==3) && $6 >= 15 && $8 == 100 && $9 == 0 {printf("%s\t%s\t%s\t%dx%s\n", $1, $2, $3, $6, $16);}' \ ../simpleRepeat/simpleRepeat.bed > microsat.bed hgLoadBed xenTro10 microsat microsat.bed - # Read 244331 elements of size 4 from microsat.bed + # Read 14242 elements of size 4 from microsat.bed ########################################################################## -## WINDOWMASKER (TBD - 2021-02-03 - Hiram) +## WINDOWMASKER (DONE - 2021-02-22 - Hiram) mkdir /hive/data/genomes/xenTro10/bed/windowMasker cd /hive/data/genomes/xenTro10/bed/windowMasker time (doWindowMasker.pl -buildDir=`pwd` -workhorse=hgwdev \ -dbHost=hgwdev xenTro10) > do.log 2>&1 # real 103m59.019s # Masking statistics cat faSize.xenTro10.cleanWMSdust.txt -# 2647915728 bases (21334956 N's 2626580772 real 1677807978 upper -# 948772794 lower) in 176 sequences in 1 files -# Total size: mean 15044975.7 sd 44833491.7 min 746 (chrUn_NW_023637828v1) -# max 260522016 (chr1) median 44754 -# %35.83 masked total, %36.12 masked real +# 1451301209 bases (2839231 N's 1448461978 real 842864016 upper +# 605597962 lower) in 167 sequences in 1 files +# Total size: mean 8690426.4 sd 36088840.6 min 582 (chrUn_NW_022279502v1) +# max 217471166 (chr1) median 4995 +# %41.73 masked total, %41.81 masked real ########################################################################## -# cpgIslands - (TBD - 2021-02-04 - Hiram) +# cpgIslands - (DONE - 2021-02-22 - Hiram) mkdir /hive/data/genomes/xenTro10/bed/cpgIslands cd /hive/data/genomes/xenTro10/bed/cpgIslands time (doCpgIslands.pl -dbHost=hgwdev -bigClusterHub=ku \ -workhorse=hgwdev -smallClusterHub=ku xenTro10) > do.log 2>&1 - # real 3m40.227s + # real 1m53.411s sed -e 's/^/ # /;' fb.xenTro10.cpgIslandExt.txt - # 10397605 bases of 2626580772 (0.396%) in intersection + # 6982065 bases of 1448461978 (0.482%) in intersection ############################################################################## -# genscan - (TBD - 2021-02-04 - 02-15 - Hiram) +# genscan - (DONE - 2021-02-22 - Hiram) mkdir /hive/data/genomes/xenTro10/bed/genscan cd /hive/data/genomes/xenTro10/bed/genscan time (doGenscan.pl -buildDir=`pwd` -workhorse=hgwdev -dbHost=hgwdev \ -bigClusterHub=ku xenTro10) > do.log 2>&1 # real 73m39.179s - # two jobs broken: -./runGsBig2M.csh chr2 000 gtf/000/chr2.gtf pep/000/chr2.pep subopt/000/chr2.bed & -./runGsBig2M.csh chr11 000 gtf/000/chr11.gtf pep/000/chr11.pep subopt/000/chr11.bed -wait - # real 68m25.959s + # one job broken: +./runGsBig2M.csh chr8 000 gtf/000/chr8.gtf pep/000/chr8.pep subopt/000/chr8.bed + # real 60m47.074s # continuing time (doGenscan.pl -buildDir=`pwd` -workhorse=hgwdev -dbHost=hgwdev \ -continue=makeBed -bigClusterHub=ku xenTro10) > makeBed.log 2>&1 - # real 0m43.231s + # real 0m32.046s sed -e 's/^/ # /;' fb.xenTro10.genscan.txt - # 55066515 bases of 2626580772 (2.097%) in intersection + # 53001337 bases of 1448461978 (3.659%) in intersection sed -e 's/^/ # /;' fb.xenTro10.genscanSubopt.txt - # 57355860 bases of 2626580772 (2.184%) in intersection + # 36453881 bases of 1448461978 (2.517%) in intersection ######################################################################### # ncbiGene (TBD - 2020-09-03 - Hiram) # don't need to do this on GCF/RefSeq assemblies, they have RefSeq genes mkdir /hive/data/genomes/xenTro10/bed/xenoRefGene cd /hive/data/genomes/xenTro10/bed/xenoRefGene time (~/kent/src/hg/utils/automation/doXenoRefGene.pl -buildDir=`pwd` \ -bigClusterHub=ku -workhorse=hgwdev -dbHost=hgwdev xenTro10) > do.log 2>&1 & # real 67m18.015s ######################################################################### -# Create kluster run files (TBD - 2021-02-04 - Hiram) +# Create kluster run files (DONE - 2021-02-22 - Hiram) # numerator is xenTro10 gapless bases "real" as reported by: featureBits -noRandom -noHap xenTro10 gap - # 19396258 bases of 2614093470 (0.742%) in intersection + # 2833531 bases of 1446503719 (0.196%) in intersection # ^^^ # denominator is hg19 gapless bases as reported by: # featureBits -noRandom -noHap hg19 gap # 234344806 bases of 2861349177 (8.190%) in intersection # 1024 is threshold used for human -repMatch: - calc \( 2614093470 / 2861349177 \) \* 1024 - # ( 2614093470 / 2861349177 ) * 1024 = 935.513825 + calc \( 1446503719 / 2861349177 \) \* 1024 + # ( 1446503719 / 2861349177 ) * 1024 = 517.664821 - # ==> use -repMatch=1000 according to size scaled down from 1024 for human. + # ==> use -repMatch=500 according to size scaled down from 1024 for human. cd /hive/data/genomes/xenTro10 time blat xenTro10.2bit \ /dev/null /dev/null -tileSize=11 -makeOoc=jkStuff/xenTro10.11.ooc \ - -repMatch=1000 - # Wrote 27322 overused 11-mers to jkStuff/xenTro10.11.ooc - # real 0m26.195s + -repMatch=500 + # Wrote 35277 overused 11-mers to jkStuff/xenTro10.11.ooc + # real 0m12.311s + # xenTro9 at repMatch 500 was: + # Wrote 31375 overused 11-mers to jkStuff/xenTro9.11.ooc + # xenTro7 was: + # Wrote 31229 overused 11-mers to jkStuff/xenTro7.11.ooc # rn6 at repMatch=1000 # Wrote 27021 overused 11-mers to jkStuff/rn6.11.ooc # survey sizes of all gaps: hgsql -N -e 'select size from gap;' xenTro10 | ave stdin | sed -e 's/^/# /;' -# Q1 100.000000 -# median 200.000000 -# Q3 22550.000000 -# average 36721.094664 +# Q1 1000.000000 +# median 1000.000000 +# Q3 1000.000000 +# average 4150.922515 # min 10.000000 -# max 660122.000000 -# count 581 -# total 21334956.000000 -# standard deviation 95181.017236 +# max 89993.000000 +# count 684 +# total 2839231.000000 +# standard deviation 10589.035103 -hgsql -N -e 'select size from gap;' xenTro10 | sort -n | uniq -c | sed -e 's/^/# /;' + # scan the gap sizes: + hgsql -N -e 'select size from gap;' xenTro10 | sort -nr | uniq -c \ + | sed -e 's/^/# /;' | less - # There are no non-bridged gaps on this genome + # There are no non-bridged gaps on this genome, this survey does nothing # survey sizes of non-bridged gaps: hgsql -N -e 'select size from gap where bridge="no" order by size;' \ xenTro10 | sort | uniq -c | sort -k2,2n | sed -e 's/^/# /;' # 7 100 # 8 50000 # 22 1000000 - # and survey the number bridged gaps over 50,000 bases: - hgsql -N -e 'select size from gap where bridge="yes" and size > 49999;' \ + # and survey the number bridged gaps over 10,000 bases: + hgsql -N -e 'select size from gap where bridge="yes" and size > 9999;' \ xenTro10 | wc -l - # 92 + # 65 - # forget the non-bridged of size 100, use 50,000 and allow bridged + # forget the non-bridged of size 100, use 10,000 and allow bridged - # use gap size of 50000 to construct a lift file: - gapToLift -allowBridged -verbose=2 -minGap=50000 xenTro10 \ + # use gap size of 10000 to construct a lift file: + gapToLift -allowBridged -verbose=2 -minGap=10000 xenTro10 \ jkStuff/xenTro10.gaps.lft -bedFile=jkStuff/xenTro10.gaps.bed wc -l jkStuff/xenTro10.gaps* - # 268 jkStuff/xenTro10.gaps.bed - # 268 jkStuff/xenTro10.gaps.lft + # 232 jkStuff/xenTro10.gaps.bed + # 232 jkStuff/xenTro10.gaps.lft # to see the gaps sizes used: bedInvert.pl chrom.sizes jkStuff/xenTro10.gaps.bed \ | cut -f4 | sort -n | uniq -c | less ############################################################################## -# lastz/chain/net swap human/hg38 (TBD - 2021-02-04 - Hiram) +# lastz/chain/net swap human/hg38 (DONE - 2021-02-22 - Hiram) # original alignment - cd /hive/data/genomes/hg38/bed/lastzXenTro10.2021-02-04 + cd /hive/data/genomes/hg38/bed/lastzXenTro10.2021-02-22 sed -e 's/^/ # /;' fb.hg38.chainXenTro10Link.txt - # 958592205 bases of 3110768607 (30.815%) in intersection + # 146412457 bases of 3110768607 (4.707%) in intersection sed -e 's/^/ # /;' fb.hg38.chainSynXenTro10Link.txt - # 904066852 bases of 3110768607 (29.062%) in intersection + # 41291684 bases of 3110768607 (1.327%) in intersection - time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ - hg38 xenTro10) > rbest.log 2>&1 & - # real 313m26.149s + time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` hg38 xenTro10) \ + > rbest.log 2>&1 & + # real 435m37.210s sed -e 's/^/ # /;' fb.hg38.chainRBest.XenTro10.txt - # 883775977 bases of 3110768607 (28.410%) in intersection + # 73679844 bases of 3110768607 (2.369%) in intersection - # and the swap + # and for the swap: mkdir /hive/data/genomes/xenTro10/bed/blastz.hg38.swap cd /hive/data/genomes/xenTro10/bed/blastz.hg38.swap + time (doBlastzChainNet.pl -verbose=2 \ - /hive/data/genomes/hg38/bed/lastzXenTro10.2021-02-04/DEF \ - -chainMinScore=3000 -chainLinearGap=medium \ - -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ - -swap -syntenicNet) > swap.log 2>&1 - # real 74m20.215s + /hive/data/genomes/hg38/bed/lastzXenTro10.2021-02-22/DEF \ + -swap -chainMinScore=5000 -chainLinearGap=loose \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -syntenicNet) > swap.log 2>&1 + # real 79m35.244s sed -e 's/^/ # /;' fb.xenTro10.chainHg38Link.txt - # 928866703 bases of 2626580772 (35.364%) in intersection + # 150875559 bases of 1448461978 (10.416%) in intersection sed -e 's/^/ # /;' fb.xenTro10.chainSynHg38Link.txt - # 879484562 bases of 2626580772 (33.484%) in intersection + # 40317510 bases of 1448461978 (2.783%) in intersection - time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ - xenTro10 hg38) > rbest.log 2>&1 & - # real 373m6.610s + time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` xenTro10 hg38) \ + > rbest.log 2>&1 + # real 344m19.222s sed -e 's/^/ # /;' fb.xenTro10.chainRBest.Hg38.txt - # 885516265 bases of 2626580772 (33.714%) in intersection + # 71467857 bases of 1448461978 (4.934%) in intersection ############################################################################## -# lastz/chain/net swap mouse/mm39 (TBD - 2021-02-04 - Hiram) +# lastz/chain/net swap mouse/mm39 (DONE - 2021-02-22 - Hiram) # original alignment - cd /hive/data/genomes/mm39/bed/lastzXenTro10.2021-02-04 + cd /hive/data/genomes/mm39/bed/lastzXenTro10.2021-02-22 sed -e 's/^/ # /;' fb.mm39.chainXenTro10Link.txt - # 1898735724 bases of 2654624157 (71.526%) in intersection + # 53459877 bases of 2654624157 (2.014%) in intersection sed -e 's/^/ # /;' fb.mm39.chainSynXenTro10Link.txt - # 1787593557 bases of 2654624157 (67.339%) in intersection - - time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ - mm39 xenTro10) > rbest.log 2>&1 & - # real 588m38.122s - + # 22503702 bases of 2654624157 (0.848%) in intersection sed -e's/^/ # /;' fb.mm39.chainRBest.XenTro10.txt - # 1754204799 bases of 2654624157 (66.081%) in intersection + # 38090013 bases of 2654624157 (1.435%) in intersection mkdir /hive/data/genomes/xenTro10/bed/blastz.mm39.swap cd /hive/data/genomes/xenTro10/bed/blastz.mm39.swap - time (doBlastzChainNet.pl -verbose=2 \ - /hive/data/genomes/mm39/bed/lastzXenTro10.2021-02-04/DEF \ - -swap -syntenicNet \ - -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ - -chainMinScore=5000 -chainLinearGap=medium) > swap.log 2>&1 - # real 116m47.862s + time (doBlastzChainNet.pl -noDbNameCheck -swap -verbose=2 \ + /hive/data/genomes/mm39/bed/lastzXenTro10.2021-02-22/DEF \ + -syntenicNet -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ + -chainMinScore=5000 -chainLinearGap=loose) > swap.log 2>&1 + # real 11m51.841s sed -e 's/^/ # /;' fb.xenTro10.chainMm39Link.txt - # 1855165978 bases of 2626580772 (70.630%) in intersection + # 69880088 bases of 1448461978 (4.824%) in intersection sed -e 's/^/ # /;' fb.xenTro10.chainSynMm39Link.txt - # 1763550257 bases of 2626580772 (67.142%) in intersection + # 22992776 bases of 1448461978 (1.587%) in intersection time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ xenTro10 mm39) > rbest.log 2>&1 - # real 605m42.354s + # real 189m59.538s sed -e 's/^/ # /;' fb.xenTro10.chainRBest.Mm39.txt - # 1754416686 bases of 2626580772 (66.795%) in intersection + + # 37509757 bases of 1448461978 (2.590%) in intersection ############################################################################## -# lastz/chain/net swap mouse/mm10 (TBD - 2021-02-15 - Hiram) +# lastz/chain/net swap mouse/mm10 (DONE - 2021-02-22 - Hiram) # original alignment - cd /hive/data/genomes/mm10/bed/lastzXenTro10.2021-02-15 + cd /hive/data/genomes/mm10/bed/lastzXenTro10.2021-02-22 sed -e 's/^/ # /;' fb.mm10.chainXenTro10Link.txt - # 1896928045 bases of 2652783500 (71.507%) in intersection + # 96546694 bases of 2652783500 (3.639%) in intersection sed -e 's/^/ # /;' fb.mm10.chainSynXenTro10Link.txt - # 1787142074 bases of 2652783500 (67.369%) in intersection - - time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ - mm10 xenTro10) > rbest.log 2>&1 & - # real 578m13.711s - + # 34676951 bases of 2652783500 (1.307%) in intersection sed -e 's/^/ # /;' fb.mm10.chainRBest.XenTro10.txt - # 1753198266 bases of 2652783500 (66.089%) in intersection + # 62288287 bases of 2652783500 (2.348%) in intersection + # and for the swap mkdir /hive/data/genomes/xenTro10/bed/blastz.mm10.swap cd /hive/data/genomes/xenTro10/bed/blastz.mm10.swap time (doBlastzChainNet.pl -verbose=2 \ - /hive/data/genomes/mm10/bed/lastzXenTro10.2021-02-15/DEF \ - -swap -syntenicNet \ - -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ - -chainMinScore=5000 -chainLinearGap=medium) > swap.log 2>&1 - # real 112m36.899s + /hive/data/genomes/mm10/bed/lastzXenTro10.2021-02-22/DEF \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -syntenicNet -swap -chainMinScore=5000 -chainLinearGap=loose) \ + > swap.log 2>&1 & + # real 24m33.940s sed -e 's/^/ # /;' fb.xenTro10.chainMm10Link.txt - # 1853300495 bases of 2626580772 (70.559%) in intersection + # 121679610 bases of 1448461978 (8.401%) in intersection sed -e 's/^/ # /;' fb.xenTro10.chainSynMm10Link.txt - # 1762899567 bases of 2626580772 (67.118%) in intersection + # 35210769 bases of 1448461978 (2.431%) in intersection - time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ - xenTro10 mm10) > rbest.log 2>&1 - # real 599m24.766s + time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` xenTro10 mm10) \ + > rbest.log 2>&1 & + # real 372m38.637s sed -e 's/^/ # /;' fb.xenTro10.chainRBest.Mm10.txt - # 1753558422 bases of 2626580772 (66.762%) in intersection + # 58901471 bases of 1448461978 (4.066%) in intersection ############################################################################## -# GENBANK AUTO UPDATE (TBD - 2021-02-04 - Hiram) +# GENBANK AUTO UPDATE (DONE - 2021-03-01 - Hiram) ssh hgwdev cd $HOME/kent/src/hg/makeDb/genbank git pull # /cluster/data/genbank/data/organism.lst shows: # organism mrnaCnt estCnt refSeqCnt - # Rattus norvegicus 130246 1103640 17794 + # Xenopus tropicalis 22512 1271482 8614 # edit etc/genbank.conf to add xenTro10 just before rn6 -# xenTro10 (xenTro GCF_015227675.2 mRat10.0) +# xenTro10 'Xenopus tropicalis' GCF_000004195.4_UCB_Xtro_10.0 xenTro10.serverGenome = /hive/data/genomes/xenTro10/xenTro10.2bit xenTro10.ooc = /hive/data/genomes/xenTro10/jkStuff/xenTro10.11.ooc xenTro10.lift = /hive/data/genomes/xenTro10/jkStuff/xenTro10.gaps.lft +xenTro10.perChromTables = no +xenTro10.downloadDir = xenTro10 +# xenTro10.mgc = yes xenTro10.refseq.mrna.native.pslCDnaFilter = ${finished.refseq.mrna.native.pslCDnaFilter} xenTro10.refseq.mrna.xeno.pslCDnaFilter = ${finished.refseq.mrna.xeno.pslCDnaFilter} xenTro10.genbank.mrna.native.pslCDnaFilter = ${finished.genbank.mrna.native.pslCDnaFilter} xenTro10.genbank.mrna.xeno.pslCDnaFilter = ${finished.genbank.mrna.xeno.pslCDnaFilter} xenTro10.genbank.est.native.pslCDnaFilter = ${finished.genbank.est.native.pslCDnaFilter} -xenTro10.downloadDir = xenTro10 -xenTro10.refseq.mrna.xeno.load = yes -xenTro10.refseq.mrna.xeno.loadDesc = yes -xenTro10.genbank.mrna.xeno.load = yes -xenTro10.perChromTables = no -xenTro10.mgc = yes +# DO NOT NEED genbank.mrna.xeno except for human, mouse +# defaults yes: genbank.mrna.native.load, genbank.mrna.native.loadDesc, +# genbank.est.native.load, refseq.mrna.native.load, refseq.mrna.native.loadDesc, +# refseq.mrna.xeno.load , refseq.mrna.xeno.loadDesc # xenTro10.upstreamGeneTbl = ensGene -# xenTro10.upstreamMaf = multiz13way /hive/data/genomes/xenTro10/bed/multiz13way/species.list.txt +# xenTro10.upstreamMaf = multiz9way /hive/data/genomes/xenTro7/bed/multiz9way/species.list # verify the files specified exist before checking in the file: grep ^xenTro10 etc/genbank.conf | grep hive | awk '{print $NF}' | xargs ls -og -# -rw-rw-r-- 1 109296 Feb 4 09:28 /hive/data/genomes/xenTro10/jkStuff/xenTro10.11.ooc -# -rw-rw-r-- 1 13791 Feb 4 09:34 /hive/data/genomes/xenTro10/jkStuff/xenTro10.gaps.lft -# -rw-rw-r-- 1 692151553 Feb 4 09:20 /hive/data/genomes/xenTro10/xenTro10.2bit +# -rw-rw-r-- 1 141116 Feb 22 21:40 /hive/data/genomes/xenTro10/jkStuff/xenTro10.11.ooc +# -rw-rw-r-- 1 11952 Feb 22 21:44 /hive/data/genomes/xenTro10/jkStuff/xenTro10.gaps.lft +# -rw-rw-r-- 1 376253990 Feb 22 21:31 /hive/data/genomes/xenTro10/xenTro10.2bit git commit -m "Added xenTro10 xenTro refs #24693" etc/genbank.conf git push # update /cluster/data/genbank/: make etc-update # enable daily alignment and update of hgwdev cd ~/kent/src/hg/makeDb/genbank git pull # add xenTro10 to: # etc/hgwdev.dbs etc/align.dbs git commit -m "Added xenTro10 - xenTro refs #24693" etc/hgwdev.dbs etc/align.dbs git push make etc-update # wait a few days for genbank magic to take place, the tracks will # appear ############################################################################# -# augustus gene track (TBD - 2021-02-04 - Hiram) +# augustus gene track (DONE - 2021-02-22 - Hiram) mkdir /hive/data/genomes/xenTro10/bed/augustus cd /hive/data/genomes/xenTro10/bed/augustus + # verify you have the correct species here, check the make doc file: + # makeDb/doc/augustusGene.txt + time (doAugustus.pl -buildDir=`pwd` -bigClusterHub=ku \ - -species=human -dbHost=hgwdev \ + -species=chicken -dbHost=hgwdev \ -workhorse=hgwdev xenTro10) > do.log 2>&1 - # real 119m10.900s + # real 48m47.831s - cat fb.xenTro10.augustusGene.txt - # 49646030 bases of 2626580772 (1.890%) in intersection + sed -e 's/^/ # /;' fb.xenTro10.augustusGene.txt + # 41549240 bases of 1448461978 (2.869%) in intersection ######################################################################### -# ncbiRefSeq (TBD - 2021-02-04 - Hiram) +# ncbiRefSeq (DONE - 2021-02-22 - Hiram) mkdir /hive/data/genomes/xenTro10/bed/ncbiRefSeq cd /hive/data/genomes/xenTro10/bed/ncbiRefSeq time (~/kent/src/hg/utils/automation/doNcbiRefSeq.pl -buildDir=`pwd` \ -bigClusterHub=ku -dbHost=hgwdev \ -fileServer=hgwdev -smallClusterHub=hgwdev -workhorse=hgwdev \ - GCF_015227675.2_mRat10.0 xenTro10) > do.log 2>&1 & - # real 6m52.120s + GCF_000004195.4_UCB_Xtro_10.0 xenTro10) > do.log 2>&1 & + # real 3m44.945s - cat fb.ncbiRefSeq.xenTro10.txt - # 107770866 bases of 2626580772 (4.103%) in intersection + sed -e 's/^/ # /;' fb.ncbiRefSeq.xenTro10.txt + # 68195666 bases of 1448461978 (4.708%) in intersection # add: include ../../refSeqComposite.ra - # to the rat/xenTro10/trackDb.ra to turn on the track in the browser - + # to the xenTro/xenTro10/trackDb.ra to turn on the track in the browser joinerCheck says: - xenTro10.ncbiRefSeqLink.protAcc - hits 74754 of 74755 (99.999%) -Error: 1 of 74755 elements (0.001%) of xenTro10.ncbiRefSeqLink.protAcc are not in key ncbiRefSeqPepTable.name line 8640 of all.joiner -Example miss: NP_536324.1 + xenTro10.ncbiRefSeqLink.protAcc - hits 45093 of 45094 (99.998%) +Error: 1 of 45094 elements (0.002%) of xenTro10.ncbiRefSeqLink.protAcc are not in key ncbiRefSeqPepTable.name line 8923 of /cluster/home/hiram/kent/src/hg/makeDb/schema/all.joiner +Example miss: NP_001120457.1 + + # for some reason one of the proteins is missing from + # GCF_000004195.4_UCB_Xtro_10.0_protein.faa.gz + # however, it is in the GCF_000004195.4_UCB_Xtro_10.0_rna.gbff.gz + # obtain it from that file, and create download/NP_001120457.1.faa.gz + # then, add it to the table reload: +export db=xenTro10 +export asmId=GCF_000004195.4_UCB_Xtro_10.0 - # XXX 2021-02-04 - ready for this after genbank runs +zcat download/${asmId}_protein.faa.gz download/NP_001120457.1.faa.gz \ + | sed -e 's/ .*//;' | faToTab -type=protein -keepAccSuffix stdin stdout \ + | sort | join -t$'\t' $db.ncbiRefSeqLink.protAcc.list - \ + > fixed.$db.ncbiRefSeqPepTable.tab + +hgLoadSqlTab $db ncbiRefSeqPepTable ~/kent/src/hg/lib/pepPred.sql \ + fixed.$db.ncbiRefSeqPepTable.tab + +joinerCheck -keys \ + -identifier=ncbiRefSeqPepTable -database=$db \ + /cluster/home/hiram/kent/src/hg/makeDb/schema/all.joiner + +Checking keys on database xenTro10 + xenTro10.ncbiRefSeqLink.protAcc - hits 45094 of 45094 (100.000%) ok + +joinerCheck -keys \ + -identifier=ncbiRefSeq -database=$db \ + /cluster/home/hiram/kent/src/hg/makeDb/schema/all.joiner + +Checking keys on database xenTro10 + xenTro10.ncbiRefSeqLink.id - hits 50564 of 50564 (100.000%) ok + xenTro10.ncbiRefSeqCurated.name - hits 8787 of 8787 (100.000%) ok + xenTro10.ncbiRefSeqPredicted.name - hits 41783 of 41783 (100.000%) ok + xenTro10.ncbiRefSeqPsl.qName - hits 50576 of 50576 (100.000%) ok + xenTro10.ncbiRefSeqCds.id - hits 45080 of 45080 (100.000%) ok + xenTro10.seqNcbiRefSeq.acc - hits 50564 of 50564 (100.000%) ok featureBits -enrichment xenTro10 refGene ncbiRefSeq - # refGene 0.402%, ncbiRefSeq 3.148%, both 0.402%, cover 99.90%, enrich 31.73x + # refGene 1.354%, ncbiRefSeq 4.708%, both 1.351%, cover 99.79%, enrich 21.20x + featureBits -enrichment xenTro10 ncbiRefSeq refGene - # ncbiRefSeq 3.148%, refGene 0.402%, both 0.402%, cover 12.76%, enrich 31.73x + # ncbiRefSeq 4.708%, refGene 1.354%, both 1.351%, cover 28.70%, enrich 21.20x featureBits -enrichment xenTro10 ncbiRefSeqCurated refGene - # ncbiRefSeqCurated 0.401%, refGene 0.402%, both 0.400%, cover 99.66%, enrich 247.79x + # ncbiRefSeqCurated 1.353%, refGene 1.354%, both 1.347%, cover 99.56%, enrich 73.53x featureBits -enrichment xenTro10 refGene ncbiRefSeqCurated - # refGene 0.402%, ncbiRefSeqCurated 0.401%, both 0.400%, cover 99.33%, enrich 247.79x + # refGene 1.354%, ncbiRefSeqCurated 1.353%, both 1.347%, cover 99.45%, enrich 73.53x + +############################################################################## +# LIFTOVER TO xenTro9 (DONE - 2021-02-22 - Hiram) + ssh hgwdev + mkdir /hive/data/genomes/xenTro10/bed/blat.xenTro9.2021-02-22 + cd /hive/data/genomes/xenTro10/bed/blat.xenTro9.2021-02-22 + doSameSpeciesLiftOver.pl -verbose=2 \ + -debug -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ + -query2Bit=/hive/data/genomes/xenTro9/xenTro9.2bit \ + -querySizes=/hive/data/genomes/xenTro9/chrom.sizes \ + -ooc=/hive/data/genomes/xenTro10/jkStuff/xenTro10.11.ooc \ + xenTro10 xenTro9 + time (doSameSpeciesLiftOver.pl -verbose=2 \ + -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ + -query2Bit=/hive/data/genomes/xenTro9/xenTro9.2bit \ + -querySizes=/hive/data/genomes/xenTro9/chrom.sizes \ + -ooc=/hive/data/genomes/xenTro10/jkStuff/xenTro10.11.ooc \ + xenTro10 xenTro9) > doLiftOverToXenTro9.log 2>&1 + # real 672m20.484s + + # see if the liftOver menus function in the browser from xenTro10 to xenTro9 ############################################################################## -# LIFTOVER TO rn6 (TBD - 2021-02-04 - Hiram) +# LIFTOVER TO xenTro7 (DONE - 2021-02-23 - Hiram) ssh hgwdev - mkdir /hive/data/genomes/xenTro10/bed/blat.rn6.2021-02-04 - cd /hive/data/genomes/xenTro10/bed/blat.rn6.2021-02-04 + mkdir /hive/data/genomes/xenTro10/bed/blat.xenTro7.2021-02-23 + cd /hive/data/genomes/xenTro10/bed/blat.xenTro7.2021-02-23 doSameSpeciesLiftOver.pl -verbose=2 \ -debug -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ - -query2Bit=/hive/data/genomes/rn6/rn6.2bit \ - -querySizes=/hive/data/genomes/rn6/chrom.sizes \ + -query2Bit=/hive/data/genomes/xenTro7/xenTro7.2bit \ + -querySizes=/hive/data/genomes/xenTro7/chrom.sizes \ -ooc=/hive/data/genomes/xenTro10/jkStuff/xenTro10.11.ooc \ - xenTro10 rn6 + xenTro10 xenTro7 time (doSameSpeciesLiftOver.pl -verbose=2 \ -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ - -query2Bit=/hive/data/genomes/rn6/rn6.2bit \ - -querySizes=/hive/data/genomes/rn6/chrom.sizes \ + -query2Bit=/hive/data/genomes/xenTro7/xenTro7.2bit \ + -querySizes=/hive/data/genomes/xenTro7/chrom.sizes \ -ooc=/hive/data/genomes/xenTro10/jkStuff/xenTro10.11.ooc \ - xenTro10 rn6) > doLiftOverToXenTro10.log 2>&1 - # real 248m51.413s + xenTro10 xenTro7) > doLiftOverToXenTro7.log 2>&1 + # real 583m31.856s - # see if the liftOver menus function in the browser from xenTro10 to rn6 + # see if the liftOver menus function in the browser from xenTro10 to xenTro7 ############################################################################## -# BLATSERVERS ENTRY (TBD - 2021-02-04 - Hiram) +# BLATSERVERS ENTRY (DONE - 2021-02-23 - Hiram) # After getting a blat server assigned by the Blat Server Gods, ssh hgwdev hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \ - VALUES ("xenTro10", "blat1b", "17910", "1", "0"); \ + VALUES ("xenTro10", "blat1b", "17912", "1", "0"); \ INSERT INTO blatServers (db, host, port, isTrans, canPcr) \ - VALUES ("xenTro10", "blat1b", "17911", "0", "1");' \ + VALUES ("xenTro10", "blat1b", "17913", "0", "1");' \ hgcentraltest # test it with some sequence ############################################################################## -## reset default position to same as rn6 via blat of the DNA from rn6 -## (TBD - 2021-02-04 - Hiram) +## reset default position to same as xenTro9 via liftOver result +## (DONE - 2021-02-23 - Hiram) ssh hgwdev - hgsql -e 'update dbDb set defaultPos="chr1:79348972-79379997" + hgsql -e 'update dbDb set defaultPos="chr9:21842126-21855658" where name="xenTro10";' hgcentraltest ############################################################################## -# crispr whole genome (TBD - 2020-09-04 -> 2020-09-10 - Hiram) +# crispr whole genome (DONE - 2021-02-23 -> 2021-02-25 - Hiram) mkdir /hive/data/genomes/xenTro10/bed/crisprAll cd /hive/data/genomes/xenTro10/bed/crisprAll # need to have augustus genes done. This will not work with genscan # the large shoulder argument will cause the entire genome to be scanned # this takes a while for a new genome to get the bwa indexing done time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 -stop=ranges \ xenTro10 augustusGene -shoulder=250000000 -tableName=crisprAll \ -fileServer=hgwdev \ -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ -workhorse=hgwdev) >> ranges.log 2>&1 - # real 64m40.351s + # real 38m36.466s time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 \ -continue=guides -stop=load xenTro10 augustusGene \ -shoulder=250000000 -tableName=crisprAll -fileServer=hgwdev \ -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ -workhorse=hgwdev) > specScores.log 2>&1 - # real 7968m4.344s - - # broken specScores step due to power failure ku reset - # first part of specScores -# Completed: 925541 of 3066093 jobs -# Crashed: 503 jobs -# Other count: 1999108 jobs -# CPU time in finished jobs: 84291533s 1404858.88m 23414.31h 975.60d 2.673 y -# IO & Wait Time: 2560453s 42674.22m 711.24h 29.63d 0.081 y -# Average job time: 94s 1.56m 0.03h 0.00d -# Longest finished job: 709s 11.82m 0.20h 0.01d -# Submission to last job: 87660s 1461.00m 24.35h 1.01d - # second part of specScores -# Completed: 2140552 of 2140552 jobs -# CPU time in finished jobs: 190374829s 3172913.81m 52881.90h 2203.41d 6.037 y -# IO & Wait Time: 4108443s 68474.05m 1141.23h 47.55d 0.130 y -# Average job time: 91s 1.51m 0.03h 0.00d -# Longest finished job: 193s 3.22m 0.05h 0.00d -# Submission to last job: 208681s 3478.02m 57.97h 2.42d - - # and putting together the results - time find tmp/outGuides -type f | xargs cut -f3-6 > ../specScores.tab - - # real 806m23.857s - # user 4m0.906s - # sys 13m39.775s - - # continuing with effScores: - time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 \ - -continue=effScores -stop=load xenTro10 augustusGene \ - -shoulder=250000000 -tableName=crisprAll -fileServer=hgwdev \ - -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ - -workhorse=hgwdev) > effScores.log 2>&1 - # real 1964m18.261s - + # real 2373m12.731s - cat guides/run.time | sed -e 's/^/# /;' + sed -e 's/^/# /;' guides/run.time # Completed: 100 of 100 jobs -# CPU time in finished jobs: 13041s 217.34m 3.62h 0.15d 0.000 y -# IO & Wait Time: 340s 5.67m 0.09h 0.00d 0.000 y -# Average job time: 134s 2.23m 0.04h 0.00d -# Longest finished job: 512s 8.53m 0.14h 0.01d -# Submission to last job: 513s 8.55m 0.14h 0.01d - - cat specScores/run.time | sed -e 's/^/# /;' -# Completed: 2140552 of 2140552 jobs -# CPU time in finished jobs: 190374829s 3172913.81m 52881.90h 2203.41d 6.037 y -# IO & Wait Time: 4108443s 68474.05m 1141.23h 47.55d 0.130 y -# Average job time: 91s 1.51m 0.03h 0.00d -# Longest finished job: 193s 3.22m 0.05h 0.00d -# Submission to last job: 208681s 3478.02m 57.97h 2.42d +# CPU time in finished jobs: 6359s 105.98m 1.77h 0.07d 0.000 y +# IO & Wait Time: 294s 4.90m 0.08h 0.00d 0.000 y +# Average job time: 67s 1.11m 0.02h 0.00d +# Longest finished job: 231s 3.85m 0.06h 0.00d +# Submission to last job: 239s 3.98m 0.07h 0.00d + + sed -e 's/^/# /;' specScores/run.time +# Completed: 1325325 of 1325325 jobs +# CPU time in finished jobs: 78416373s 1306939.56m 21782.33h 907.60d 2.487 y +# IO & Wait Time: 1869408s 31156.79m 519.28h 21.64d 0.059 y +# Average job time: 61s 1.01m 0.02h 0.00d +# Longest finished job: 223s 3.72m 0.06h 0.00d +# Submission to last job: 85588s 1426.47m 23.77h 0.99d + + sed -e 's/^/# /;' effScores/run.time +# Completed: 15465 of 15465 jobs +# CPU time in finished jobs: 8306113s 138435.22m 2307.25h 96.14d 0.263 y +# IO & Wait Time: 114582s 1909.70m 31.83h 1.33d 0.004 y +# Average job time: 545s 9.08m 0.15h 0.01d +# Longest finished job: 8586s 143.10m 2.38h 0.10d +# Submission to last job: 15682s 261.37m 4.36h 0.18d + + sed -e 's/^/# /;' offTargets/run.time +# Completed: 66267 of 66267 jobs +# CPU time in finished jobs: 747369s 12456.15m 207.60h 8.65d 0.024 y +# IO & Wait Time: 643458s 10724.30m 178.74h 7.45d 0.020 y +# Average job time: 21s 0.35m 0.01h 0.00d +# Longest finished job: 44s 0.73m 0.01h 0.00d +# Submission to last job: 1463s 24.38m 0.41h 0.02d grep "Number of specScores" specScores.log -# Number of specScores: 229203141 - - cat effScores/run.time | sed -e 's/^/# /;' -# Completed: 27591 of 27591 jobs -# CPU time in finished jobs: 13780234s 229670.56m 3827.84h 159.49d 0.437 y -# IO & Wait Time: 93812s 1563.54m 26.06h 1.09d 0.003 y -# Average job time: 503s 8.38m 0.14h 0.01d -# Longest finished job: 52764s 879.40m 14.66h 0.61d -# Submission to last job: 70493s 1174.88m 19.58h 0.82d - - cat offTargets/run.time | sed -e 's/^/# /;' -# Completed: 153305 of 153305 jobs -# CPU time in finished jobs: 2423705s 40395.08m 673.25h 28.05d 0.077 y -# IO & Wait Time: 1680214s 28003.57m 466.73h 19.45d 0.053 y -# Average job time: 27s 0.45m 0.01h 0.00d -# Longest finished job: 60s 1.00m 0.02h 0.00d -# Submission to last job: 5880s 98.00m 1.63h 0.07d +# Number of specScores: 98782381 bigBedInfo crispr.bb | sed -e 's/^/# /;' # version: 4 # fieldCount: 22 # hasHeaderExtension: yes # isCompressed: yes # isSwapped: 0 # extraIndexCount: 0 -# itemCount: 274,741,534 -# primaryDataSize: 12,314,717,925 -# primaryIndexSize: 17,246,852 +# itemCount: 154,656,370 +# primaryDataSize: 6,243,636,535 +# primaryIndexSize: 9,708,128 # zoomLevels: 10 -# chromCount: 31 -# basesCovered: 2,157,342,037 -# meanDepth (of bases covered): 2.929093 +# chromCount: 10 +# basesCovered: 1,180,495,027 +# meanDepth (of bases covered): 3.013224 # minDepth: 1.000000 -# maxDepth: 33.000000 -# std of depth: 1.944611 +# maxDepth: 32.000000 +# std of depth: 2.066347 ######################################################################### -# all.joiner update, downloads and in pushQ - (WORKING - 2019-11-20 - Hiram) +# all.joiner update, downloads and in pushQ - (DONE - 2021-03-31 - Hiram) # had incorrect orderKey specified in beginning rn6.config.ra # correct to 18019 by looking at the output of: # hgsql -e 'select name,organism,orderKey from dbDb order by orderKey;' \ # hgcentraltest | less # oryCun1 Rabbit 18010 # regenRn1 Rat 18020 # regenRn0 Rat 18021 # rn6 Rat 18031 # xenTro10 Rat 18032 # rn5 Rat 18032 # rn4 Rat 18033 # rn3 Rat 18034 # rn2 Rat 18035 # tauEry1 Red crested turaco 18360 hgsql -e 'update dbDb set orderKey=18019 where name="xenTro10";' hgcentraltest cd $HOME/kent/src/hg/makeDb/schema # verify all the business is done for release ~/kent/src/hg/utils/automation/verifyBrowser.pl xenTro10 -# 66 tables in database xenTro10 - Dog, Canis lupus familiaris -# verified 55 tables in database xenTro10, 11 extra tables, 14 optional tables +# 76 tables in database xenTro10 - X. tropicalis, Xenopus tropicalis +# verified 74 tables in database xenTro10, 2 extra tables, 30 optional tables +# NCBI RefSeq genes 10 optional tables # chainNetRBestHg38 3 optional tables # chainNetRBestMm10 3 optional tables +# chainNetRBestMm39 3 optional tables # chainNetSynHg38 3 optional tables # chainNetSynMm10 3 optional tables +# chainNetSynMm39 3 optional tables # gapOverlap 1 optional tables # tandemDups 1 optional tables -# 1 chainCanFam3 - extra table -# 2 chainCanFam3Link - extra table -# 3 chainRBestCanFam3 - extra table -# 4 chainRBestCanFam3Link - extra table -# . . . etc . . . -# 8 crisprAllTargets - extra table -# 9 netCanFam3 - extra table -# 10 netRBestCanFam3 - extra table -# 11 netSynCanFam3 - extra table -# 13 genbank tables found -# verified 28 required tables, 1 missing tables -# 1 ucscToRefSeq - missing table +# 1 crisprAllRanges - extra table +# 2 crisprAllTargets - extra table +# 12 genbank tables found +# verified 32 required tables, 0 missing tables # hg38 chainNet to xenTro10 found 3 required tables # mm10 chainNet to xenTro10 found 3 required tables +# mm39 chainNet to xenTro10 found 3 required tables # hg38 chainNet RBest and syntenic to xenTro10 found 6 optional tables -# mm10 chainNet RBest and syntenic to xenTro10 found 3 optional tables -# liftOver to previous versions: 1, from previous versions: 1 +# mm10 chainNet RBest and syntenic to xenTro10 found 6 optional tables +# mm39 chainNet RBest and syntenic to xenTro10 found 6 optional tables +# liftOver to previous versions: 2, from previous versions: 2 +# blatServers: xenTro10 blat1b 17912 1 0 0 xenTro10 blat1b 17913 0 1 0 # fixup all.joiner until this is a clean output joinerCheck -database=xenTro10 -tableCoverage all.joiner joinerCheck -database=xenTro10 -times all.joiner joinerCheck -database=xenTro10 -keys all.joiner # when clean, check in: git commit -m 'adding rules for xenTro10 refs #24693' all.joiner git push # run up a 'make alpha' in hg/hgTables to get this all.joiner file # into the hgwdev/genome-test system cd /hive/data/genomes/xenTro10 time (makeDownloads.pl xenTro10) > downloads.log 2>&1 - # real 20m11.930s + # real 10m33.025s # now ready for pushQ entry mkdir /hive/data/genomes/xenTro10/pushQ cd /hive/data/genomes/xenTro10/pushQ time ($HOME/kent/src/hg/utils/automation/makePushQSql.pl -redmineList xenTro10) > xenTro10.pushQ.sql 2> stderr.out - # real 13m21.313s + # real 18m15.613s # remove the tandemDups and gapOverlap from the file list: sed -i -e "/tandemDups/d" redmine.xenTro10.table.list sed -i -e "/Tandem Dups/d" redmine.xenTro10.releaseLog.txt sed -i -e "/gapOverlap/d" redmine.xenTro10.table.list sed -i -e "/Gap Overlaps/d" redmine.xenTro10.releaseLog.txt - # remove the multiz7way tables: - sed -i -e "/multiz7way/d" redmine.xenTro10.table.list + # edit the file list and expand the directory wildcards, to find them: + grep "\*\/" redmine.xenTro10.file.list + # two of them + ls /gbdb/xenTro*/liftOver/xenTro*ToXenTro10.over.chain.gz \ + /usr/local/apache/htdocs-hgdownload/goldenPath/xenTro*/liftOver/xenTro*ToXenTro10.over.chain.gz + +/gbdb/xenTro7/liftOver/xenTro7ToXenTro10.over.chain.gz +/gbdb/xenTro9/liftOver/xenTro9ToXenTro10.over.chain.gz +/usr/local/apache/htdocs-hgdownload/goldenPath/xenTro7/liftOver/xenTro7ToXenTro10.over.chain.gz +/usr/local/apache/htdocs-hgdownload/goldenPath/xenTro9/liftOver/xenTro9ToXenTro10.over.chain.gz - # edit the file list and expand the wildcards: .../calJac*/... # check for errors in stderr.out, some are OK, e.g.: # redmine.xenTro10.releaseLog.txt WARNING: xenTro10 does not have seq -WARNING: hgwdev does not have phyloPng-generated /usr/local/apache/htdocs/images/phylo/xenTro10_7way.gif (or png) for multiz7way. +WARNING: xenTro10 does not have extFile WARNING: Could not tell (from trackDb, all.joiner and hardcoded lists of supporting and genbank tables) which tracks to assign these tables to: chainRBestHg38 chainRBestHg38Link - chainRBestMacFas5 - chainRBestMacFas5Link ... etc crisprAllRanges gbLoaded netRBestHg38 - netRBestMacFas5 netRBestMm10 netRBestMm39 netSynHg38 - netSynMacFas5 netSynMm10 netSynMm39 # verify the file list does correctly match to files cat redmine.xenTro10.file.list | while read L do eval ls $L > /dev/null done # should be silent, missing files will show as errors # verify database tables, how many to expect: wc -l redmine.xenTro10.table.list - # 70 redmine.xenTro10.table.list + # 62 redmine.xenTro10.table.list # how many actual: awk -F'.' '{printf "hgsql -N %s -e '"'"'show table status like \"%s\";'"'"'\n", $1, $2}' redmine.xenTro10.table.list | sh | wc -l - # 70 + # 62 # would be a smaller number actual if some were missing # add the path names to the listing files in the redmine issue # in the three appropriate entry boxes: # /hive/data/genomes/xenTro10/pushQ/redmine.xenTro10.file.list # /hive/data/genomes/xenTro10/pushQ/redmine.xenTro10.releaseLog.txt # /hive/data/genomes/xenTro10/pushQ/redmine.xenTro10.table.list #########################################################################