2e3edf11e977a5c3a12aa2e1fed9cbfa25ffe928 hiram Mon Apr 26 15:02:21 2021 -0700 cleaning up crisprAll working data about 326 Tb no redmine diff --git src/hg/makeDb/doc/rn7/initialBuild.txt src/hg/makeDb/doc/rn7/initialBuild.txt index 599f016..ef8517d 100644 --- src/hg/makeDb/doc/rn7/initialBuild.txt +++ src/hg/makeDb/doc/rn7/initialBuild.txt @@ -1,1377 +1,1384 @@ # for emacs: -*- mode: sh; -*- # This file describes browser build for the rn7 # GCF_009663435.1 # Can use existing photograph (otherwise find one before starting here) ######################################################################### # Initial steps, reuse existing photograph (DONE - 2021-02-03 - Hiram) # To start this initialBuild.txt document, from a previous assembly document: mkdir ~/kent/src/hg/makeDb/doc/rn7 cd ~/kent/src/hg/makeDb/doc/rn7 sed -e 's/calJac4/rn7/g; s/CalJac4/Rn7/g; s/DONE/TBD/g;' \ ../calJac4/initialBuild.txt > initialBuild.txt mkdir -p /hive/data/genomes/rn7/refseq cd /hive/data/genomes/rn7 # reuse existing photo from rn6: cp -p ../rn6/photoReference.txt . cat photoReference.txt photoCreditURL http://www.genome.gov/dmd/img.cfm?node=Photos/Animals/Rat%20(brown%20norway)&id=79111 photoCreditName NHGRI press photos ## download from NCBI cd /hive/data/genomes/rn7/refseq time rsync -L -a -P --stats \ rsync://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/015/227/675/GCF_015227675.2_mRatBN7.2/ ./ # sent 3,249 bytes received 4,735,520,193 bytes 46,655,403.37 bytes/sec # total size is 4,734,351,911 speedup is 1.00 # real 1m41.513s # this information is from the top of # rn7/refseq/*_assembly_report.txt # (aka: rn7/refseq/GCF_015227675.2_mRatBN7.2_assembly_report.txt # Assembly name: mRatBN7.2 # Organism name: Rattus norvegicus (Norway rat) # Infraspecific name: strain=BN/NHsdMcwi # Sex: male # Taxid: 10116 # BioSample: SAMN16261960 # BioProject: PRJNA662791 # Submitter: Wellcome Sanger Institute # Date: 2020-11-10 # Assembly type: haploid (principal pseudohaplotype of diploid) # Release type: minor # Assembly level: Chromosome # Genome representation: full # WGS project: JACYVU01 # Assembly method: FALCON-Unzip v. 2020 # Expected final version: yes # Genome coverage: 92.0x # Sequencing technology: PacBio Sequel; 10X Genomics Chromium; BioNano; Arima Hi-C # GenBank assembly accession: GCA_015227675.2 # RefSeq assembly accession: GCF_015227675.2 # RefSeq assembly and GenBank assemblies identical: no # Linked assembly: GCA_016700215.1 (alternate pseudohaplotype of diploid) # ## Assembly-Units: ## GenBank Unit Accession RefSeq Unit Accession Assembly-Unit name ## GCA_015227685.1 GCF_015227685.1 Primary Assembly ## GCA_000002045.1 GCF_000002045.1 non-nuclear # check assembly size for later reference: time faSize G*BN7.2_genomic.fna.gz # 2647915728 bases (21334956 N's 2626580772 real 1692490994 upper # 934089778 lower) in 176 sequences in 1 files # Total size: mean 15044975.7 sd 44833491.7 min 746 (NW_023637828.1) # max 260522016 (NC_051336.1) median 44754 # %35.28 masked total, %35.56 masked real # real 0m37.002s # Survey types of gaps: zgrep -v "^#" *gaps.txt.gz | cut -f5 | sort | uniq -c | sed -e 's/^/# /;' # 581 within_scaffold # And total size in gaps: zgrep -v "^#" *gaps.txt.gz | awk '{print $3-$2+1}' | ave stdin \ | sed -e 's/^/# /;' # Q1 100.000000 # median 200.000000 # Q3 22550.000000 # average 36721.094664 # min 10.000000 # max 660122.000000 # count 581 # total 21334956.000000 # standard deviation 95181.017236 # survey the sequence to see if it has IUPAC characters: time zgrep -v "^>" G*BN7.2_genomic.fna.gz \ | perl -ne '{print join("\n",split(//))}' \ | sed -e '/^$/d' | sort | uniq -c | sort -rn | sed -e 's/^/# /;' # 480465436 T # 480274173 A # 365876597 C # 365874788 G # 281434031 t # 281431374 a # 185631190 g # 185593183 c # 21334956 N real 29m30.166s user 44m12.938s sys 1m20.452s ############################################################################# # establish config.ra file (DONE - 2021-02-03 - Hiram) cd /hive/data/genomes/rn7 ~/kent/src/hg/utils/automation/prepConfig.pl rn7 mammal rat \ refseq/*_assembly_report.txt > rn7.config.ra # fix commonName: commonName Norway rat to: commonName Rat # fix orderKey: orderKey 16952 to orderKey 18032 # compare with previous version to see if it is sane: diff rn7.config.ra ../rn6/rn6.config.ra # verify it really does look sane cat rn7.config.ra # config parameters for makeGenomeDb.pl: db rn7 clade mammal scientificName Rattus norvegicus commonName Rat assemblyDate Nov. 2020 assemblyLabel Wellcome Sanger Institute assemblyShortLabel mRatBN7.2 orderKey 18032 # mitochondrial sequence included in refseq release # mitoAcc NC_001665.2 mitoAcc none fastaFiles /hive/data/genomes/rn7/ucsc/*.fa.gz agpFiles /hive/data/genomes/rn7/ucsc/*.agp # qualFiles none dbDbSpeciesDir rat photoCreditURL http://www.genome.gov/dmd/img.cfm?node=Photos/Animals/Rat%20(brown%20norway)&id=79111 photoCreditName NHGRI press photos ncbiGenomeId 73 ncbiAssemblyId 9079111 ncbiAssemblyName mRatBN7.2 ncbiBioProject 662791 ncbiBioSample SAMN16261960 genBankAccessionID GCF_015227675.2 taxId 10116 ############################################################################# # setup UCSC named files (DONE - 2021-02-03 - Hiram) mkdir /hive/data/genomes/rn7/ucsc cd /hive/data/genomes/rn7/ucsc # check for duplicate sequences: time faToTwoBit -noMask ../refseq/G*BN7.2_genomic.fna.gz refseq.2bit # real 0m38.943s twoBitDup refseq.2bit # no output is a good result, otherwise, would have to eliminate duplicates # the scripts creating the fasta here will be creating a refseq.2bit file # to be removed later # compare gaps with what the gaps.gz file reported: twoBitInfo -nBed refseq.2bit refseq.gap.bed awk '{print $3-$2}' *.gap.bed | ave stdin | sed -e 's/^/# /;' # Q1 100.000000 # median 200.000000 # Q3 22550.000000 # average 36721.094664 # min 10.000000 # max 660122.000000 # count 581 # total 21334956.000000 # standard deviation 95181.017236 # this is exactly the same set of gaps found above time ~/kent/src/hg/utils/automation/ucscCompositeAgp.pl \ ../refseq/G*BN7.2_genomic.fna.gz \ ../refseq/*_assembly_structure/Primary_Assembly NC_051336.1 chr1 NC_051337.1 chr2 NC_051338.1 chr3 NC_051339.1 chr4 NC_051340.1 chr5 NC_051341.1 chr6 NC_051342.1 chr7 NC_051343.1 chr8 NC_051344.1 chr9 NC_051345.1 chr10 NC_051346.1 chr11 NC_051347.1 chr12 NC_051348.1 chr13 NC_051349.1 chr14 NC_051350.1 chr15 NC_051351.1 chr16 NC_051352.1 chr17 NC_051353.1 chr18 NC_051354.1 chr19 NC_051355.1 chr20 NC_051356.1 chrX NC_051357.1 chrY real 11m5.085s time ~/kent/src/hg/utils/automation/unplacedWithChroms.pl \ ../refseq/*_assembly_structure/Primary_Assembly # processed 141 sequences into chrUn.fa.gz # real 0m2.448s time ~/kent/src/hg/utils/automation/unlocalizedWithChroms.pl \ ../refseq/*_assembly_structure/Primary_Assembly # 19 # Y # 12 # processed 12 sequences into chr*_random.gz 3 files # real 0m1.106s # bash syntax here mitoAcc=`grep "^# mitoAcc" ../rn7.config.ra | awk '{print $NF}'` printf "# mitoAcc %s\n" "$mitoAcc" # mitoAcc NC_001665.2 zcat \ ../refseq/*_assembly_structure/non-nuclear/assem*/AGP/chrMT.comp.agp.gz \ | grep -v "^#" | sed -e "s/^$mitoAcc/chrM/;" > chrM.agp cat chrM.agp # chrM 1 16313 1 O NC_001665.2 1 16313 + printf ">chrM\n" > chrM.fa twoBitToFa -noMask refseq.2bit:$mitoAcc stdout | grep -v "^>" >> chrM.fa gzip chrM.fa faSize chrM.fa.gz # 16313 bases (0 N's 16313 real 16313 upper 0 lower) in 1 sequences in 1 files # verify fasta and AGPs agree time faToTwoBit *.fa.gz test.2bit # real 0m50.257s cat *.agp | checkAgpAndFa stdin test.2bit 2>&1 | tail -4 # All AGP and FASTA entries agree - both files are valid # and no sequence lost from orginal: twoBitToFa test.2bit stdout | faSize stdin # 2647915728 bases (21334956 N's 2626580772 real 2626580772 upper 0 lower) # in 176 sequences in 1 files # Total size: mean 15044975.7 sd 44833491.7 min 746 (chrUn_NW_023637828v1) # max 260522016 (chr1) median 44754 # same numbers as above (except for upper/lower masking) # 2647915728 bases (21334956 N's 2626580772 real 1692490994 upper # 934089778 lower) in 176 sequences in 1 files # See if the AGP files define all the gaps: # categories of gaps: awk '$5 == "N"' *.agp | cut -f7 | sort | uniq -c | sed -e 's/^/# /;' # 581 scaffold # the gaps file defined: # 581 within_scaffold awk '$5 == "N"' *.agp | awk '{print $3-$2+1}' | ave stdin \ | sed -e 's/^/# /;' # Q1 100.000000 # median 200.000000 # Q3 22550.000000 # average 36721.094664 # min 10.000000 # max 660122.000000 # count 581 # total 21334956.000000 # standard deviation 95181.017236 # survey of AGP types of gaps: # beware, can also be type U in col 5, doesn't happen here: awk '$5 == "U"' *.agp (no output) awk '$5 == "N"' *.agp | awk '{print $7,$NF}' | sort | uniq -c \ | sed -e 's/^/# /;' # 581 scaffold map;proximity_ligation # name equivalences in the assembly_report file: grep -v "^#" \ ../refseq/G*BN7.2_assembly_report.txt \ | awk '{printf "%s\t%s\n", $1,$5}' | sort > ncbi.assembly.genbank.equivalence grep -v "^#" \ ../refseq/G*BN7.2_assembly_report.txt \ | awk '{printf "%s\t%s\n", $1,$7}' | sort > ncbi.assembly.refseq.equivalence join -t$'\t' ncbi.assembly.genbank.equivalence \ ncbi.assembly.refseq.equivalence > ncbi.genbank.refseq.names (watch our for the MT sequence has 'na' for genbank name) # no longer need these temporary 2bit files rm test.2bit refseq.2bit refseq.gap.bed ############################################################################# # Initial database build (DONE - 2021-02-03 - Hiram) # verify sequence and AGP are OK: cd /hive/data/genomes/rn7 time (makeGenomeDb.pl -workhorse=hgwdev -dbHost=hgwdev -fileServer=hgwdev \ -stop=agp rn7.config.ra) > agp.log 2>&1 # real 2m28.118s # then finish it off: time (makeGenomeDb.pl -workhorse=hgwdev -dbHost=hgwdev \ -fileServer=hgwdev -continue=db rn7.config.ra) > db.log 2>&1 # real 15m3.169s # check in the trackDb files created in TemporaryTrackDbCheckout/ # and add rn7 to trackDb/makefile refs #26682 # fixing up the images reference to rn7.jpg # temporary symlink until masked sequence is available cd /hive/data/genomes/rn7 ln -s `pwd`/rn7.unmasked.2bit /gbdb/rn7/rn7.2bit ############################################################################# # verify gap table vs NCBI gap file (DONE - 2021-02-03 - Hiram) mkdir /hive/data/genomes/rn7/bed/gap cd /hive/data/genomes/rn7/bed/gap zgrep -v "^#" ../../refseq/G*_gaps.txt.gz \ | awk '{printf "%s\t%d\t%d\t%s_%s\n", $1,$2-1,$3,$5,$6}' \ | sort -k1,1 -k2,2n > refseq.gap.bed # type survey: cut -f4 *.bed | sort | uniq -c | sed -e 's/^/# /;' # 581 within_scaffold_map;proximity_ligation # how much defined by NCBI: awk '{print $3-$2}' *.bed | ave stdin | grep -w total # total 21334956.000000 # how much in the gap table: hgsql -e 'select * from gap;' rn7 | awk '{print $4-$3}' \ | ave stdin | grep -w total # total 21334956.000000 # gap table type survey: hgsql -N -e 'select type from gap;' rn7 \ | sort | uniq -c | sed -e 's/^/ #/;' # 581 scaffold # should be same numbers everywhere, investigate anomalies ############################################################################## # cpgIslands on UNMASKED sequence (DONE - 2021-02-03 - Hiram) mkdir /hive/data/genomes/rn7/bed/cpgIslandsUnmasked cd /hive/data/genomes/rn7/bed/cpgIslandsUnmasked time (doCpgIslands.pl -dbHost=hgwdev -bigClusterHub=ku -buildDir=`pwd` \ -tableName=cpgIslandExtUnmasked \ -maskedSeq=/hive/data/genomes/rn7/rn7.unmasked.2bit \ -workhorse=hgwdev -smallClusterHub=ku rn7) > do.log 2>&1 # real 16m40.237s cat fb.rn7.cpgIslandExtUnmasked.txt # 12050281 bases of 2626580772 (0.459%) in intersection ############################################################################# # cytoBandIdeo - (DONE - 2021-02-03 - Hiram) mkdir /hive/data/genomes/rn7/bed/cytoBand cd /hive/data/genomes/rn7/bed/cytoBand makeCytoBandIdeo.csh rn7 ############################################################################# # run up idKeys files for chromAlias/ncbiRefSeq (DONE - 2021-02-03 - Hiram) mkdir /hive/data/genomes/rn7/bed/idKeys cd /hive/data/genomes/rn7/bed/idKeys time (doIdKeys.pl \ -twoBit=/hive/data/genomes/rn7/rn7.unmasked.2bit \ -buildDir=`pwd` rn7) > do.log 2>&1 & # real 0m37.959s cat rn7.keySignature.txt # 40260b0b04686933dabe31bb3dff9f3e ############################################################################# # gapOverlap (DONE - 2020-02-03 - Hiram) mkdir /hive/data/genomes/rn7/bed/gapOverlap cd /hive/data/genomes/rn7/bed/gapOverlap time (doGapOverlap.pl \ -twoBit=/hive/data/genomes/rn7/rn7.unmasked.2bit rn7 ) \ > do.log 2>&1 & # real 1m49.039s cat fb.rn7.gapOverlap.txt # 3284 bases of 2647915728 (0.000%) in intersection ############################################################################# # tandemDups (DONE - 2020-02-03 - Hiram) mkdir /hive/data/genomes/rn7/bed/tandemDups cd /hive/data/genomes/rn7/bed/tandemDups time (~/kent/src/hg/utils/automation/doTandemDup.pl \ -twoBit=/hive/data/genomes/rn7/rn7.unmasked.2bit rn7) \ > do.log 2>&1 & # real 415m56.286s cat fb.rn7.tandemDups.txt # 92889085 bases of 2647915728 (3.508%) in intersection bigBedInfo rn7.tandemDups.bb | sed -e 's/^/# /;' # version: 4 # fieldCount: 13 # hasHeaderExtension: yes # isCompressed: yes # isSwapped: 0 # extraIndexCount: 0 # itemCount: 1,172,090 # primaryDataSize: 30,363,941 # primaryIndexSize: 86,972 # zoomLevels: 9 # chromCount: 150 # basesCovered: 1,472,207,966 # meanDepth (of bases covered): 6.716224 # minDepth: 1.000000 # maxDepth: 279.000000 # std of depth: 12.806697 ######################################################################### # ucscToINSDC and ucscToRefSeq table/track (DONE - 2021-02-03 - Hiram) # construct idKeys for the refseq and genbank sequence mkdir /hive/data/genomes/rn7/refseq/idKeys cd /hive/data/genomes/rn7/refseq/idKeys faToTwoBit ../G*BN7.2_genomic.fna.gz rn7.refseq.2bit time (doIdKeys.pl -buildDir=`pwd` \ -twoBit=`pwd`/rn7.refseq.2bit refseqRn7) > do.log 2>&1 & # real 6m36.946s sed -e 's/^/ # /;' refseqRn7.keySignature.txt # 40260b0b04686933dabe31bb3dff9f3e mkdir /hive/data/genomes/rn7/genbank cd /hive/data/genomes/rn7/genbank faToTwoBit \ /hive/data/outside/ncbi/genomes/GCA/015/227/675/GCA_015227675.2_mRatBN7.2/GCA_015227675.2_mRatBN7.2_genomic.fna.gz rn7.genbank.2bit time (doIdKeys.pl -buildDir=`pwd` \ -twoBit=`pwd`/rn7.genbank.2bit genbankRn7) > do.log 2>&1 & # real 2m56.082s sed -e 's/^/ # /;' genbankRn7.keySignature.txt # 40260b0b04686933dabe31bb3dff9f3e mkdir /hive/data/genomes/rn7/bed/chromAlias cd /hive/data/genomes/rn7/bed/chromAlias join -t$'\t' ../idKeys/rn7.idKeys.txt \ ../../genbank/genbankRn7.idKeys.txt | cut -f2- \ | sort -k1,1 | join -t$'\t' <(sort -k1,1 ../../chrom.sizes) - \ | awk '{printf "%s\t0\t%d\t%s\n", $1, $2, $3}' \ | sort -k1,1 -k2,2n > ucscToINSDC.bed join -t$'\t' ../idKeys/rn7.idKeys.txt \ ../../refseq/idKeys/refseqRn7.idKeys.txt | cut -f2- \ | sort -k1,1 | join -t$'\t' <(sort -k1,1 ../../chrom.sizes) - \ | awk '{printf "%s\t0\t%d\t%s\n", $1, $2, $3}' \ | sort -k1,1 -k2,2n > ucscToRefSeq.bed # IF the genbank list is missing chrM, look it up in Entrez nucleotide, then: # grep chrM ucscToRefSeq.bed | sed -e 's/NC_025586.1/KM588314.1/;' \ # >> ucscToINSDC.bed # # and re-sort # cat ucscToINSDC.bed | sort -k1,1 -k2,2n > t # mv t ucscToINSDC.bed # should be same line counts throughout: wc -l * ../../chrom.sizes # 176 ucscToINSDC.bed # 176 ucscToRefSeq.bed # 176 ../../chrom.sizes export chrSize=`cut -f1 ucscToINSDC.bed | awk '{print length($0)}' | sort -n | tail -1` echo $chrSize # 27 # use the $chrSize in this sed sed -e "s/21/$chrSize/" $HOME/kent/src/hg/lib/ucscToINSDC.sql \ | hgLoadSqlTab rn7 ucscToINSDC stdin ucscToINSDC.bed export chrSize=`cut -f1 ucscToRefSeq.bed | awk '{print length($0)}' | sort -n | tail -1` echo $chrSize sed -e "s/21/$chrSize/" $HOME/kent/src/hg/lib/ucscToINSDC.sql \ | sed -e 's/INSDC/RefSeq/g;' \ | hgLoadSqlTab rn7 ucscToRefSeq stdin ucscToRefSeq.bed # should be quiet for all OK checkTableCoords rn7 ucscToINSDC checkTableCoords rn7 ucscToRefSeq # should cover %100 entirely: featureBits -countGaps rn7 ucscToINSDC # 2647915728 bases of 2647915728 (100.000%) in intersection featureBits -countGaps rn7 ucscToRefSeq # 2647915728 bases of 2647915728 (100.000%) in intersection ######################################################################### # add chromAlias table (DONE - 2021-02-03 - Hiram) mkdir /hive/data/genomes/rn7/bed/chromAlias cd /hive/data/genomes/rn7/bed/chromAlias grep -v "^#" ../../refseq/G*BN7.2_assembly_report.txt \ | awk '{printf "%s\t%s\n", $5,$1}' | sort > ncbi.genbank.txt grep -v "^#" ../../refseq/G*BN7.2_assembly_report.txt \ | awk '{printf "%s\t%s\n", $7,$1}' | sort > ncbi.refseq.txt hgsql -N -e 'select chrom,name from ucscToINSDC;' rn7 \ | sort -k1,1 > ucsc.genbank.tab hgsql -N -e 'select chrom,name from ucscToRefSeq;' rn7 \ | sort -k1,1 > ucsc.refseq.tab # the awk removes lines where the UCSC name is identical to the NCBI name join -t$'\t' -1 2 <(sort -k2,2 ucsc.refseq.tab) ncbi.refseq.txt \ | cut -f2-3 | awk '$1 != $2' | sort > ucsc.assembly.tab # IF needing to add chrM definition to genbank file # printf "chrM\tNC_005089.1\n" > ucsc.genbank.tab # genbank and refseq should be the same, assembly can be less wc -l *.tab ../../chrom.sizes # 176 ucsc.assembly.tab # 176 ucsc.genbank.tab # 176 ucsc.refseq.tab # 176 ../../chrom.sizes ~/kent/src/hg/utils/automation/chromAlias.pl ucsc.*.tab \ > rn7.chromAlias.tab # working: assembly # working: genbank # working: refseq for t in assembly genbank refseq do c0=`cat ucsc.$t.tab | wc -l` c1=`grep $t rn7.chromAlias.tab | wc -l` ok="OK" if [ "$c0" -ne "$c1" ]; then ok="ERROR" fi printf "# checking $t: $c0 =? $c1 $ok\n" done # checking assembly: 176 =? 176 OK # checking genbank: 176 =? 176 OK # checking refseq: 176 =? 176 OK # verify chrM is here properly: grep chrM rn7.chromAlias.tab # KM588314.1 chrM genbank # MT chrM assembly # NC_025586.1 chrM refseq hgLoadSqlTab rn7 chromAlias ~/kent/src/hg/lib/chromAlias.sql \ rn7.chromAlias.tab ######################################################################### # fixup search rule for assembly track/gold table (DONE - 2021-02-03 - Hiram) cd ~/kent/src/hg/makeDb/trackDb/rat/rn7 # preview prefixes and suffixes: hgsql -N -e "select frag from gold;" rn7 \ | sed -e 's/[0-9.]\+//;' | sort | uniq -c | sed -e 's/^/# /;' # 756 JACYVU # 1 NC_ # implies a rule: '[JN][AC][CUYV0-9_]+(\.[0-9_]+)?' # verify this rule will find them all and eliminate them all: hgsql -N -e "select frag from gold;" rn7 | wc -l # 757 hgsql -N -e "select frag from gold;" rn7 \ | egrep -e '[JN][AC][CUYV0-9_]+(\.[0-9_]+)?' | wc -l # 757 hgsql -N -e "select frag from gold;" rn7 \ | egrep -v -e '[JN][AC][CUYV0-9_]+(\.[0-9_]+)?' | wc -l # 0 # hence, add to trackDb/rat/rn7/trackDb.ra searchTable gold shortCircuit 1 termRegex [JN][AC][CUYV0-9_]+(\.[0-9_]+)? query select chrom,chromStart,chromEnd,frag from %s where frag like '%s%%' searchPriority 8 # verify searches work in the position box git commit -m 'adding search rule for gold/assembly track refs #26682' \ trackDb.ra ########################################################################## # running repeat masker (DONE - 2020-02-03 - Hiram) # using new repeat masker version 4.1.0 mkdir /hive/data/genomes/rn7/bed/repeatMasker cd /hive/data/genomes/rn7/bed/repeatMasker time (doRepeatMasker.pl -buildDir=`pwd` \ -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ -smallClusterHub=hgwdev rn7) > do.log 2>&1 # real 397m24.001s cat faSize.rmsk.txt # 2647915728 bases (21334956 N's 2626580772 real 1457026804 upper # 1169553968 lower) in 176 sequences in 1 files # Total size: mean 15044975.7 sd 44833491.7 min 746 (chrUn_NW_023637828v1) # max 260522016 (chr1) median 44754 # %44.17 masked total, %44.53 masked real egrep -i "versi|relea" do.log # RepeatMasker,v 1.332 2017/04/17 19:01:11 rhubley Exp $ # CC Dfam_Consensus RELEASE 20181026; * # CC RepBase RELEASE 20181026; sed -e 's/^/# /;' versionInfo.txt # The repeat files provided for this assembly were generated using RepeatMasker. # Smit, AFA, Hubley, R & Green, P., # RepeatMasker Open-4.0. # 1996-2010 . # # VERSION: # RepeatMasker version development-$Id: RepeatMasker,v 1.332 2017/04/17 19:01:11 rhubley Exp $ # Search Engine: Crossmatch [ 1.090518 ] # Master RepeatMasker Database: /hive/data/staging/data/RepeatMasker181121/Libraries/RepeatMaskerLib.embl ( Complete Database: dc20181026-rb20181026 ) # # # RepeatMasker version development-$Id: RepeatMasker,v 1.332 2017/04/17 19:01:11 rhubley Exp $ # CC Dfam_Consensus RELEASE 20181026; * # CC RepBase RELEASE 20181026; * # # RepeatMasker engine: -engine crossmatch -s # # RepeatMasker library options: -species 'Rattus norvegicus' # # PARAMETERS: # /hive/data/staging/data/RepeatMasker/RepeatMasker -engine crossmatch -s -align -species 'Rattus norvegicus' time featureBits -countGaps rn7 rmsk # 1169555321 bases of 2647915728 (44.169%) in intersection # real 0m36.935s # why is it different than the faSize above ? # because rmsk masks out some N's as well as bases, the faSize count above # separates out the N's from the bases, it doesn't show lower case N's # faster way to get the same result on high contig count assemblies: time hgsql -N -e 'select genoName,genoStart,genoEnd from rmsk;' rn7 \ | bedSingleCover.pl stdin | ave -col=4 stdin | grep "^total" # total 1169555321.000000 # real 0m23.489s ########################################################################## # running simple repeat (DONE - 2020-02-03 - Hiram) mkdir /hive/data/genomes/rn7/bed/simpleRepeat cd /hive/data/genomes/rn7/bed/simpleRepeat time (doSimpleRepeat.pl -buildDir=`pwd` -bigClusterHub=ku \ -dbHost=hgwdev -workhorse=hgwdev -smallClusterHub=ku \ -trf409=6 rn7) > do.log 2>&1 & # real 288m35.826s cat fb.simpleRepeat # 94485470 bases of 2626580772 (3.597%) in intersection cd /hive/data/genomes/rn7 # if using the Window Masker result: cd /hive/data/genomes/rn7 # twoBitMask bed/windowMasker/rn7.cleanWMSdust.2bit \ # -add bed/simpleRepeat/trfMask.bed rn7.2bit # you can safely ignore the warning about fields >= 13 # add to rmsk after it is done: twoBitMask rn7.rmsk.2bit \ -add bed/simpleRepeat/trfMask.bed rn7.2bit # you can safely ignore the warning about fields >= 13 twoBitToFa rn7.2bit stdout | faSize stdin > faSize.rn7.2bit.txt cat faSize.rn7.2bit.txt # 2647915728 bases (21334956 N's 2626580772 real 1452560391 upper # 1174020381 lower) in 176 sequences in 1 files # Total size: mean 15044975.7 sd 44833491.7 min 746 (chrUn_NW_023637828v1) # max 260522016 (chr1) median 44754 # %44.34 masked total, %44.70 masked real # reset symlink rm /gbdb/rn7/rn7.2bit ln -s `pwd`/rn7.2bit /gbdb/rn7/rn7.2bit ######################################################################### # CREATE MICROSAT TRACK (DONE - 2021-02-03 - Hiram) ssh hgwdev mkdir /hive/data/genomes/rn7/bed/microsat cd /hive/data/genomes/rn7/bed/microsat awk '($5==2 || $5==3) && $6 >= 15 && $8 == 100 && $9 == 0 {printf("%s\t%s\t%s\t%dx%s\n", $1, $2, $3, $6, $16);}' \ ../simpleRepeat/simpleRepeat.bed > microsat.bed hgLoadBed rn7 microsat microsat.bed # Read 244331 elements of size 4 from microsat.bed ########################################################################## ## WINDOWMASKER (DONE - 2021-02-03 - Hiram) mkdir /hive/data/genomes/rn7/bed/windowMasker cd /hive/data/genomes/rn7/bed/windowMasker time (doWindowMasker.pl -buildDir=`pwd` -workhorse=hgwdev \ -dbHost=hgwdev rn7) > do.log 2>&1 # real 103m59.019s # Masking statistics cat faSize.rn7.cleanWMSdust.txt # 2647915728 bases (21334956 N's 2626580772 real 1677807978 upper # 948772794 lower) in 176 sequences in 1 files # Total size: mean 15044975.7 sd 44833491.7 min 746 (chrUn_NW_023637828v1) # max 260522016 (chr1) median 44754 # %35.83 masked total, %36.12 masked real ########################################################################## # cpgIslands - (DONE - 2021-02-04 - Hiram) mkdir /hive/data/genomes/rn7/bed/cpgIslands cd /hive/data/genomes/rn7/bed/cpgIslands time (doCpgIslands.pl -dbHost=hgwdev -bigClusterHub=ku \ -workhorse=hgwdev -smallClusterHub=ku rn7) > do.log 2>&1 # real 3m40.227s sed -e 's/^/ # /;' fb.rn7.cpgIslandExt.txt # 10397605 bases of 2626580772 (0.396%) in intersection ############################################################################## # genscan - (DONE - 2021-02-04 - 02-15 - Hiram) mkdir /hive/data/genomes/rn7/bed/genscan cd /hive/data/genomes/rn7/bed/genscan time (doGenscan.pl -buildDir=`pwd` -workhorse=hgwdev -dbHost=hgwdev \ -bigClusterHub=ku rn7) > do.log 2>&1 # real 73m39.179s # two jobs broken: ./runGsBig2M.csh chr2 000 gtf/000/chr2.gtf pep/000/chr2.pep subopt/000/chr2.bed & ./runGsBig2M.csh chr11 000 gtf/000/chr11.gtf pep/000/chr11.pep subopt/000/chr11.bed wait # real 68m25.959s # continuing time (doGenscan.pl -buildDir=`pwd` -workhorse=hgwdev -dbHost=hgwdev \ -continue=makeBed -bigClusterHub=ku rn7) > makeBed.log 2>&1 # real 0m43.231s sed -e 's/^/ # /;' fb.rn7.genscan.txt # 55066515 bases of 2626580772 (2.097%) in intersection sed -e 's/^/ # /;' fb.rn7.genscanSubopt.txt # 57355860 bases of 2626580772 (2.184%) in intersection ######################################################################### # ncbiGene (TBD - 2020-09-03 - Hiram) # don't need to do this on GCF/RefSeq assemblies, they have RefSeq genes mkdir /hive/data/genomes/rn7/bed/xenoRefGene cd /hive/data/genomes/rn7/bed/xenoRefGene time (~/kent/src/hg/utils/automation/doXenoRefGene.pl -buildDir=`pwd` \ -bigClusterHub=ku -workhorse=hgwdev -dbHost=hgwdev rn7) > do.log 2>&1 & # real 67m18.015s ######################################################################### # Create kluster run files (DONE - 2021-02-04 - Hiram) # numerator is rn7 gapless bases "real" as reported by: featureBits -noRandom -noHap rn7 gap # 19396258 bases of 2614093470 (0.742%) in intersection # ^^^ # denominator is hg19 gapless bases as reported by: # featureBits -noRandom -noHap hg19 gap # 234344806 bases of 2861349177 (8.190%) in intersection # 1024 is threshold used for human -repMatch: calc \( 2614093470 / 2861349177 \) \* 1024 # ( 2614093470 / 2861349177 ) * 1024 = 935.513825 # ==> use -repMatch=1000 according to size scaled down from 1024 for human. cd /hive/data/genomes/rn7 time blat rn7.2bit \ /dev/null /dev/null -tileSize=11 -makeOoc=jkStuff/rn7.11.ooc \ -repMatch=1000 # Wrote 27322 overused 11-mers to jkStuff/rn7.11.ooc # real 0m26.195s # rn6 at repMatch=1000 # Wrote 27021 overused 11-mers to jkStuff/rn6.11.ooc # survey sizes of all gaps: hgsql -N -e 'select size from gap;' rn7 | ave stdin | sed -e 's/^/# /;' # Q1 100.000000 # median 200.000000 # Q3 22550.000000 # average 36721.094664 # min 10.000000 # max 660122.000000 # count 581 # total 21334956.000000 # standard deviation 95181.017236 hgsql -N -e 'select size from gap;' rn7 | sort -n | uniq -c | sed -e 's/^/# /;' # There are no non-bridged gaps on this genome # survey sizes of non-bridged gaps: hgsql -N -e 'select size from gap where bridge="no" order by size;' \ rn7 | sort | uniq -c | sort -k2,2n | sed -e 's/^/# /;' # 7 100 # 8 50000 # 22 1000000 # and survey the number bridged gaps over 50,000 bases: hgsql -N -e 'select size from gap where bridge="yes" and size > 49999;' \ rn7 | wc -l # 92 # forget the non-bridged of size 100, use 50,000 and allow bridged # use gap size of 50000 to construct a lift file: gapToLift -allowBridged -verbose=2 -minGap=50000 rn7 \ jkStuff/rn7.gaps.lft -bedFile=jkStuff/rn7.gaps.bed wc -l jkStuff/rn7.gaps* # 268 jkStuff/rn7.gaps.bed # 268 jkStuff/rn7.gaps.lft # to see the gaps sizes used: bedInvert.pl chrom.sizes jkStuff/rn7.gaps.bed \ | cut -f4 | sort -n | uniq -c | less ############################################################################## # lastz/chain/net swap human/hg38 (DONE - 2021-02-04 - Hiram) # original alignment cd /hive/data/genomes/hg38/bed/lastzRn7.2021-02-04 sed -e 's/^/ # /;' fb.hg38.chainRn7Link.txt # 958592205 bases of 3110768607 (30.815%) in intersection sed -e 's/^/ # /;' fb.hg38.chainSynRn7Link.txt # 904066852 bases of 3110768607 (29.062%) in intersection time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ hg38 rn7) > rbest.log 2>&1 & # real 313m26.149s sed -e 's/^/ # /;' fb.hg38.chainRBest.Rn7.txt # 883775977 bases of 3110768607 (28.410%) in intersection # and the swap mkdir /hive/data/genomes/rn7/bed/blastz.hg38.swap cd /hive/data/genomes/rn7/bed/blastz.hg38.swap time (doBlastzChainNet.pl -verbose=2 \ /hive/data/genomes/hg38/bed/lastzRn7.2021-02-04/DEF \ -chainMinScore=3000 -chainLinearGap=medium \ -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ -swap -syntenicNet) > swap.log 2>&1 # real 74m20.215s sed -e 's/^/ # /;' fb.rn7.chainHg38Link.txt # 928866703 bases of 2626580772 (35.364%) in intersection sed -e 's/^/ # /;' fb.rn7.chainSynHg38Link.txt # 879484562 bases of 2626580772 (33.484%) in intersection time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ rn7 hg38) > rbest.log 2>&1 & # real 373m6.610s sed -e 's/^/ # /;' fb.rn7.chainRBest.Hg38.txt # 885516265 bases of 2626580772 (33.714%) in intersection ############################################################################## # lastz/chain/net swap mouse/mm39 (DONE - 2021-02-04 - Hiram) # original alignment cd /hive/data/genomes/mm39/bed/lastzRn7.2021-02-04 sed -e 's/^/ # /;' fb.mm39.chainRn7Link.txt # 1898735724 bases of 2654624157 (71.526%) in intersection sed -e 's/^/ # /;' fb.mm39.chainSynRn7Link.txt # 1787593557 bases of 2654624157 (67.339%) in intersection time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ mm39 rn7) > rbest.log 2>&1 & # real 588m38.122s sed -e 's/^/ # /;' fb.mm39.chainRBest.Rn7.txt # 1754204799 bases of 2654624157 (66.081%) in intersection mkdir /hive/data/genomes/rn7/bed/blastz.mm39.swap cd /hive/data/genomes/rn7/bed/blastz.mm39.swap time (doBlastzChainNet.pl -verbose=2 \ /hive/data/genomes/mm39/bed/lastzRn7.2021-02-04/DEF \ -swap -syntenicNet \ -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ -chainMinScore=5000 -chainLinearGap=medium) > swap.log 2>&1 # real 116m47.862s sed -e 's/^/ # /;' fb.rn7.chainMm39Link.txt # 1855165978 bases of 2626580772 (70.630%) in intersection sed -e 's/^/ # /;' fb.rn7.chainSynMm39Link.txt # 1763550257 bases of 2626580772 (67.142%) in intersection time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ rn7 mm39) > rbest.log 2>&1 # real 605m42.354s sed -e 's/^/ # /;' fb.rn7.chainRBest.Mm39.txt # 1754416686 bases of 2626580772 (66.795%) in intersection ############################################################################## # lastz/chain/net swap mouse/mm10 (DONE - 2021-02-15 - Hiram) # original alignment cd /hive/data/genomes/mm10/bed/lastzRn7.2021-02-15 sed -e 's/^/ # /;' fb.mm10.chainRn7Link.txt # 1896928045 bases of 2652783500 (71.507%) in intersection sed -e 's/^/ # /;' fb.mm10.chainSynRn7Link.txt # 1787142074 bases of 2652783500 (67.369%) in intersection time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ mm10 rn7) > rbest.log 2>&1 & # real 578m13.711s sed -e 's/^/ # /;' fb.mm10.chainRBest.Rn7.txt # 1753198266 bases of 2652783500 (66.089%) in intersection mkdir /hive/data/genomes/rn7/bed/blastz.mm10.swap cd /hive/data/genomes/rn7/bed/blastz.mm10.swap time (doBlastzChainNet.pl -verbose=2 \ /hive/data/genomes/mm10/bed/lastzRn7.2021-02-15/DEF \ -swap -syntenicNet \ -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ -chainMinScore=5000 -chainLinearGap=medium) > swap.log 2>&1 # real 112m36.899s sed -e 's/^/ # /;' fb.rn7.chainMm10Link.txt # 1853300495 bases of 2626580772 (70.559%) in intersection sed -e 's/^/ # /;' fb.rn7.chainSynMm10Link.txt # 1762899567 bases of 2626580772 (67.118%) in intersection time (doRecipBest.pl -load -workhorse=hgwdev -buildDir=`pwd` \ rn7 mm10) > rbest.log 2>&1 # real 599m24.766s sed -e 's/^/ # /;' fb.rn7.chainRBest.Mm10.txt # 1753558422 bases of 2626580772 (66.762%) in intersection ############################################################################## # GENBANK AUTO UPDATE (DONE - 2021-02-04 - Hiram) ssh hgwdev cd $HOME/kent/src/hg/makeDb/genbank git pull # /cluster/data/genbank/data/organism.lst shows: # organism mrnaCnt estCnt refSeqCnt # Rattus norvegicus 130246 1103640 17794 # edit etc/genbank.conf to add rn7 just before rn6 # rn7 (rat GCF_015227675.2 mRatBN7.2) rn7.serverGenome = /hive/data/genomes/rn7/rn7.2bit rn7.ooc = /hive/data/genomes/rn7/jkStuff/rn7.11.ooc rn7.lift = /hive/data/genomes/rn7/jkStuff/rn7.gaps.lft rn7.refseq.mrna.native.pslCDnaFilter = ${finished.refseq.mrna.native.pslCDnaFilter} rn7.refseq.mrna.xeno.pslCDnaFilter = ${finished.refseq.mrna.xeno.pslCDnaFilter} rn7.genbank.mrna.native.pslCDnaFilter = ${finished.genbank.mrna.native.pslCDnaFilter} rn7.genbank.mrna.xeno.pslCDnaFilter = ${finished.genbank.mrna.xeno.pslCDnaFilter} rn7.genbank.est.native.pslCDnaFilter = ${finished.genbank.est.native.pslCDnaFilter} rn7.downloadDir = rn7 rn7.refseq.mrna.xeno.load = yes rn7.refseq.mrna.xeno.loadDesc = yes rn7.genbank.mrna.xeno.load = yes rn7.perChromTables = no rn7.mgc = yes # rn7.upstreamGeneTbl = ensGene # rn7.upstreamMaf = multiz13way /hive/data/genomes/rn7/bed/multiz13way/species.list.txt # verify the files specified exist before checking in the file: grep ^rn7 etc/genbank.conf | grep hive | awk '{print $NF}' | xargs ls -og # -rw-rw-r-- 1 109296 Feb 4 09:28 /hive/data/genomes/rn7/jkStuff/rn7.11.ooc # -rw-rw-r-- 1 13791 Feb 4 09:34 /hive/data/genomes/rn7/jkStuff/rn7.gaps.lft # -rw-rw-r-- 1 692151553 Feb 4 09:20 /hive/data/genomes/rn7/rn7.2bit git commit -m "Added rn7 rat refs #26682" etc/genbank.conf git push # update /cluster/data/genbank/: make etc-update # enable daily alignment and update of hgwdev cd ~/kent/src/hg/makeDb/genbank git pull # add rn7 to: # etc/hgwdev.dbs etc/align.dbs git commit -m "Added rn7 - rat refs #26682" etc/hgwdev.dbs etc/align.dbs git push make etc-update # wait a few days for genbank magic to take place, the tracks will # appear ############################################################################# # augustus gene track (DONE - 2021-02-04 - Hiram) mkdir /hive/data/genomes/rn7/bed/augustus cd /hive/data/genomes/rn7/bed/augustus time (doAugustus.pl -buildDir=`pwd` -bigClusterHub=ku \ -species=human -dbHost=hgwdev \ -workhorse=hgwdev rn7) > do.log 2>&1 # real 119m10.900s cat fb.rn7.augustusGene.txt # 49646030 bases of 2626580772 (1.890%) in intersection ######################################################################### # ncbiRefSeq (DONE - 2021-02-04 - Hiram) mkdir /hive/data/genomes/rn7/bed/ncbiRefSeq cd /hive/data/genomes/rn7/bed/ncbiRefSeq time (~/kent/src/hg/utils/automation/doNcbiRefSeq.pl -buildDir=`pwd` \ -bigClusterHub=ku -dbHost=hgwdev \ -fileServer=hgwdev -smallClusterHub=hgwdev -workhorse=hgwdev \ GCF_015227675.2_mRatBN7.2 rn7) > do.log 2>&1 & # real 6m52.120s cat fb.ncbiRefSeq.rn7.txt # 107770866 bases of 2626580772 (4.103%) in intersection # add: include ../../refSeqComposite.ra # to the rat/rn7/trackDb.ra to turn on the track in the browser joinerCheck says: rn7.ncbiRefSeqLink.protAcc - hits 74754 of 74755 (99.999%) Error: 1 of 74755 elements (0.001%) of rn7.ncbiRefSeqLink.protAcc are not in key ncbiRefSeqPepTable.name line 8640 of all.joiner Example miss: NP_536324.1 # for some reason one of the proteins is missing from # GCF_015227675.2_mRatBN7.2_protein.faa.gz # however, it is in the GCF_015227675.2_mRatBN7.2_rna.gbff.gz file # obtain it from that file, and create download/NP_536324.1.faa.gz # then, add it to the table reload: export db=rn7 export asmId=GCF_015227675.2_mRatBN7.2 zcat download/${asmId}_protein.faa.gz download/NP_536324.1.faa.gz \ | sed -e 's/ .*//;' | faToTab -type=protein -keepAccSuffix stdin stdout \ | sort | join -t$'\t' $db.ncbiRefSeqLink.protAcc.list - \ > fixed.$db.ncbiRefSeqPepTable.tab hgLoadSqlTab $db ncbiRefSeqPepTable ~/kent/src/hg/lib/pepPred.sql \ fixed.$db.ncbiRefSeqPepTable.tab # Now both of these checks are quiet: joinerCheck -keys \ -identifier=ncbiRefSeqPepTable -database=$db \ /cluster/home/hiram/kent/src/hg/makeDb/schema/all.joiner Checking keys on database rn7 rn7.ncbiRefSeqLink.protAcc - hits 74755 of 74755 (100.000%) ok joinerCheck -keys \ -identifier=ncbiRefSeq -database=$db \ /cluster/home/hiram/kent/src/hg/makeDb/schema/all.joiner Checking keys on database rn7 rn7.ncbiRefSeqLink.id - hits 99139 of 99139 (100.000%) ok rn7.ncbiRefSeqCurated.name - hits 18403 of 18403 (100.000%) ok rn7.ncbiRefSeqPredicted.name - hits 80736 of 80736 (100.000%) ok rn7.ncbiRefSeqPsl.qName - hits 99145 of 99145 (100.000%) ok rn7.ncbiRefSeqCds.id - hits 74741 of 74741 (100.000%) ok rn7.seqNcbiRefSeq.acc - hits 99139 of 99139 (100.000%) ok featureBits -enrichment rn7 refGene ncbiRefSeq # refGene 1.522%, ncbiRefSeq 4.103%, both 1.520%, cover 99.87%, enrich 24.34x featureBits -enrichment rn7 ncbiRefSeq refGene # ncbiRefSeq 4.103%, refGene 1.522%, both 1.520%, cover 37.05%, enrich 24.34x featureBits -enrichment rn7 ncbiRefSeqCurated refGene # ncbiRefSeqCurated 1.529%, refGene 1.522%, both 1.514%, cover 99.03%, enrich 65.07x featureBits -enrichment rn7 refGene ncbiRefSeqCurated # refGene 1.522%, ncbiRefSeqCurated 1.529%, both 1.514%, cover 99.49%, enrich 65.07x ############################################################################## # LIFTOVER TO rn6 (DONE - 2021-02-04 - Hiram) ssh hgwdev mkdir /hive/data/genomes/rn7/bed/blat.rn6.2021-02-04 cd /hive/data/genomes/rn7/bed/blat.rn6.2021-02-04 doSameSpeciesLiftOver.pl -verbose=2 \ -debug -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ -query2Bit=/hive/data/genomes/rn6/rn6.2bit \ -querySizes=/hive/data/genomes/rn6/chrom.sizes \ -ooc=/hive/data/genomes/rn7/jkStuff/rn7.11.ooc \ rn7 rn6 time (doSameSpeciesLiftOver.pl -verbose=2 \ -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ -query2Bit=/hive/data/genomes/rn6/rn6.2bit \ -querySizes=/hive/data/genomes/rn6/chrom.sizes \ -ooc=/hive/data/genomes/rn7/jkStuff/rn7.11.ooc \ rn7 rn6) > doLiftOverToRn7.log 2>&1 # real 248m51.413s # see if the liftOver menus function in the browser from rn7 to rn6 ############################################################################## # BLATSERVERS ENTRY (DONE - 2021-02-04 - Hiram) # After getting a blat server assigned by the Blat Server Gods, ssh hgwdev hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \ VALUES ("rn7", "blat1b", "17910", "1", "0"); \ INSERT INTO blatServers (db, host, port, isTrans, canPcr) \ VALUES ("rn7", "blat1b", "17911", "0", "1");' \ hgcentraltest # test it with some sequence ############################################################################## ## reset default position to same as rn6 via blat of the DNA from rn6 ## (DONE - 2021-02-04 - Hiram) ssh hgwdev hgsql -e 'update dbDb set defaultPos="chr1:79348972-79379997" where name="rn7";' hgcentraltest ############################################################################## # crispr whole genome (DONE - 2021-02-04 -> 2020-02-11 - Hiram) mkdir /hive/data/genomes/rn7/bed/crisprAll cd /hive/data/genomes/rn7/bed/crisprAll # need to have augustus genes done. This will not work with genscan # the large shoulder argument will cause the entire genome to be scanned # this takes a while for a new genome to get the bwa indexing done time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 -stop=ranges \ rn7 augustusGene -shoulder=250000000 -tableName=crisprAll \ -fileServer=hgwdev \ -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ -workhorse=hgwdev) >> ranges.log 2>&1 # real 64m40.351s time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 \ -continue=guides -stop=load rn7 augustusGene \ -shoulder=250000000 -tableName=crisprAll -fileServer=hgwdev \ -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ -workhorse=hgwdev) > specScores.log 2>&1 # real 7968m4.344s # broken specScores step due to power failure ku reset # first part of specScores # Completed: 925541 of 3066093 jobs # Crashed: 503 jobs # Other count: 1999108 jobs # CPU time in finished jobs: 84291533s 1404858.88m 23414.31h 975.60d 2.673 y # IO & Wait Time: 2560453s 42674.22m 711.24h 29.63d 0.081 y # Average job time: 94s 1.56m 0.03h 0.00d # Longest finished job: 709s 11.82m 0.20h 0.01d # Submission to last job: 87660s 1461.00m 24.35h 1.01d # second part of specScores # Completed: 2140552 of 2140552 jobs # CPU time in finished jobs: 190374829s 3172913.81m 52881.90h 2203.41d 6.037 y # IO & Wait Time: 4108443s 68474.05m 1141.23h 47.55d 0.130 y # Average job time: 91s 1.51m 0.03h 0.00d # Longest finished job: 193s 3.22m 0.05h 0.00d # Submission to last job: 208681s 3478.02m 57.97h 2.42d # and putting together the results time find tmp/outGuides -type f | xargs cut -f3-6 > ../specScores.tab # real 806m23.857s # user 4m0.906s # sys 13m39.775s # continuing with effScores: time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 \ -continue=effScores -stop=load rn7 augustusGene \ -shoulder=250000000 -tableName=crisprAll -fileServer=hgwdev \ -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ -workhorse=hgwdev) > effScores.log 2>&1 # real 1964m18.261s cat guides/run.time | sed -e 's/^/# /;' # Completed: 100 of 100 jobs # CPU time in finished jobs: 13041s 217.34m 3.62h 0.15d 0.000 y # IO & Wait Time: 340s 5.67m 0.09h 0.00d 0.000 y # Average job time: 134s 2.23m 0.04h 0.00d # Longest finished job: 512s 8.53m 0.14h 0.01d # Submission to last job: 513s 8.55m 0.14h 0.01d cat specScores/run.time | sed -e 's/^/# /;' # Completed: 2140552 of 2140552 jobs # CPU time in finished jobs: 190374829s 3172913.81m 52881.90h 2203.41d 6.037 y # IO & Wait Time: 4108443s 68474.05m 1141.23h 47.55d 0.130 y # Average job time: 91s 1.51m 0.03h 0.00d # Longest finished job: 193s 3.22m 0.05h 0.00d # Submission to last job: 208681s 3478.02m 57.97h 2.42d grep "Number of specScores" specScores.log # Number of specScores: 229203141 cat effScores/run.time | sed -e 's/^/# /;' # Completed: 27591 of 27591 jobs # CPU time in finished jobs: 13780234s 229670.56m 3827.84h 159.49d 0.437 y # IO & Wait Time: 93812s 1563.54m 26.06h 1.09d 0.003 y # Average job time: 503s 8.38m 0.14h 0.01d # Longest finished job: 52764s 879.40m 14.66h 0.61d # Submission to last job: 70493s 1174.88m 19.58h 0.82d cat offTargets/run.time | sed -e 's/^/# /;' # Completed: 153305 of 153305 jobs # CPU time in finished jobs: 2423705s 40395.08m 673.25h 28.05d 0.077 y # IO & Wait Time: 1680214s 28003.57m 466.73h 19.45d 0.053 y # Average job time: 27s 0.45m 0.01h 0.00d # Longest finished job: 60s 1.00m 0.02h 0.00d # Submission to last job: 5880s 98.00m 1.63h 0.07d bigBedInfo crispr.bb | sed -e 's/^/# /;' # version: 4 # fieldCount: 22 # hasHeaderExtension: yes # isCompressed: yes # isSwapped: 0 # extraIndexCount: 0 # itemCount: 274,741,534 # primaryDataSize: 12,314,717,925 # primaryIndexSize: 17,246,852 # zoomLevels: 10 # chromCount: 31 # basesCovered: 2,157,342,037 # meanDepth (of bases covered): 2.929093 # minDepth: 1.000000 # maxDepth: 33.000000 # std of depth: 1.944611 + # hive cleaning 2021-04-25 - Hiram + time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 \ + -continue=cleanup rn7 -tableName=crisprAll -fileServer=hgwdev \ + -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ + -workhorse=hgwdev) > cleanup.log 2>&1 + # real 221m15.143s + ######################################################################### # all.joiner update, downloads and in pushQ - (DONE - 2021-03-31 - Hiram) # had incorrect orderKey specified in beginning rn6.config.ra # correct to 18019 by looking at the output of: # hgsql -e 'select name,organism,orderKey from dbDb order by orderKey;' \ # hgcentraltest | less # oryCun1 Rabbit 18010 # regenRn1 Rat 18020 # regenRn0 Rat 18021 # rn6 Rat 18031 # rn7 Rat 18032 # rn5 Rat 18032 # rn4 Rat 18033 # rn3 Rat 18034 # rn2 Rat 18035 # tauEry1 Red crested turaco 18360 hgsql -e 'update dbDb set orderKey=18019 where name="rn7";' hgcentraltest cd $HOME/kent/src/hg/makeDb/schema # verify all the business is done for release ~/kent/src/hg/utils/automation/verifyBrowser.pl rn7 # 79 tables in database rn7 - Rat, Rattus norvegicus # verified 77 tables in database rn7, 2 extra tables, 30 optional tables # NCBI RefSeq genes 10 optional tables # chainNetRBestHg38 3 optional tables # chainNetRBestMm10 3 optional tables # chainNetRBestMm39 3 optional tables # chainNetSynHg38 3 optional tables # chainNetSynMm10 3 optional tables # chainNetSynMm39 3 optional tables # gapOverlap 1 optional tables # tandemDups 1 optional tables # 1 crisprAllRanges - extra table # 2 crisprAllTargets - extra table # 15 genbank tables found # verified 32 required tables, 0 missing tables # hg38 chainNet to rn7 found 3 required tables # mm10 chainNet to rn7 found 3 required tables # mm39 chainNet to rn7 found 3 required tables # hg38 chainNet RBest and syntenic to rn7 found 6 optional tables # mm10 chainNet RBest and syntenic to rn7 found 6 optional tables # mm39 chainNet RBest and syntenic to rn7 found 6 optional tables # liftOver to previous versions: 1, from previous versions: 1 # blatServers: rn7 blat1b 17910 1 0 0 rn7 blat1b 17911 0 1 0 # fixup all.joiner until this is a clean output joinerCheck -database=rn7 -tableCoverage all.joiner joinerCheck -database=rn7 -times all.joiner joinerCheck -database=rn7 -keys all.joiner # when clean, check in: git commit -m 'adding rules for rn7 refs #26682' all.joiner git push # run up a 'make alpha' in hg/hgTables to get this all.joiner file # into the hgwdev/genome-test system cd /hive/data/genomes/rn7 time (makeDownloads.pl rn7) > downloads.log 2>&1 # real 19m6.278s # now ready for pushQ entry mkdir /hive/data/genomes/rn7/pushQ cd /hive/data/genomes/rn7/pushQ time ($HOME/kent/src/hg/utils/automation/makePushQSql.pl -redmineList rn7) > rn7.pushQ.sql 2> stderr.out # real 18m30.674s # remove the tandemDups and gapOverlap from the file list: sed -i -e "/tandemDups/d" redmine.rn7.table.list sed -i -e "/Tandem Dups/d" redmine.rn7.releaseLog.txt sed -i -e "/gapOverlap/d" redmine.rn7.table.list sed -i -e "/Gap Overlaps/d" redmine.rn7.releaseLog.txt # edit the file list and expand the directory wildcards, to find them: grep "\*\/" redmine.rn7.file.list # there were two, expand them via 'ls' # check for errors in stderr.out, some are OK, e.g.: WARNING: rn7 does not have seq WARNING: rn7 does not have extFile WARNING: Could not tell (from trackDb, all.joiner and hardcoded lists of supporting and genbank tables) which tracks to assign these tables to: chainRBestHg38 chainRBestHg38Link ... etc crisprAllRanges gbLoaded netRBestHg38 netRBestMm10 netRBestMm39 netSynHg38 netSynMm10 netSynMm39 # verify the file list does correctly match to files cat redmine.rn7.file.list | while read L do eval ls $L > /dev/null done # should be silent, missing files will show as errors # verify database tables, how many to expect: wc -l redmine.rn7.table.list # 65 redmine.rn7.table.list # how many actual: awk -F'.' '{printf "hgsql -N %s -e '"'"'show table status like \"%s\";'"'"'\n", $1, $2}' redmine.rn7.table.list | sh | wc -l # 65 # would be a smaller number actual if some were missing # add the path names to the listing files in the redmine issue # in the three appropriate entry boxes: # /hive/data/genomes/rn7/pushQ/redmine.rn7.file.list # /hive/data/genomes/rn7/pushQ/redmine.rn7.releaseLog.txt # /hive/data/genomes/rn7/pushQ/redmine.rn7.table.list #########################################################################