9e1d18bbaeabc8922c4aba1baa71102a436d1cc2 hiram Tue Apr 27 14:12:08 2021 -0700 hive cleaning running cleanup step for crispr tracks no redmine diff --git src/hg/makeDb/doc/enhLutNer1/initialBuild.txt src/hg/makeDb/doc/enhLutNer1/initialBuild.txt index ce9d39b..de601a0 100644 --- src/hg/makeDb/doc/enhLutNer1/initialBuild.txt +++ src/hg/makeDb/doc/enhLutNer1/initialBuild.txt @@ -1,866 +1,872 @@ # for emacs: -*- mode: sh; -*- # This file describes browser build for the enhLutNer1 assembly (Enhydra Lutris Nereis) ######################################################################### # Initial steps, photograph provided by the Monterey Bay Aquarium (DONE - 2019-10-09 - Jonathan) # To start this initialBuild.txt document, from a previous assembly document: mkdir ~/kent/src/hg/makeDb/doc/enhLutNer1 cd ~/kent/src/hg/makeDb/doc/enhLutNer1 sed -e 's/rheMac10/enhLutNer1/g; s/RheMac10/EnhLutNer1/g; s/TBD/TBD/g;' \ ../rheMac10/initialBuild.txt > initialBuild.txt mkdir -p /hive/data/genomes/enhLutNer1/genbank cd /hive/data/genomes/enhLutNer1 # Photo provided by the Monterey Bay Aquarium (in pdf, converted to png) printf "photoCreditURL https://www.montereybayaquarium.org/ photoCreditName Courtesy of the Monterey Bay Aquarium " > photoReference.txt ## download from NCBI cd /hive/data/genomes/enhLutNer1/genbank time rsync -L -a -P --stats \ rsync://ftp.ncbi.nlm.nih.gov/genomes/genbank/vertebrate_mammalian/Enhydra_lutris/all_assembly_versions/GCA_006410715.1_ASM641071v1/ ./ # sent 240 bytes received 1,945,221,740 bytes 54,794,985.35 bytes/sec # total size is 1,944,746,108 speedup is 1.00 # real 0m34.708s # this information is from the top of # enhLutNer1/genbank/*_assembly_report.txt # (aka: enhLutNer1/genbank/GCA_006410715.1_ASM641071v1_assembly_report.txt # Assembly name: ASM641071v1 # Organism name: Enhydra lutris nereis (sea otter) # Isolate: Gidget # Sex: female # Taxid: 1049777 # BioSample: SAMN09238016 # BioProject: PRJNA472597 # Submitter: UCLA # Date: 2019-06-24 # Assembly type: haploid # Release type: major # Assembly level: Scaffold # Genome representation: full # WGS project: QQQE01 # Assembly method: Meraculous + Dovetail HiRise v. MAY-2016 # Expected final version: yes # Genome coverage: 80.0x # Sequencing technology: Illumina HiSeq # GenBank assembly accession: GCA_006410715.1 # ## Assembly-Units: ## GenBank Unit Accession RefSeq Unit Accession Assembly-Unit name ## GCA_006410725.1 Primary Assembly # # check assembly size for later reference: faSize G*v1_genomic.fna.gz # 2425597608 bases (11943786 N's 2413653822 real 1675884180 upper # 737769642 lower) in 55496 sequences in 1 files # Total size: mean 43707.6 sd 750707.1 min 1000 (QQQE01055496.1) # max 56733013 (QQQE01000071.1) median 2458 # %30.42 masked total, %30.57 masked real ############################################################################# # fixup to UCSC naming scheme (DONE - 2019-10-11 - Jonathan) # This step copied/modified separately from ../ambMex1/initialBuild.txt mkdir /hive/data/genomes/enhLutNer1/ucsc cd /hive/data/genomes/enhLutNer1/ucsc # verify no duplicate sequences: note the use of the -long argument # on this gigantic amount of sequence time faToTwoBit ../genbank/*1_genomic.fna.gz genbank.2bit # real 0m47.443s time twoBitDup genbank.2bit # real 0m12.307s # should be silent output, otherwise the duplicates need to be removed # since this is an unplaced contig assembly, verify all names are .1: twoBitInfo genbank.2bit stdout | awk '{print $1}' \ | sed -e 's/[0-9]\+//;' | sort | uniq -c # 55496 QQQE.1 # in this case, all the .1's can be changed to: v1 time zcat ../genbank/GCA_006410715.1_ASM641071v1_genomic.fna.gz \ | sed -e 's/.1 Enhydra.*/v1/;' | gzip -c \ > ucsc.enhLutNer1.fa.gz # real 9m37.801s # -rw-rw-r-- 1 793146354 Oct 11 14:25 ucsc.enhLutNer1.fa.gz # and there is no AGP file with the assembly, construct one: time hgFakeAgp -minContigGap=1 ucsc.enhLutNer1.fa.gz ucsc.enhLutNer1.fake.agp # real 0m28.416s # verify fasta and AGP match: time faToTwoBit ucsc.enhLutNer1.fa.gz test.2bit # real 0m53.075s # verify still silent: time twoBitDup test.2bit # real 0m8.486s # and check AGP vs. fasta correspondence: time cat *.agp | checkAgpAndFa stdin test.2bit 2>&1 | tail # All AGP and FASTA entries agree - both files are valid # real 0m5.140s # verify nothing lost compared to genbank: time twoBitToFa test.2bit stdout | faSize stdin # 2425597608 bases (11943786 N's 2413653822 real 1675884180 upper # 737769642 lower) in 55496 sequences in 1 files # Total size: mean 43707.6 sd 750707.1 min 1000 (QQQE01055496v1) # max 56733013 (QQQE01000071v1) median 2458 # %30.42 masked total, %30.57 masked real # real 0m28.771s # the original genbank count: # 2425597608 bases (11943786 N's 2413653822 real 1675884180 upper # 737769642 lower) in 55496 sequences in 1 files # no longer needed: rm -f genbank.2bit test.2bit ############################################################################# # establish config.ra file (DONE - Jonathan - 2019-10-14) cd /hive/data/genomes/enhLutNer1 ~/kent/src/hg/utils/automation/prepConfig.pl enhLutNer1 mammal otters \ genbank/*_assembly_report.txt > enhLutNer1.config.ra sed -e 's/mitoAcc notFound/mitoAcc none/' -i enhLutNer1.config.ra # verify it really does look sane cat enhLutNer1.config.ra # config parameters for makeGenomeDb.pl: db enhLutNer1 clade mammal genomeCladePriority 35 scientificName Enhydra lutris nereis commonName Sea otter assemblyDate Jun. 2019 assemblyLabel UCLA assemblyShortLabel ASM641071v1 orderKey 19259 mitoAcc none fastaFiles /hive/data/genomes/enhLutNer1/ucsc/*.fa.gz agpFiles /hive/data/genomes/enhLutNer1/ucsc/*.agp # qualFiles none dbDbSpeciesDir otters photoCreditURL https://www.montereybayaquarium.org/ photoCreditName Courtesy of the Monterey Bay Aquarium ncbiGenomeId 8272 ncbiAssemblyId 3544501 ncbiAssemblyName ASM641071v1 ncbiBioProject 472597 ncbiBioSample SAMN09238016 genBankAccessionID GCA_006410715.1 taxId 1049777 # Since we're releasing this along with the Northern sea otter assembly and # want to differentiate them, let's make the common name a bit more # detailed. sed -e 's/Sea otter/Southern sea otter/' -i enhLutNer1.config.ra ############################################################################# # Initial database build (DONE - 2019-11-15 - Jonathan) # verify sequence and AGP are OK: cd /hive/data/genomes/enhLutNer1 time (makeGenomeDb.pl -workhorse=hgwdev -dbHost=hgwdev -fileServer=hgwdev \ -stop=agp enhLutNer1.config.ra) > agp.log 2>&1 # real 2m17.995s # then finish it off: time (makeGenomeDb.pl -workhorse=hgwdev -dbHost=hgwdev \ -fileServer=hgwdev -continue=db enhLutNer1.config.ra) > db.log 2>&1 # real 14m58.961s # Log file reveals I forgot to put Enhydra_lutris_nereis.png into place. Fixed that. time (makeGenomeDb.pl -workhorse=hgwdev -dbHost=hgwdev \ -fileServer=hgwdev -continue=trackDb enhLutNer1.config.ra) >> db.log 2>&1 # real 0m7.728s # check in the trackDb files created in TemporaryTrackDbCheckout/ # and add enhLutNer1 to trackDb/makefile # temporary symlink until masked sequence is available cd /hive/data/genomes/enhLutNer1 ln -s `pwd`/enhLutNer1.unmasked.2bit /gbdb/enhLutNer1/enhLutNer1.2bit ############################################################################## # cpgIslands on UNMASKED sequence (DONE - 2019-11-15 - Jonathan) mkdir /hive/data/genomes/enhLutNer1/bed/cpgIslandsUnmasked cd /hive/data/genomes/enhLutNer1/bed/cpgIslandsUnmasked time (doCpgIslands.pl -dbHost=hgwdev -bigClusterHub=ku -buildDir=`pwd` \ -tableName=cpgIslandExtUnmasked \ -maskedSeq=/hive/data/genomes/enhLutNer1/enhLutNer1.unmasked.2bit \ -workhorse=hgwdev -smallClusterHub=ku enhLutNer1) > do.log 2>&1 # real 5m27.460s cat fb.enhLutNer1.cpgIslandExtUnmasked.txt # 45112255 bases of 2413653822 (1.869%) in intersection ############################################################################# # cytoBandIdeo - (DONE - 2019-11-15 - Jonathan) mkdir /hive/data/genomes/enhLutNer1/bed/cytoBand cd /hive/data/genomes/enhLutNer1/bed/cytoBand makeCytoBandIdeo.csh enhLutNer1 ############################################################################# # run up idKeys files for chromAlias/ncbiRefSeq (DONE - 2019-11-15 - Jonathan) mkdir /hive/data/genomes/enhLutNer1/bed/idKeys cd /hive/data/genomes/enhLutNer1/bed/idKeys time (doIdKeys.pl \ -twoBit=/hive/data/genomes/enhLutNer1/enhLutNer1.unmasked.2bit \ -buildDir=`pwd` enhLutNer1) > do.log 2>&1 & # real 20m55.006s cat enhLutNer1.keySignature.txt # 6ccb6138a2432071b6103be8b66bb46d ############################################################################# # gapOverlap (DONE - 2019-11-15 - Jonathan) mkdir /hive/data/genomes/enhLutNer1/bed/gapOverlap cd /hive/data/genomes/enhLutNer1/bed/gapOverlap time (doGapOverlap.pl \ -twoBit=/hive/data/genomes/enhLutNer1/enhLutNer1.unmasked.2bit enhLutNer1 ) \ > do.log 2>&1 & # real 3m3.656s cat fb.enhLutNer1.gapOverlap.txt # 290750 bases of 2425597608 (0.012%) in intersection ############################################################################# # tandemDups (DONE - 2019-11-15 - Jonathan) mkdir /hive/data/genomes/enhLutNer1/bed/tandemDups cd /hive/data/genomes/enhLutNer1/bed/tandemDups time (~/kent/src/hg/utils/automation/doTandemDup.pl \ -twoBit=/hive/data/genomes/enhLutNer1/enhLutNer1.unmasked.2bit enhLutNer1) \ > do.log 2>&1 & # real 41m29.596s cat fb.enhLutNer1.tandemDups.txt # 22686356 bases of 2425597608 (0.935%) in intersection bigBedInfo enhLutNer1.tandemDups.bb | sed -e 's/^/# /;' # version: 4 # fieldCount: 13 # hasHeaderExtension: yes # isCompressed: yes # isSwapped: 0 # extraIndexCount: 0 # itemCount: 363,243 # primaryDataSize: 10,216,241 # primaryIndexSize: 110,800 # zoomLevels: 8 # chromCount: 2600 # basesCovered: 1,194,001,583 # meanDepth (of bases covered): 3.077659 # minDepth: 1.000000 # maxDepth: 158.000000 # std of depth: 3.224928 ######################################################################### # ucscToINSDC table/track (DONE - 2019-12-04 - Jonathan) # construct idKeys for the genbank sequence mkdir /hive/data/genomes/enhLutNer1/genbank/idKeys cd /hive/data/genomes/enhLutNer1/genbank/idKeys faToTwoBit ../GCA_006410715.1_ASM641071v1_genomic.fna.gz enhLutNer1.genbank.2bit time (doIdKeys.pl -buildDir=`pwd` \ -twoBit=`pwd`/enhLutNer1.genbank.2bit genbankEnhLutNer1) > do.log 2>&1 & # real 19m47.915s cat genbankEnhLutNer1.keySignature.txt # 6ccb6138a2432071b6103be8b66bb46d mkdir /hive/data/genomes/enhLutNer1/bed/chromAlias cd /hive/data/genomes/enhLutNer1/bed/chromAlias join -t$'\t' ../idKeys/enhLutNer1.idKeys.txt \ ../../genbank/idKeys/genbankEnhLutNer1.idKeys.txt | cut -f2- \ | sort -k1,1 | join -t$'\t' <(sort -k1,1 ../../chrom.sizes) - \ | awk '{printf "%s\t0\t%d\t%s\n", $1, $2, $3}' \ | sort -k1,1 -k2,2n > ucscToINSDC.bed # should be same line counts throughout: wc -l * ../../chrom.sizes # 55496 ucscToINSDC.bed # 55496 ../../chrom.sizes export chrSize=`cut -f1 ucscToINSDC.bed | awk '{print length($0)}' | sort -n | tail -1` echo $chrSize # 14 # use the $chrSize in this sed sed -e "s/21/$chrSize/" $HOME/kent/src/hg/lib/ucscToINSDC.sql \ | hgLoadSqlTab enhLutNer1 ucscToINSDC stdin ucscToINSDC.bed # should be quiet for all OK checkTableCoords enhLutNer1 # should cover %100 entirely: featureBits -countGaps enhLutNer1 ucscToINSDC # 2425597608 bases of 2425597608 (100.000%) in intersection ######################################################################### # add chromAlias table (DONE - 2019-12-04 - Jonathan) mkdir /hive/data/genomes/enhLutNer1/bed/chromAlias cd /hive/data/genomes/enhLutNer1/bed/chromAlias hgsql -N -e 'select chrom,name from ucscToINSDC;' enhLutNer1 \ | sort -k1,1 > ucsc.genbank.tab # Commented out pending adding enhLutNer1 Ensembl build # ### Adding Ensembl alias with v95 release, after idKeys made: 2019-01-16 # join -t$'\t' ../idKeys/enhLutNer1.idKeys.txt \ # ../../ens95/ensEnhLutNer1.idKeys.txt | cut -f2- \ # | sort -k1,1 | join -t$'\t' <(sort -k1,1 ../../chrom.sizes) - \ # | awk '{printf "%s\t0\t%d\t%s\n", $1, $2, $3}' \ # | sort -k1,1 -k2,2n > ucscToEns.bed # cut -f1,4 ucscToEns.bed | sort > ucsc.ensembl.tab wc -l *.bed # 55496 ucscToINSDC.bed ~/kent/src/hg/utils/automation/chromAlias.pl ucsc.*.tab \ > enhLutNer1.chromAlias.tab for t in genbank do c0=`cat ucsc.$t.tab | wc -l` c1=`grep $t enhLutNer1.chromAlias.tab | wc -l` ok="OK" if [ "$c0" -ne "$c1" ]; then ok="ERROR" fi printf "# checking $t: $c0 =? $c1 $ok\n" done # checking genbank: 55496 =? 55496 OK hgLoadSqlTab enhLutNer1 chromAlias ~/kent/src/hg/lib/chromAlias.sql \ enhLutNer1.chromAlias.tab ######################################################################### # fixup search rule for assembly track/gold table (DONE - 2019-12-06 - Jonathan) cd ~/kent/src/hg/makeDb/trackDb/otters/enhLutNer1 # preview prefixes and suffixes: hgsql -N -e "select frag from gold;" enhLutNer1 \ | sed -e 's/[0-9][0-9]*//;' | sort | uniq -c # result truncated - 4307 lines like the following 35 QQQEv1_988 35 QQQEv1_989 290 QQQEv1_99 35 QQQEv1_990 35 QQQEv1_991 35 QQQEv1_992 35 QQQEv1_993 35 QQQEv1_994 35 QQQEv1_995 35 QQQEv1_996 35 QQQEv1_997 35 QQQEv1_998 35 QQQEv1_999 # implies a rule: 'QQQE[0-9]+(\.[0-9]+)?' # verify this rule will find them all and eliminate them all: hgsql -N -e "select frag from gold;" enhLutNer1 | wc -l # 212165 hgsql -N -e "select frag from gold;" enhLutNer1 \ | egrep -e 'QQQE[0-9]+(\.[0-9]+)?' | wc -l # 212165 hgsql -N -e "select frag from gold;" enhLutNer1 \ | egrep -v -e 'QQQE[0-9]+(\.[0-9]+)?' | wc -l # 0 # hence, add to trackDb/otters/enhLutNer1/trackDb.ra searchTable gold shortCircuit 1 termRegex [ANQ][CN][V0-9_][O0-9][0-9]+(\.[0-9]+)? query select chrom,chromStart,chromEnd,frag from %s where frag like '%s%%' searchPriority 8 # verify searches work in the position box ########################################################################## # running repeat masker (DONE - 2019-12-09 - Jonathan) mkdir /hive/data/genomes/enhLutNer1/bed/repeatMasker cd /hive/data/genomes/enhLutNer1/bed/repeatMasker time (doRepeatMasker.pl -buildDir=`pwd` \ -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ -smallClusterHub=ku enhLutNer1) > do.log 2>&1 # real 328m48.349s cat faSize.rmsk.txt # 2425597608 bases (11943786 N's 2413653822 real 1460514973 upper # 953138849 lower) in 55496 sequences in 1 files # Total size: mean 43707.6 sd 750707.1 min 1000 (QQQE01055496v1) # max 56733013 (QQQE01000071v1) median 2458 # %39.30 masked total, %39.49 masked real egrep -i "versi|relea" do.log # RepeatMasker version development-$Id: RepeatMasker,v 1.332 2017/04/17 19:01:11 rhubley Exp $ # February 01 2017 (open-4-0-8) 1.332 version of RepeatMasker # CC Dfam_Consensus RELEASE 20181026; * # CC RepBase RELEASE 20181026; time featureBits -countGaps enhLutNer1 rmsk # 953703279 bases of 2425597608 (39.318%) in intersection # real 0m46.575s # why is it different than the faSize above ? # because rmsk masks out some N's as well as bases, the faSize count above # separates out the N's from the bases, it doesn't show lower case N's # faster way to get the same result on high contig count assemblies: time hgsql -N -e 'select genoName,genoStart,genoEnd from rmsk;' enhLutNer1 \ | bedSingleCover.pl stdin | ave -col=4 stdin | grep "^total" # total 953703279.000000 # real 1m29.363s ########################################################################## # running simple repeat (DONE - 2019-12-09 - Jonathan) # The '-trf409 5' is a bit smaller than human which is 6 mkdir /hive/data/genomes/enhLutNer1/bed/simpleRepeat cd /hive/data/genomes/enhLutNer1/bed/simpleRepeat time (doSimpleRepeat.pl -buildDir=`pwd` -bigClusterHub=ku \ -dbHost=hgwdev -workhorse=hgwdev -smallClusterHub=ku \ -trf409=5 enhLutNer1) > do.log 2>&1 # real 27m25.848s cat fb.simpleRepeat # 29255764 bases of 2413653822 (1.212%) in intersection cd /hive/data/genomes/enhLutNer1 # if using the Window Masker result instead of rmsk (because of greater coverage): # twoBitMask bed/windowMasker/enhLutNer1.cleanWMSdust.2bit \ # -add bed/simpleRepeat/trfMask.bed enhLutNer1.2bit # you can safely ignore the warning about fields >= 13 # add to rmsk after it is done: twoBitMask enhLutNer1.rmsk.2bit \ -add bed/simpleRepeat/trfMask.bed enhLutNer1.2bit # you can safely ignore the warning about fields >= 13 twoBitToFa enhLutNer1.2bit stdout | faSize stdin > faSize.enhLutNer1.2bit.txt cat faSize.enhLutNer1.2bit.txt # 2425597608 bases (11943786 N's 2413653822 real 1459552465 upper # 954101357 lower) in 55496 sequences in 1 files # Total size: mean 43707.6 sd 750707.1 min 1000 (QQQE01055496v1) # max 56733013 (QQQE01000071v1) median 2458 # %39.33 masked total, %39.53 masked real rm /gbdb/enhLutNer1/enhLutNer1.2bit ln -s `pwd`/enhLutNer1.2bit /gbdb/enhLutNer1/enhLutNer1.2bit ######################################################################### # CREATE MICROSAT TRACK (DONE - 2019-12-09 - Jonathan) ssh hgwdev mkdir /hive/data/genomes/enhLutNer1/bed/microsat cd /hive/data/genomes/enhLutNer1/bed/microsat awk '($5==2 || $5==3) && $6 >= 15 && $8 == 100 && $9 == 0 {printf("%s\t%s\t%s\t%dx%s\n", $1, $2, $3, $6, $16);}' \ ../simpleRepeat/simpleRepeat.bed > microsat.bed hgLoadBed enhLutNer1 microsat microsat.bed # Read 52659 elements of size 4 from microsat.bed ########################################################################## ## WINDOWMASKER (DONE - 2019-12-09 - Jonathan) mkdir /hive/data/genomes/enhLutNer1/bed/windowMasker cd /hive/data/genomes/enhLutNer1/bed/windowMasker time (doWindowMasker.pl -buildDir=`pwd` -workhorse=hgwdev \ -dbHost=hgwdev enhLutNer1) > do.log 2>&1 # real 122m18.256s # Masking statistics cat faSize.enhLutNer1.cleanWMSdust.txt # 2425597608 bases (11943786 N's 2413653822 real 1662278860 upper # 751374962 lower) in 55496 sequences in 1 files # Total size: mean 43707.6 sd 750707.1 min 1000 (QQQE01055496v1) # max 56733013 (QQQE01000071v1) median 2458 # %30.98 masked total, %31.13 masked real cat fb.enhLutNer1.rmsk.windowmaskerSdust.txt # 478644513 bases of 2425597608 (19.733%) in intersection # reminder to go back to the trf results and re-incorporate them if switching from rmsk to wm masking ########################################################################## # cpgIslands - (DONE - 2019-12-16 - Jonathan) mkdir /hive/data/genomes/enhLutNer1/bed/cpgIslands cd /hive/data/genomes/enhLutNer1/bed/cpgIslands time (doCpgIslands.pl -dbHost=hgwdev -bigClusterHub=ku \ -workhorse=hgwdev -smallClusterHub=ku enhLutNer1) > do.log 2>&1 # real 8m36.572s cat fb.enhLutNer1.cpgIslandExt.txt # 40415592 bases of 2413653822 (1.674%) in intersection ############################################################################## # genscan - (DONE - 2019-12-19 - Jonathan) mkdir /hive/data/genomes/enhLutNer1/bed/genscan cd /hive/data/genomes/enhLutNer1/bed/genscan time (doGenscan.pl -buildDir=`pwd` -workhorse=hgwdev -dbHost=hgwdev \ -bigClusterHub=ku enhLutNer1) > do.log 2>&1 # some failed. para time, to set up time file # then para status | grep -v done to get only the ones that failed. # grab those, put in script, run on dev. maybe change -window=2400000 to -window=2000000 # - Hiram says the jobs generally complete fine when he does that. # only a couple jobs failed, so probably just run those in the background on dev. # $ para status | grep -v done # 55496 jobs in batch # 1 jobs (including everybody's) in Parasol queue or running. # Checking finished jobs # #state tries real cpu host jobid cmd # crash 4 92.00 90.03 ku-06.gi 66499300 ./runGsBig.csh QQQE01000220v1 000 gtf/000/QQQE01000220v1.gtf pep/000/QQQE01000220v1.pep subopt/000/QQQE01000220v1.bed # crash 4 51.00 46.12 ku-19.gi 66502474 ./runGsBig.csh QQQE01000173v1 000 gtf/000/QQQE01000173v1.gtf pep/000/QQQE01000173v1.pep subopt/000/QQQE01000173v1.bed cp runGsBig.csh runGsNotQuiteAsBig.csh # Change 2400000 windows to 2000000 and temp dir to /data/tmp ./runGsNotQuiteAsBig.csh QQQE01000220v1 000 gtf/000/QQQE01000220v1.gtf pep/000/QQQE01000220v1.pep subopt/000/QQQE01000220v1.bed > 220v1.log 2>&1 ./runGsNotQuiteAsBig.csh QQQE01000173v1 000 gtf/000/QQQE01000173v1.gtf pep/000/QQQE01000173v1.pep subopt/000/QQQE01000173v1.bed > 173v1.log 2>&1 time (doGenscan.pl -buildDir=`pwd` -workhorse=hgwdev -dbHost=hgwdev \ -bigClusterHub=ku -continue=makeBed enhLutNer1) > do.log2 2>&1 # real 4m14.067s cat fb.enhLutNer1.genscan.txt # 61083503 bases of 2413653822 (2.531%) in intersection cat fb.enhLutNer1.genscanSubopt.txt # 59601148 bases of 2413653822 (2.469%) in intersection ######################################################################### # Create kluster run files (DONE - 2019-12-20 - Jonathan) # numerator is enhLutNer1 gapless bases "real" as reported by: featureBits -noRandom -noHap enhLutNer1 gap # 11943786 bases of 2413653822 (0.495%) in intersection # ^^^ # denominator is hg19 gapless bases as reported by: # featureBits -noRandom -noHap hg19 gap # 234344806 bases of 2861349177 (8.190%) in intersection # 1024 is threshold used for human -repMatch: calc \( 2413653822 / 2861349177 \) \* 1024 # ( 2413653822 / 2861349177 ) * 1024 = 863.781860 # ==> use -repMatch=350 according to size scaled down from 1024 for human. # and rounded down to nearest 50 cd /hive/data/genomes/enhLutNer1 blat enhLutNer1.2bit \ /dev/null /dev/null -tileSize=11 -makeOoc=jkStuff/enhLutNer1.11.ooc \ -repMatch=850 # Wrote 23338 overused 11-mers to jkStuff/enhLutNer1.11.ooc # rheMac8 at repMatch=900: # Wrote 43065 overused 11-mers to jkStuff/rheMac8.11.ooc # check non-bridged gaps to see what the typical size is: hgsql -N \ -e 'select * from gap where bridge="no" order by size;' enhLutNer1 \ | sort -k7,7nr | ave -col=7 stdin # No numerical data column 7 of stdin # There are no unbridged gaps here, so no lift to create # gapToLift -verbose=2 -minGap=100 enhLutNer1 jkStuff/enhLutNer1.nonBridged.lft \ # -bedFile=jkStuff/enhLutNer1.nonBridged.bed # wc -l jkStuff/enhLutNer1.nonBri* # # 2979 jkStuff/enhLutNer1.nonBridged.bed # # 2979 jkStuff/enhLutNer1.nonBridged.lft ######################################################################## # lastz/chain/net swap human/hg38 (DONE - 2019-12-22 - Jonathan) # Waiting on hg38/lastzRuns.txt run # original alignment cd /hive/data/genomes/hg38/bed/lastzEnhLutNer1.2019-07-03 cat fb.hg38.chainEnhLutNer1Link.txt # 1537745313 bases of 3095998939 (49.669%) in intersection cat fb.hg38.chainSynEnhLutNer1Link.txt # 1454400074 bases of 3095998939 (46.977%) in intersection cat fb.hg38.chainRBest.EnhLutNer1.txt # 1413203552 bases of 3095998939 (45.646%) in intersection # and for the swap: mkdir /hive/data/genomes/enhLutNer1/bed/blastz.hg38.swap cd /hive/data/genomes/enhLutNer1/bed/blastz.hg38.swap time (doBlastzChainNet.pl -verbose=2 \ /hive/data/genomes/hg38/bed/lastzEnhLutNer1.2019-12-20/DEF \ -swap -chainMinScore=3000 -chainLinearGap=medium \ -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ -noDbNameCheck -syntenicNet) > swap.log 2>&1 # real 88m38.087s cat fb.enhLutNer1.chainHg38Link.txt # 1526639908 bases of 2413653822 (63.250%) in intersection cat fb.enhLutNer1.chainSynHg38Link.txt # 1403546142 bases of 2413653822 (58.150%) in intersection time (doRecipBest.pl -workhorse=hgwdev -load -buildDir=`pwd` enhLutNer1 hg38) > rbest.log 2>&1 # real 319m54.030s cat fb.enhLutNer1.chainRBest.Hg38.txt # 1414347523 bases of 2413653822 (58.598%) in intersection ######################################################################### # lastz/chain/net swap mouse/mm10 (DONE - 2020-04-16 - Jonathan) # original alignment cd /hive/data/genomes/mm10/bed/lastzEnhLutNer1.2020-04-15 cat fb.mm10.chainEnhLutNer1Link.txt # 772059271 bases of 2652783500 (29.104%) in intersection cat fb.mm10.chainSynEnhLutNer1Link.txt # 717097454 bases of 2652783500 (27.032%) in intersection cat fb.mm10.chainRBest.EnhLutNer1.txt # 734878489 bases of 2652783500 (27.702%) in intersection mkdir /hive/data/genomes/enhLutNer1/bed/blastz.mm10.swap cd /hive/data/genomes/enhLutNer1/bed/blastz.mm10.swap time (doBlastzChainNet.pl -verbose=2 \ /hive/data/genomes/mm10/bed/lastzEnhLutNer1.2020-04-15/DEF \ -swap -syntenicNet \ -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ -chainMinScore=3000 -chainLinearGap=medium) > swap.log 2>&1 # real 48m44.604s cat fb.enhLutNer1.chainMm10Link.txt # 787727864 bases of 2413653822 (32.636%) in intersection cat fb.enhLutNer1.chainSynMm10Link.txt # 712950571 bases of 2413653822 (29.538%) in intersection time (doRecipBest.pl -workhorse=hgwdev -load -buildDir=`pwd` enhLutNer1 mm10) > rbest.log 2>&1 # real 167m45.490s cat fb.enhLutNer1.chainRBest.Mm10.txt # 734620004 bases of 2413653822 (30.436%) in intersection ############################################################################## # GENBANK AUTO UPDATE (DONE - 2019-12-23 - Jonathan) ssh hgwdev cd $HOME/kent/src/hg/makeDb/genbank git pull # /cluster/data/genbank/data/organism.lst shows: # #organism mrnaCnt estCnt refSeqCnt # Macaca mulatta 381495 60276 5820 # edit etc/genbank.conf to add enhLutNer1 just after musFur1 (ferret), since # they're somewhat related # enhLutNer1 Enhydra lutris nereis (southern sea otter) (genbank assembly GCA_006410715.1) enhLutNer1.serverGenome = /hive/data/genomes/enhLutNer1/enhLutNer1.2bit enhLutNer1.ooc = /hive/data/genomes/enhLutNer1/jkStuff/enhLutNer1.11.ooc enhLutNer1.lift = no enhLutNer1.perChromTables = no enhLutNer1.refseq.mrna.native.pslCDnaFilter = ${ordered.refseq.mrna.native.pslCDnaFilter} enhLutNer1.refseq.mrna.xeno.pslCDnaFilter = ${ordered.refseq.mrna.xeno.pslCDnaFilter} enhLutNer1.genbank.mrna.native.pslCDnaFilter = ${ordered.genbank.mrna.native.pslCDnaFilter} enhLutNer1.genbank.mrna.xeno.pslCDnaFilter = ${ordered.genbank.mrna.xeno.pslCDnaFilter} enhLutNer1.genbank.est.native.pslCDnaFilter = ${ordered.genbank.est.native.pslCDnaFilter} enhLutNer1.genbank.est.xeno.pslCDnaFilter = ${ordered.genbank.est.xeno.pslCDnaFilter} enhLutNer1.downloadDir = enhLutNer1 enhLutNer1.refseq.mrna.native.load = yes enhLutNer1.refseq.mrna.xeno.load = yes # verify the files specified exist before checking in the file: grep ^enhLutNer1 etc/genbank.conf | grep hive | awk '{print $NF}' | xargs ls -og # -rw-rw-r-- 1 639922406 Dec 9 14:48 /hive/data/genomes/enhLutNer1/enhLutNer1.2bit # -rw-rw-r-- 1 93360 Dec 20 11:39 /hive/data/genomes/enhLutNer1/jkStuff/enhLutNer1.11.ooc # Reminder to add the new species to gbGenome.c in the genbank directory. git commit -m "Added enhLutNer1 otter/Enhydra lutris nereis; refs #23771" etc/genbank.conf git push # update /cluster/data/genbank/: make etc-update # enable daily alignment and update of hgwdev cd ~/kent/src/hg/makeDb/genbank git pull # add enhLutNer1 to: # etc/hgwdev.dbs # etc/align.dbs git add etc/align.dbs etc/hgwdev.dbs git commit -m "Added enhLutNer1 - Southern sea otter refs #21152" git push make etc-update # wait a few days for genbank magic to take place, the tracks will # appear. Actually, apparently these days we have to request it from the # genbank handler (currently Chris) ############################################################################# # augustus gene track (DONE - 2020-01-03 - Jonathan) mkdir /hive/data/genomes/enhLutNer1/bed/augustus cd /hive/data/genomes/enhLutNer1/bed/augustus time (doAugustus.pl -buildDir=`pwd` -bigClusterHub=ku \ -species=human -dbHost=hgwdev \ -workhorse=hgwdev enhLutNer1) > do.log 2>&1 # real 95m23.998s cat fb.enhLutNer1.augustusGene.txt # 50979954 bases of 2413653822 (2.112%) in intersection ######################################################################### # BLATSERVERS ENTRY (DONE - 2020-03-02 - Jonathan) # After getting a blat server assigned by the Blat Server Gods, ssh hgwdev hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \ VALUES ("enhLutNer1", "blat1b", "17902", "1", "0"); \ INSERT INTO blatServers (db, host, port, isTrans, canPcr) \ VALUES ("enhLutNer1", "blat1b", "17903", "0", "1");' \ hgcentraltest # test it with some sequence ############################################################################ # Set default position (DONE - 2020-03-02 - Jonathan) # Setting to a putative region for BEND7, a gene identified as being # under positive selection (see https://doi.org/10.1093/molbev/msz101) ssh hgwdev hgsql -e 'update dbDb set defaultPos="QQQE01000156v1:7095823-7210817" where name="enhLutNer1";' hgcentraltest ######################################################################### # crispr 10K shoulders (DONE - 2018-10-16 - Jonathan) # Gone sideways on me - the usage message is wrong and this will NOT use the # default build directory of $db/bed/crispr - apparently it's dumping stuff # into the current directory. Now I have to figure out what all it did # create and clean up after it before trying to get the ranges step # to run (also it looks like it tried to start guides?) mkdir /hive/data/genomes/enhLutNer1/bed/crispr cd /hive/data/genomes/enhLutNer1/bed/crispr # No native RefSeq track, so using xenoRefGene time (~/kent/src/hg/utils/automation/doCrispr.pl \ -stop=indexFa -buildDir=`pwd` -smallClusterHub=ku enhLutNer1 xenoRefGene) \ > indexFa.log 2>&1 # real 59m2.708s time (~/kent/src/hg/utils/automation/doCrispr.pl \ -continue=ranges -stop=guides -buildDir=`pwd` -smallClusterHub=ku \ enhLutNer1 xenoRefGene) > guides.log 2>&1 # real 7m5.619s time (~/kent/src/hg/utils/automation/doCrispr.pl \ -continue=specScoreJobList -stop=specScores -buildDir=`pwd` \ -smallClusterHub=ku enhLutNer1 xenoRefGene) > specScores.log # Completed: 1520504 of 1520504 jobs # CPU time in finished jobs: 131133376s 2185556.27m 36425.94h 1517.75d 4.158 y # IO & Wait Time: 3406121s 56768.68m 946.14h 39.42d 0.108 y # Average job time: 88s 1.47m 0.02h 0.00d # Longest finished job: 373s 6.22m 0.10h 0.00d # Submission to last job: 160229s 2670.48m 44.51h 1.85d cd specScores time find tmp/outGuides -type f | xargs cut -f3-6 > ../specScores.tab # real 355m56.098s cd .. wc -l specScores.tab # 115041122 specScores.tab time (~/kent/src/hg/utils/automation/doCrispr.pl \ -continue=effScores -stop=load \ -buildDir=`pwd` -smallClusterHub=ku enhLutNer1 ncbiRefSeq) \ > load.log # real 393m15.373s + # hive cleaning 2021-04-26 - Hiram - 10.86 Tb + time (~/kent/src/hg/utils/automation/doCrispr.pl -verbose=2 \ + -continue=cleanup enhLutNer1 -fileServer=hgwdev \ + -buildDir=`pwd` -smallClusterHub=hgwdev -bigClusterHub=ku \ + -workhorse=hgwdev) > cleanup.log 2>&1 + ######################################################################### # all.joiner update, downloads and in pushQ - (TBD - 2019-07-03,08-30 - Jonathan) xyz cd $HOME/kent/src/hg/makeDb/schema # verify all the business is done for release ~/kent/src/hg/utils/automation/verifyBrowser.pl enhLutNer1 # fixup all.joiner until this is a clean output joinerCheck -database=enhLutNer1 -tableCoverage all.joiner joinerCheck -database=enhLutNer1 -times all.joiner joinerCheck -database=enhLutNer1 -keys all.joiner # when clean, check in: git commit -m 'adding rules for enhLutNer1, refs #21152' all.joiner git push # run up a 'make alpha' in hg/hgTables to get this all.joiner file # into the hgwdev/genome-test system cd /hive/data/genomes/enhLutNer1 time (makeDownloads.pl enhLutNer1) > downloads.log 2>&1 # real 15m58.971s # now ready for pushQ entry mkdir /hive/data/genomes/enhLutNer1/pushQ cd /hive/data/genomes/enhLutNer1/pushQ time ($HOME/kent/src/hg/utils/automation/makePushQSql.pl -redmineList enhLutNer1) > enhLutNer1.pushQ.sql 2> stderr.out # real 15m52.548s # remove the tandemDups and gapOverlap from the file list: sed -i -e "/tandemDups/d" redmine.enhLutNer1.table.list sed -i -e "/Tandem Dups/d" redmine.enhLutNer1.releaseLog.txt sed -i -e "/gapOverlap/d" redmine.enhLutNer1.table.list sed -i -e "/Gap Overlaps/d" redmine.enhLutNer1.releaseLog.txt # check for errors in stderr.out, some are OK, e.g.: # WARNING: hgwdev does not have /gbdb/enhLutNer1/wib/gc5Base.wib # WARNING: hgwdev does not have /gbdb/enhLutNer1/wib/quality.wib # WARNING: hgwdev does not have /gbdb/enhLutNer1/bbi/quality.bw # WARNING: enhLutNer1 does not have seq # WARNING: enhLutNer1 does not have extFile # verify the file list does correctly match to files cat redmine.enhLutNer1.file.list | while read L do eval ls $L > /dev/null done # should be silent, missing files will show as errors # verify database tables, how many to expect: wc -l redmine.enhLutNer1.table.list # 63 redmine.enhLutNer1.table.list # how many actual: awk -F'.' '{printf "hgsql -N %s -e '"'"'show table status like \"%s\";'"'"'\n", $1, $2}' redmine.enhLutNer1.table.list | sh | wc -l # 63 # would be a smaller number actual if some were missing # add the path names to the listing files in the redmine issue # in the three appropriate entry boxes: # /hive/data/genomes/enhLutNer1/pushQ/redmine.enhLutNer1.file.list # /hive/data/genomes/enhLutNer1/pushQ/redmine.enhLutNer1.releaseLog.txt # /hive/data/genomes/enhLutNer1/pushQ/redmine.enhLutNer1.table.list #########################################################################