e18510fd8cce5a41aac912da2ca49fa910e2bc5f markd Fri Nov 12 07:35:55 2021 -0800 updated doc for metabat common name fix diff --git src/hg/makeDb/doc/myoLuc2.txt src/hg/makeDb/doc/myoLuc2.txt index 4ac6837..1db36f7 100644 --- src/hg/makeDb/doc/myoLuc2.txt +++ src/hg/makeDb/doc/myoLuc2.txt @@ -1,692 +1,698 @@ # for emacs: -*- mode: sh; -*- # Myotis lucifugus (microbat) -- Broad Institute of MIT and Harvard +# NOTE: "microbat" is not the ncorrect common name, it is "Little brown bat" + # http://www.ncbi.nlm.nih.gov/genome/614 # http://www.ncbi.nlm.nih.gov/bioproject/16951 # http://www.ncbi.nlm.nih.gov/Traces/wgs/?val=ABVD00 # 2.18X WGS # file template copied from susScr2.txt # Myotis lucifugus # (NCBI Project ID: 16951, Accession: GCA_000005525.1) # by Broad Institute of MIT and Harvard # ftp://ftp.ncbi.nlm.nih.gov/genbank/genomes/Eukaryotes/vertebrates_mammals/Myotis_lucifugus/Myoluc2.0/ ########################################################################## # Download sequence (DONE - 2010-10-22 - Chin) mkdir /hive/data/genomes/myoLuc2 cd /hive/data/genomes/myoLuc2 mkdir genbank cd genbank wget --timestamping -r --cut-dirs=6 --level=0 -nH -x \ --no-remove-listing -np \ "ftp://ftp.ncbi.nlm.nih.gov/genbank/genomes/Eukaryotes/vertebrates_mammals/Myotis_lucifugus/Myoluc2.0/*" # FINISHED --2010-10-22 09:11:24-- # Downloaded: 12 files, 802M in 18m 12s (752 KB/s) # Read ASSEMBLY_INFO # stay at genbank directory # Process the unplaced scaffolds, filter out the # The ".1" at the end (i.e. ABQO010000034.1) of contig name, since # MySQL does not allow "." as part of the table name and # will casue problems in genbank task step later export S=Primary_Assembly/unplaced_scaffolds zcat ${S}/AGP/unplaced.scaf.agp.gz | grep "^#" > myoLuc2.agp # append the gap records zcat ${S}/AGP/unplaced.scaf.agp.gz | grep -v "^#" \ | sed -e "s/\.1//" >> myoLuc2.agp gzip myoLuc2.agp & zcat ${S}/FASTA/unplaced.scaf.fa.gz \ | sed -e "s#^>.*|gb|#>#; s#|.*##" -e "s/\.1//" \ | gzip > myoLuc2.fa.gz zcat myoLuc2.fa.gz | grep "^>" | wc # 11654 11654 149528 faSize Primary_Assembly/unplaced_scaffolds/FASTA/unplaced.scaf.fa.gz # 2034575300 bases (68155432 N's 1966419868 real 1966419868 upper # 0 lower) in 11654 sequences in 1 files # N50 mkdir N50 zcat myoLuc2.fa.gz | faCount stdin | awk '/^(GL|AAPE)/ {print $1, $2}' > N50/chrom.sizes n50.pl N50/chrom.sizes # reading: N50/chrom.sizes # contig count: 11654, total size: 2034575300, one half size: 1017287650 # cumulative N50 count contig contig size 1015221828 105 GL429874 4309204 1017287650 one half size 1019515143 106 GL429872 4293315 ######################################################################### # Initial makeGenomeDb.pl (DONE - 2010-11-02 - Chin) cd /hive/data/genomes/myoLuc2 cat << '_EOF_' > myoLuc2.config.ra # Config parameters for makeGenomeDb.pl: db myoLuc2 clade mammal genomeCladePriority 35 scientificName Myotis lucifugus commonName Microbat assemblyDate Jul. 2010 assemblyLabel Broad Institute (NCBI Project ID: 16951, Accession: GCA_000147115.1) assemblyShortLabel Broad Institute Myoluc2.0 orderKey 236.5 mitoAcc none fastaFiles /hive/data/genomes/myoLuc2/genbank/myoLuc2.fa.gz agpFiles /hive/data/genomes/myoLuc2/genbank/myoLuc2.agp.gz # qualFiles none dbDbSpeciesDir microbat taxId 59463 '_EOF_' # << happy emacs time makeGenomeDb.pl -noGoldGapSplit -workhorse=hgwdev myoLuc2.config.ra \ > makeGenomeDb.log 2>&1 & # real 15m33.626s # add the trackDb entries to the source tree, and the 2bit link: ln -s `pwd`/myoLuc2.unmasked.2bit /gbdb/myoLuc2/myoLuc2.2bit # Per instructions in makeGenomeDb.log: # cd ~/kent/src/hg/makeDb/trackDb # edit makefile to add myoLuc2 to DBS. # git add microbat/myoLuc2/*.{ra,html} # git commit -m "Added myoLuc2 to DBS." makefile # git commit -m "Initial descriptions for myoLuc2." microbat/myoLuc2/*.{ra,html} # git pull; git push # Run make update DBS=myoLuc2 and make alpha when done. # (optional) Clean up /cluster/data/myoLuc2/TemporaryTrackDbCheckout ######################################################################### # RepeatMasker (DONE 2010-11-03 - Chin) mkdir /hive/data/genomes/myoLuc2/bed/repeatMasker cd /hive/data/genomes/myoLuc2/bed/repeatMasker time nice -n +19 doRepeatMasker.pl -buildDir=`pwd` \ -workhorse=hgwdev -bigClusterHub=swarm -noSplit myoLuc2 > do.log 2>&1 & # real 253m45.336s cat faSize.rmsk.txt # 2034575300 bases (68155432 N's 1966419868 real 1304219212 upper # 662200656 lower) in 11654 sequences in 1 files # %32.55 masked total, %33.68 masked real ######################################################################### # simpleRepeats ( DONE 2010-11-03 - Chin) mkdir /hive/data/genomes/myoLuc2/bed/simpleRepeat cd /hive/data/genomes/myoLuc2/bed/simpleRepeat time nice -n +19 doSimpleRepeat.pl -buildDir=`pwd` -workhorse=hgwdev \ -bigClusterHub=swarm -smallClusterHub=swarm myoLuc2 > do.log 2>&1 & # real 9m30.973s cat fb.simpleRepeat # 43294476 bases of 1966419868 (2.202%) in intersection # add to the repeatMasker cd /hive/data/genomes/myoLuc2 twoBitMask myoLuc2.rmsk.2bit -add bed/simpleRepeat/trfMask.bed myoLuc2.2bit # safe to ignore warnings about >=13 fields twoBitToFa myoLuc2.2bit stdout | faSize stdin > myoLuc2.2bit.faSize.txt cat myoLuc2.2bit.faSize.txt # 2034575300 bases (68155432 N's 1966419868 real 1303520693 upper # 662899175 lower) in 11654 sequences in 1 files # %32.58 masked total, %33.71 masked real # set up 2bit for download (2011-10-24 - Chin) rm /gbdb/myoLuc2/myoLuc2.2bit ln -s `pwd`/myoLuc2.2bit /gbdb/myoLuc2/myoLuc2.2bit ######################################################################### # Marking *all* gaps - they are all in the AGP file # (DONE - 2010-11-04 - Chin) mkdir /hive/data/genomes/myoLuc2/bed/allGaps cd /hive/data/genomes/myoLuc2/bed/allGaps time nice -n +19 findMotif -motif=gattaca -verbose=4 \ -strand=+ ../../myoLuc2.unmasked.2bit > findMotif.txt 2>&1 # real 0m25.895s grep "^#GAP " findMotif.txt | sed -e "s/^#GAP //" > allGaps.bed featureBits myoLuc2 -not gap -bed=notGap.bed # 1966419868 bases of 1966419868 (100.000%) in intersection featureBits myoLuc2 allGaps.bed notGap.bed -bed=new.gaps.bed # 0 bases of 1966419868 (0.000%) in intersection # they are all in the AGP file # what is the highest index in the existing gap table: hgsql -N -e "select ix from gap;" myoLuc2 | sort -n | tail -1 # 2378 featureBits -countGaps myoLuc2 gap # 68155432 bases of 2034575300 (3.350%) in intersection ########################################################################## ## WINDOWMASKER (DONE- 2013-06-21 - Hiram) mkdir /hive/data/genomes/myoLuc2/bed/windowMasker cd /hive/data/genomes/myoLuc2/bed/windowMasker time nice -n +19 doWindowMasker.pl -buildDir=`pwd` -workhorse=hgwdev \ -dbHost=hgwdev myoLuc2 > do.log 2>&1 & XXX - running - Fri Jun 21 16:03:53 PDT 2013 # real 170m17.913s # Masking statistics cat faSize.myoLuc2.cleanWMSdust.txt # 2034575300 bases (68155432 N's 1966419868 real 1285911982 # upper 680507886 lower) in 11654 sequences in 1 files # Total size: mean 174581.7 sd 1247597.7 min 3011 (AAPE02072784) # max 64990205 (GL429767) median 9365 # %33.45 masked total, %34.61 masked real # how much does this window masker and repeat masker overlap: featureBits -countGaps myoLuc2 rmsk windowmaskerSdust \ > fb.myoLuc2.rmsk.windowmaskerSdust.txt 2>&1 # 355414901 bases of 2034575300 (17.469%) in intersection ######################################################################## # Create kluster run files (DONE - 2010-11-04 - Chin) # numerator is myoLuc2 gapless bases "real" as reported by: featureBits -noRandom -noHap myoLuc2 gap # 68155432 bases of 1966419868 (3.466%) in intersection # denominator is hg19 gapless bases as reported by: # featureBits -noRandom -noHap hg19 gap # 234344806 bases of 2861349177 (8.190%) in intersection # 1024 is threshold used for human -repMatch: calc \( 1966419868 / 2861349177 \) \* 1024 # ( 1966419868 / 2861349177 ) * 1024 = 703.728843 # ==> use -repMatch=400 according to size scaled down from 1024 for human. # and rounded down to nearest 50 (in this case, 700) cd /hive/data/genomes/myoLuc2 blat myoLuc2.2bit \ /dev/null /dev/null -tileSize=11 -makeOoc=jkStuff/myoLuc2.11.ooc \ -repMatch=700 & # Loading myoLuc2.2bit # Counting myoLuc2.2bit # Writing jkStuff/myoLuc2.11.ooc # Wrote 7846 overused 11-mers to jkStuff/myoLuc2.11.ooc # Done making jkStuff/myoLuc2.11.ooc mkdir /hive/data/staging/data/myoLuc2 cp -p myoLuc2.2bit jkStuff/myoLuc2.11.ooc /hive/data/staging/data/myoLuc2 cp -p chrom.sizes /hive/data/staging/data/myoLuc2 # check for non-bridged gaps to see what the typical size is: hgsql -e "select bridge from gap;" myoLuc2 | sort | uniq # bridge # yes # all gap are bridged, done # ask cluster-admin to copy (evry time if any file chsnged) # /hive/data/staging/data/myoLuc2 directory to cluster nodes # /scratch/data/myoLuc2 ######################################################################## # GENBANK AUTO UPDATE (DONE 2011-02-15 - Chin) ssh hgwdev cd $HOME/kent/src/hg/makeDb/genbank git pull # edit etc/genbank.conf to add myoLuc2 just after ponAbe2 # myoLuc2 (Microbat) myoLuc2.serverGenome = /hive/data/genomes/myoLuc2/myoLuc2.2bit myoLuc2.clusterGenome = /scratch/data/myoLuc2/myoLuc2.2bit myoLuc2.ooc = /scratch/data/myoLuc2/myoLuc2.11.ooc myoLuc2.lift = no myoLuc2.perChromTables = no myoLuc2.refseq.mrna.native.pslCDnaFilter = ${ordered.refseq.mrna.native.pslCDnaFilter} myoLuc2.refseq.mrna.xeno.pslCDnaFilter = ${ordered.refseq.mrna.xeno.pslCDnaFilter} myoLuc2.genbank.mrna.native.pslCDnaFilter = ${ordered.genbank.mrna.native.pslCDnaFilter} myoLuc2.genbank.mrna.xeno.pslCDnaFilter = ${ordered.genbank.mrna.xeno.pslCDnaFilter} myoLuc2.downloadDir = myoLuc2 myoLuc2.refseq.mrna.native.load = no myoLuc2.refseq.mrna.xeno.load = yes myoLuc2.refseq.mrna.xeno.loadDesc = yes myoLuc2.genbank.mrna.native.load = yes myoLuc2.genbank.mrna.native.loadDesc = yes myoLuc2.genbank.mrna.xeno.load = yes myoLuc2.genbank.mrna.xeno.loadDesc = yes myoLuc2.genbank.est.native.load = no myoLuc2.genbank.est.native.loadDesc = no git add etc/genbank.conf git commit -m "Added myoLuc2" etc/genbank.conf git pull git push # update /cluster/data/genbank/: make etc-update # Edit src/lib/gbGenome.c to add new species. With these two lines: # static char *myoLucNames[] = {"Myotis lucifugus", NULL}; # ... later ... # {"myoLuc", myoLucNames}, # gbGenome.c is in # /cluster/home/chinhli/kent/src/hg/makeDb/genbank/src/lib # make and checkin git add src/lib/gbGenome.c git commit -m "adding myoLuc2 microbat" src/lib/gbGenome.c git pull git push make install-server ssh genbank screen # control this business with a screen since it takes a while cd /cluster/data/genbank time nice -n +19 ./bin/gbAlignStep -initial myoLuc2 & # logFile: var/build/logs/2010.11.04-14:14:12.myoLuc2.initalign.log # real 185m34.759s # To re-do, rm the dir first: # /cluster/data/genbank/data/aligned/genbank.176.0/myoLuc2 # load database when finished ssh hgwdev cd /cluster/data/genbank time nice -n +19 ./bin/gbDbLoadStep -drop -initialLoad myoLuc2 & # logFile: var/dbload/hgwdev/logs/2011.02.15-10:12:57.dbload.log # real 17m13.500s # rerun load step 2011-10-07 2011-10-11 # logFile: var/dbload/hgwdev/logs/2011.10.11-10:45:07.dbload.log # real 27m47.836s # enable daily alignment and update of hgwdev cd ~/kent/src/hg/makeDb/genbank git pull # add myoLuc2 to: etc/align.dbs etc/hgwdev.dbs git add etc/align.dbs git add etc/hgwdev.dbs git commit -m "Added myoLuc2 - Microbat" etc/align.dbs etc/hgwdev.dbs make etc-update ############################################################################ # myoLuc2 - Microbat - Ensembl Genes (pending - 2011-02-15 - Chin) # Ensembl Genes version 61 contains myoLuc1 assembly (Mar 2006) by Broad # Institute. . # The sequence downloaded from ncbi is Myoluc2.0 dated 2010-07-21 by Broad # Institute. # We need to wait for the newer release of Ensemble Genes. ######################################################################### # reset default position to RHO location as result from RHO gene search # (DONE 2011-02-15 - Chin) hgsql -e \ 'update dbDb set defaultPos="GL429775:12,072,919-12,378,895" where name="myoLuc2";' \ hgcentraltest # NOTE - create myoLuc2 entry in all.joiner since this is a new species ############################################################################ # running cpgIsland business (DONE -2011-05-16 - Chin) mkdir /hive/data/genomes/myoLuc2/bed/cpgIsland cd /hive/data/genomes/myoLuc2/bed/cpgIsland cvs -d /projects/compbio/cvsroot checkout -P hg3rdParty/cpgIslands cd hg3rdParty/cpgIslands # needed to fixup this source, adding include to readseq.c: #include "string.h" # and to cpg_lh.c: #include "unistd.h" #include "stdlib.h" # and fixing a declaration in cpg_lh.c sed -e "s#\(extern char\* malloc\)#// \1#" cpg_lh.c > tmp.c mv tmp.c cpg_lh.c make cd ../../ ln -s hg3rdParty/cpgIslands/cpglh.exe mkdir -p hardMaskedFa cut -f1 ../../chrom.sizes | while read C do echo ${C} twoBitToFa ../../myoLuc2.2bit:$C stdout \ | maskOutFa stdin hard hardMaskedFa/${C}.fa done ssh swarm cd /hive/data/genomes/myoLuc2/bed/cpgIsland mkdir results cut -f1 ../../chrom.sizes > chr.list cat << '_EOF_' > template #LOOP ./runOne $(root1) {check out exists results/$(root1).cpg} #ENDLOOP '_EOF_' # << happy emacs # the faCount business is to make sure there is enough sequence to # work with in the fasta. cpglh.exe does not like files with too # many # N's - it gets stuck cat << '_EOF_' > runOne #!/bin/csh -fe set C = `faCount hardMaskedFa/$1.fa | grep -e ^GL -e ^AAPE | awk '{print $2 - $7 }'` if ( $C > 200 ) then ./cpglh.exe hardMaskedFa/$1.fa > /scratch/tmp/$1.$$ mv /scratch/tmp/$1.$$ $2 else touch $2 endif '_EOF_' # << happy emacs chmod 775 runOne gensub2 chr.list single template jobList para create jobList para try para check ... etc para time para problems para status # then, kick it with para push # check it with plb # when all are done, para time shows: # Checking finished jobs # Completed: 11654 of 11654 jobs # CPU time in finished jobs: 145s 2.42m 0.04h 0.00d 0.000 y # IO & Wait Time: 29887s 498.11m 8.30h 0.35d 0.001 y # Average job time: 3s 0.04m 0.00h 0.00d # Longest finished job: 10s 0.17m 0.00h 0.00d # Submission to last job: 129s 2.15m 0.04h 0.00d # Transform cpglh output to bed + catDir results | awk '{ $2 = $2 - 1; width = $3 - $2; printf("%s\t%d\t%s\t%s %s\t%s\t%s\t%0.0f\t%0.1f\t%s\t%s\n", $1, $2, $3, $5,$6, width, $6, width*$7*0.01, 100.0*2*$6/width, $7, $9); }' > cpgIsland.bed ssh hgwdev cd /hive/data/genomes/myoLuc2/bed/cpgIsland hgLoadBed myoLuc2 cpgIslandExt -tab \ -sqlTable=$HOME/kent/src/hg/lib/cpgIslandExt.sql cpgIsland.bed # Loaded 53480 elements of size 10 # Sorted # Creating table definition for cpgIslandExt # Saving bed.tab # Loading myoLuc2 # cleanup rm -fr hardMaskedFa ######################################################################### # all.joiner update, downloads and in pushQ - (DONE 2011-06-30 - Chin) cd $HOME/kent/src/hg/makeDb/schema # fixup all.joiner until this is a clean output joinerCheck -database=myoLuc2 -all all.joiner mkdir /hive/data/genomes/myoLuc2/goldenPath cd /hive/data/genomes/myoLuc2/goldenPath makeDownloads.pl myoLuc2 > do.log 2>&1 # now ready for pushQ entry mkdir /hive/data/genomes/myoLuc2/pushQ cd /hive/data/genomes/myoLuc2/pushQ makePushQSql.pl myoLuc2 > myoLuc2.pushQ.sql 2> stderr.out # check for errors in stderr.out, some are OK, e.g.: # WARNING: hgwdev does not have /gbdb/myoLuc2/wib/gc5Base.wib # WARNING: hgwdev does not have /gbdb/myoLuc2/wib/quality.wib # WARNING: hgwdev does not have /gbdb/myoLuc2/bbi/quality.bw # WARNING: myoLuc2 does not have seq # WARNING: myoLuc2 does not have extFile # WARNING: myoLuc2 does not have estOrientInfo # # WARNING: Could not tell (from trackDb, all.joiner and hardcoded lists of # supporting and genbank tables) which tracks to assign these tables to: # tableList # copy it to hgwbeta scp -p myoLuc2.pushQ.sql hgwbeta:/tmp ssh hgwbeta cd /tmp hgsql qapushq < myoLuc2.pushQ.sql # in that pushQ entry walk through each entry and see if the # sizes will set properly ############################################################################ # HUMAN (hg18) PROTEINS TRACK (DONE 2011-09-02 braney ) # bash if not using bash shell already cd /cluster/data/myoLuc2 mkdir /cluster/data/myoLuc2/blastDb awk '{if ($2 > 1000000) print $1}' chrom.sizes > 1meg.lst twoBitToFa -seqList=1meg.lst myoLuc2.2bit temp.fa faSplit gap temp.fa 1000000 blastDb/x -lift=blastDb.lft # 2174 pieces of 2174 written rm temp.fa 1meg.lst awk '{if ($2 <= 1000000) print $1}' chrom.sizes > less1meg.lst twoBitToFa -seqList=less1meg.lst myoLuc2.2bit temp.fa faSplit about temp.fa 1000000 blastDb/y rm temp.fa less1meg.lst cd blastDb for i in *.fa do /hive/data/outside/blast229/formatdb -i $i -p F done rm *.fa ls *.nsq | wc -l # 2526 mkdir -p /cluster/data/myoLuc2/bed/tblastn.hg18KG cd /cluster/data/myoLuc2/bed/tblastn.hg18KG echo ../../blastDb/*.nsq | xargs ls -S | sed "s/\.nsq//" > query.lst wc -l query.lst # 2526 query.lst # we want around 250000 jobs calc `wc /cluster/data/hg18/bed/blat.hg18KG/hg18KG.psl | awk '{print $1}'`/\(250000/`wc query.lst | awk '{print $1}'`\) # 36727/(250000/2526) = 371.089608 mkdir -p kgfa split -l 371 /cluster/data/hg18/bed/blat.hg18KG/hg18KG.psl kgfa/kg cd kgfa for i in *; do nice pslxToFa $i $i.fa; rm $i; done cd .. ls -1S kgfa/*.fa > kg.lst wc kg.lst # 99 99 1287 kg.lst mkdir -p blastOut for i in `cat kg.lst`; do mkdir blastOut/`basename $i .fa`; done tcsh cd /cluster/data/myoLuc2/bed/tblastn.hg18KG cat << '_EOF_' > blastGsub #LOOP blastSome $(path1) {check in line $(path2)} {check out exists blastOut/$(root2)/q.$(root1).psl } #ENDLOOP '_EOF_' cat << '_EOF_' > blastSome #!/bin/sh BLASTMAT=/hive/data/outside/blast229/data export BLASTMAT g=`basename $2` f=/tmp/`basename $3`.$g for eVal in 0.01 0.001 0.0001 0.00001 0.000001 1E-09 1E-11 do if /hive/data/outside/blast229/blastall -M BLOSUM80 -m 0 -F no -e $eVal -p tblastn -d $1 -i $2 -o $f.8 then mv $f.8 $f.1 break; fi done if test -f $f.1 then if /cluster/bin/i386/blastToPsl $f.1 $f.2 then liftUp -nosort -type=".psl" -nohead $f.3 /cluster/data/myoLuc2/blastDb.lft carry $f.2 liftUp -nosort -type=".psl" -pslQ -nohead $3.tmp /cluster/data/hg18/bed/blat.hg18KG/protein.lft warn $f.3 if pslCheck -prot $3.tmp then mv $3.tmp $3 rm -f $f.1 $f.2 $f.3 $f.4 fi exit 0 fi fi rm -f $f.1 $f.2 $3.tmp $f.8 $f.3 $f.4 exit 1 '_EOF_' # << happy emacs chmod +x blastSome exit ssh swarm cd /cluster/data/myoLuc2/bed/tblastn.hg18KG gensub2 query.lst kg.lst blastGsub blastSpec para create blastSpec # para try, check, push, check etc. para time # Completed: 250074 of 250074 jobs # CPU time in finished jobs: 11896176s 198269.60m 3304.49h 137.69d 0.377 y # IO & Wait Time: 1490312s 24838.54m 413.98h 17.25d 0.047 y # Average job time: 54s 0.89m 0.01h 0.00d # Longest finished job: 255s 4.25m 0.07h 0.00d # Submission to last job: 13586s 226.43m 3.77h 0.16d ssh swarm cd /cluster/data/myoLuc2/bed/tblastn.hg18KG mkdir chainRun cd chainRun tcsh cat << '_EOF_' > chainGsub #LOOP chainOne $(path1) #ENDLOOP '_EOF_' cat << '_EOF_' > chainOne (cd $1; cat q.*.psl | simpleChain -prot -outPsl -maxGap=150000 stdin ../c.`basename $1`.psl) '_EOF_' chmod +x chainOne ls -1dS ../blastOut/kg?? > chain.lst gensub2 chain.lst single chainGsub chainSpec # do the cluster run for chaining para create chainSpec para try, check, push, check etc. # Completed: 99 of 99 jobs # CPU time in finished jobs: 6795s 113.26m 1.89h 0.08d 0.000 y # IO & Wait Time: 23935s 398.91m 6.65h 0.28d 0.001 y # Average job time: 310s 5.17m 0.09h 0.00d # Longest finished job: 531s 8.85m 0.15h 0.01d # Submission to last job: 538s 8.97m 0.15h 0.01d cd /cluster/data/myoLuc2/bed/tblastn.hg18KG/blastOut for i in kg?? do cat c.$i.psl | awk "(\$13 - \$12)/\$11 > 0.6 {print}" > c60.$i.psl sort -rn c60.$i.psl | pslUniq stdin u.$i.psl awk "((\$1 / \$11) ) > 0.60 { print }" c60.$i.psl > m60.$i.psl echo $i done sort u.*.psl m60* | uniq | sort -T /tmp -k 14,14 -k 16,16n -k 17,17n > ../blastHg18KG.psl cd .. pslCheck blastHg18KG.psl # checked: 70178 failed: 0 errors: 0 # load table ssh hgwdev cd /cluster/data/myoLuc2/bed/tblastn.hg18KG hgLoadPsl myoLuc2 blastHg18KG.psl # check coverage featureBits myoLuc2 blastHg18KG # 32772100 bases of 1966419868 (1.667%) in intersection featureBits myoLuc2 blastHg18KG xenoRefGene -enrichment # blastHg18KG 1.667%, xenoRefGene 2.563%, both 1.318%, cover 79.08%, enrich 30.86x rm -rf blastOut #end tblastn ############################################################################ # myoLuc2 - Microbat - Ensembl Genes version 64 (DONE - 2011-10-12 - hiram) ssh hgwdev cd /hive/data/genomes/myoLuc2 cat << '_EOF_' > myoLuc2.ensGene.ra # required db variable db myoLuc2 '_EOF_' # << happy emacs doEnsGeneUpdate.pl -ensVersion=64 myoLuc2.ensGene.ra ssh hgwdev cd /hive/data/genomes/myoLuc2/bed/ensGene.64 featureBits myoLuc2 ensGene # 32782563 bases of 1966419868 (1.667%) in intersection ############################################################################ # create ucscToINSDC name mapping (DONE - 2013-08-23 - Hiram) mkdir /hive/data/genomes/myoLuc2/bed/ucscToINSDC cd /hive/data/genomes/myoLuc2/bed/ucscToINSDC # copying these scripts from the previous load and improving them # with each instance ./translateNames.sh ./verifyAll.sh ./join.sh sed -e "s/21/12/" $HOME/kent/src/hg/lib/ucscToINSDC.sql \ | hgLoadSqlTab myoLuc2 ucscToINSDC stdin ucscToINSDC.tab checkTableCoords myoLuc2 ucscToINSDC featureBits -countGaps myoLuc2 ucscToINSDC # 2034575300 bases of 2034575300 (100.000%) in intersection # verify the track link to INSDC functions ############################################################################## # genscan - (DONE - 2013-11-12 - Hiram) mkdir /hive/data/genomes/myoLuc2/bed/genscan cd /hive/data/genomes/myoLuc2/bed/genscan time doGenscan.pl myoLuc2 > do.log 2>&1 # real 22m32.089s # 7 jobs broken, trying 2000000 window size ./lastRunGsBig.csh GL429777 000 gtf/000/GL429777.gtf pep/000/GL429777.pep subopt/000/GL429777.bed ./lastRunGsBig.csh GL429831 000 gtf/000/GL429831.gtf pep/000/GL429831.pep subopt/000/GL429831.bed ./lastRunGsBig.csh GL429775 000 gtf/000/GL429775.gtf pep/000/GL429775.pep subopt/000/GL429775.bed ./lastRunGsBig.csh GL429786 000 gtf/000/GL429786.gtf pep/000/GL429786.pep subopt/000/GL429786.bed ./lastRunGsBig.csh GL429768 000 gtf/000/GL429768.gtf pep/000/GL429768.pep subopt/000/GL429768.bed ./lastRunGsBig.csh GL429905 000 gtf/000/GL429905.gtf pep/000/GL429905.pep subopt/000/GL429905.bed ./lastRunGsBig.csh GL429863 000 gtf/000/GL429863.gtf pep/000/GL429863.pep subopt/000/GL429863.bed # real 16m15.818s # run 'para time > run.time' on ku, then continuing: time doGenscan.pl -continue=makeBed myoLuc2 > makeBed.log 2>&1 # real 6m23.659s cat fb.myoLuc2.genscan.txt # 65374897 bases of 1966419868 (3.325%) in intersection cat fb.myoLuc2.genscanSubopt.txt # 62128533 bases of 1966419868 (3.159%) in intersection ######################################################################### ############################################################################## # TransMap V3 tracks. see makeDb/doc/transMapTracks.txt (2014-12-21 markd) + ############################################################################## Changing name of organism: (DONE - Hiram - 2021-11-11) +# It was discovered that "microbat" is not is a common name, but a suborder +# of Microchiroptera consisting of ~1000 bat species. The correct +# common name for Myotis lucifugus is "Little brown bat" + hgsql -e 'update dbDb set organism="Little brown bat" where name="myoLuc2";' hgcentraltest hgsql -e 'update dbDb set genome="Little brown bat" where name="myoLuc2";' hgcentraltest hgsql -e 'update genomeClade set genome="Little brown bat" where genome="Microbat";' hgcentraltest hgsql -e 'update defaultDb set genome="Little brown bat" where name="myoLuc2";' hgcentraltest - -############################################################################## +#########################################################################