src/hg/makeDb/doc/ailMel1.txt 1.4
1.4 2010/02/12 23:57:29 hiram
rebuild Panda sequence with official genbank release
Index: src/hg/makeDb/doc/ailMel1.txt
===================================================================
RCS file: /projects/compbio/cvsroot/kent/src/hg/makeDb/doc/ailMel1.txt,v
retrieving revision 1.3
retrieving revision 1.4
diff -b -B -U 4 -r1.3 -r1.4
--- src/hg/makeDb/doc/ailMel1.txt 25 Jan 2010 19:43:27 -0000 1.3
+++ src/hg/makeDb/doc/ailMel1.txt 12 Feb 2010 23:57:29 -0000 1.4
@@ -8,8 +8,19 @@
##########################################################################
# Download sequence (DONE - 2009-12-17 - Hiram)
mkdir /hive/data/genomes/ailMel1
cd /hive/data/genomes/ailMel1
+ mkdir genbank
+ cd genbank
+
+ wget --timestamping -r --cut-dirs=6 --level=0 -nH -x \
+ --no-remove-listing -np \
+"ftp.ncbi.nlm.nih.gov:genbank/genomes/Eukaryotes/vertebrates_mammals/Ailuropoda_melanoleuca/AilMel_1.0/*"
+
+ # first pass attempt at sequence from public.genomics.org.cn
+ # 2009-12-17
+ # NCBI has cleaned the sequence and removed a couple of scaffolds that
+ # are cloning vector sequence
mkdir download
cd download
wget --timestamping \
@@ -45,50 +56,55 @@
assemblyDate Dec. 2009
assemblyLabel BGI-Shenzhen AilMel 1.0 Dec. 2009
orderKey 220
mitoAcc NC_009492
-fastaFiles /hive/data/genomes/ailMel1/download/scaffolds.fa.gz
-agpFiles /hive/data/genomes/ailMel1/download/fake.agp
+fastaFiles /hive/data/genomes/ailMel1/genbank/Primary_Assembly/unplaced_scaffolds/FASTA/unplaced.scaf.fa.gz
+agpFiles /hive/data/genomes/ailMel1/genbank/Primary_Assembly/unplaced_scaffolds/AGP/unplaced.scaf.agp.gz
# qualFiles none
dbDbSpeciesDir panda
taxId 9646
- makeGenomeDb.pl -stop=seq ailMel1.config.ra > seq.out 2>&1
+ makeGenomeDb.pl -workhorse=hgwdev -stop=seq ailMel1.config.ra > seq.out 2>&1
+# real 4m4.783s
makeGenomeDb.pl -start=agp -stop=agp ailMel1.config.ra > agp.out 2>&1
+# real 0m20.968s
makeGenomeDb.pl -start=db -stop=db ailMel1.config.ra > db.out 2>&1
+# real 6m28.168s
makeGenomeDb.pl -start=dbDb -stop=dbDb ailMel1.config.ra > dbDb.out 2>&1
makeGenomeDb.pl -start=trackDb -stop=trackDb ailMel1.config.ra > trackDb.out 2>&1
##########################################################################
-# running repeat masker (DONE - 2009-12-21 - Hiram)
+# running repeat masker (DONE - 2010-02-02 - Hiram)
mkdir /hive/data/genomes/ailMel1/bed/repeatMasker
cd /hive/data/genomes/ailMel1/bed/repeatMasker
doRepeatMasker.pl -buildDir=`pwd` -noSplit -bigClusterHub=swarm \
- -workhorse=hgwdev ailMel > do.log 2>&1
+ -workhorse=hgwdev ailMel1 > do.log 2>&1
cat faSize.rmsk.txt
-# 2279048486 bases (54196431 N's 2224852055 real 1356361826 upper
-# 868490229 lower) in 12195 sequences in 1 files
-# %38.11 masked total, %39.04 masked real
+# 2299509015 bases (54196184 N's 2245312831 real 1371210941 upper 874101890
+# lower) in 81467 sequences in 1 files
+# %38.01 masked total, %38.93 masked real
##########################################################################
-# running simple repeat (DONE - 2009-12-21 - Hiram)
+# running simple repeat (DONE - 2010-02-02 - Hiram)
mkdir /hive/data/genomes/ailMel1/bed/simpleRepeat
cd /hive/data/genomes/ailMel1/bed/simpleRepeat
- doSimpleRepeat.pl -buildDir=`pwd` -smallClusterHub=swarm \
- -workhorse=hgwdev ailMel > do.log 2>&1
-
+ time doSimpleRepeat.pl -buildDir=`pwd` -smallClusterHub=swarm \
+ -workhorse=hgwdev ailMel1 > do.log 2>&1 &
+# real 103m30.880s
cat fb.simpleRepeat
-# 24968229 bases of 2225124764 (1.122%) in intersection
+# 25064758 bases of 2245312831 (1.116%) in intersection
+ cd /hive/data/genomes/ailMel1
twoBitMask ailMel1.rmsk.2bit \
-add bed/simpleRepeat/trfMask.bed ailMel1.2bit
# you can safely ignore the warning about fields >= 13
twoBitToFa ailMel1.2bit stdout | faSize stdin > faSize.ailMel1.2bit.txt
cat faSize.ailMel1.2bit.txt
-# 2279048486 bases (54196431 N's 2224852055 real 1355773543 upper
-# 869078512 lower) in 12195 sequences in 1 files
-# %38.13 masked total, %39.06 masked real
+# 2299509015 bases (54196184 N's 2245312831 real 1370617891 upper
+# 874694940 lower) in 81467 sequences in 1 files
+# %38.04 masked total, %38.96 masked real
+
rm /gbdb/ailMel1/ailMel1.2bit
ln -s `pwd`/ailMel1.2bit /gbdb/ailMel1/ailMel1.2bit
##########################################################################
@@ -103,29 +119,29 @@
hgcentraltest
# test it with some sequence
#########################################################################
-# MAKE 11.OOC FILES FOR BLAT (DONE - 2009-12-23 - Hiram)
+# MAKE 11.OOC FILES FOR BLAT (DONE - 2010-02-03 - Hiram)
ssh kolossus
# numerator is ailMel1 gapless bases as reported by faSize
# denominator is hg17 gapless bases as reported by featureBits,
# 1024 is threshold used for human -repMatch:
- calc \( 2224852055 / 2897310462 \) \* 1024
-# ( 2224852055 / 2897310462 ) * 1024 = 786.332198
+ calc \( 2245312831 / 2897310462 \) \* 1024
+# ( 2245312831 / 2897310462 ) * 1024 = 793.563675
# ==> use -repMatch=750 according to size scaled down from 1024 for human.
# and rounded down to nearest 50
cd /hive/data/genomes/ailMel1
blat ailMel1.2bit /dev/null /dev/null -tileSize=11 \
-makeOoc=jkStuff/ailMel1.11.ooc -repMatch=750
- # Wrote 25042 overused 11-mers to jkStuff/ailMel1.11.ooc
+ # Wrote 25467 overused 11-mers to jkStuff/ailMel1.11.ooc
mkdir /hive/data/staging/data/ailMel
cp -p ailMel1.2bit chrom.sizes jkStuff/ailMel1.11.ooc \
/hive/data/staging/data/ailMel1
#########################################################################
-# genbank run
+# genbank run (DONE - 2010-02-04 - Hiram)
ssh hgwdev
cd $HOME/kent/src/hg/makeDb/genbank
# edit etc/genbank.conf to add this section just after canFam2:
# ailMel1 (panda)
@@ -157,14 +173,16 @@
ssh genbank
screen # control this business with a screen since it takes a while
cd /cluster/data/genbank
time nice -n +19 bin/gbAlignStep -initial ailMel1 &
- # logFile: var/build/logs/2009.12.24-15:03:26.ailMel1.initalign.log
+ # var/build/logs/2010.02.03-11:16:16.ailMel1.initalign.log
+ # real 326m58.232s
ssh hgwdev
cd /cluster/data/genbank
- nice ./bin/gbDbLoadStep -drop -initialLoad ailMel1 &
- # logFile: var/dbload/hgwdev/logs/2009.12.25-00:08:46.dbload.log
+ time ./bin/gbDbLoadStep -drop -initialLoad ailMel1 &
+ # logFile: var/dbload/hgwdev/logs/2010.02.04-11:12:16.dbload.log
+ # real 34m59.832s
# enable daily alignment and update of hgwdev
cd ~/kent/src/hg/makeDb/genbank
cvsup
@@ -178,8 +196,46 @@
############################################################################
# reset position to RHO location as found from blat of hg19 RHO gene
hgsql -e \
-'update dbDb set defaultPos="scaffold329:558576-566855" where name="ailMel1";' \
+'update dbDb set defaultPos="GL192818.1:558576-566855" where name="ailMel1";' \
hgcentraltest
############################################################################
+# LASTZ Opossum monDom5 (DONE - 2010-02-05 - Hiram)
+ # original alignment
+ cd /hive/data/genomes/monDom5/bed/lastzAilMel1.2010-02-04
+ cat fb.monDom5.chainAilMel1Link.txt
+ # 223510659 bases of 3501660299 (6.383%) in intersection
+
+ # and for the swap
+ mkdir /hive/data/genomes/ailMel1/bed/blastz.monDom5.swap
+ cd /hive/data/genomes/ailMel1/bed/blastz.monDom5.swap
+ time nice -n +19 doBlastzChainNet.pl -verbose=2 \
+ /hive/data/genomes/monDom5/bed/lastzAilMel1.2010-02-04/DEF \
+ -swap -noLoadChainSplit -syntenicNet \
+ -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
+ -chainMinScore=5000 -chainLinearGap=loose > swap.log 2>&1 &
+ # real 69m35.464s
+ cat fb.ailMel1.chainMonDom5Link.txt
+ # 211209682 bases of 2245312831 (9.407%) in intersection
+
+#########################################################################
+# LASTZ Mouse Mm9 (DONE - 2010-02-05 - Hiram)
+ # original alignment
+ cd /hive/data/genomes/mm9/bed/lastzAilMel1.2010-02-04
+ cat fb.mm9.chainAilMel1Link.txt
+ # 749595031 bases of 2620346127 (28.607%) in intersection
+
+ # and for the swap
+ mkdir /hive/data/genomes/ailMel1/bed/blastz.mm9.swap
+ cd /hive/data/genomes/ailMel1/bed/blastz.mm9.swap
+ time doBlastzChainNet.pl -verbose=2 \
+ /hive/data/genomes/mm9/bed/lastzAilMel1.2010-02-04/DEF \
+ -swap -noLoadChainSplit -bigClusterHub=swarm -smallClusterHub=memk \
+ -workhorse=hgwdev \
+ -chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
+ # real 54m57.140s
+ cat fb.ailMel1.chainMm9Link.txt
+ # 739076250 bases of 2245312831 (32.916%) in intersection
+
+#########################################################################