src/hg/makeDb/doc/ailMel1.txt 1.5
1.5 2010/03/30 16:01:48 chinhli
Adding BLASTZ, Chain/Net support
Index: src/hg/makeDb/doc/ailMel1.txt
===================================================================
RCS file: /projects/compbio/cvsroot/kent/src/hg/makeDb/doc/ailMel1.txt,v
retrieving revision 1.4
retrieving revision 1.5
diff -b -B -U 1000000 -r1.4 -r1.5
--- src/hg/makeDb/doc/ailMel1.txt 12 Feb 2010 23:57:29 -0000 1.4
+++ src/hg/makeDb/doc/ailMel1.txt 30 Mar 2010 16:01:48 -0000 1.5
@@ -1,241 +1,318 @@
# for emacs: -*- mode: sh; -*-
# $Id$
# Panda sequence: http://panda.genomics.org.cn/page/panda/download.jsp
# Ailuropoda melanoleuca
##########################################################################
# Download sequence (DONE - 2009-12-17 - Hiram)
mkdir /hive/data/genomes/ailMel1
cd /hive/data/genomes/ailMel1
mkdir genbank
cd genbank
wget --timestamping -r --cut-dirs=6 --level=0 -nH -x \
--no-remove-listing -np \
"ftp.ncbi.nlm.nih.gov:genbank/genomes/Eukaryotes/vertebrates_mammals/Ailuropoda_melanoleuca/AilMel_1.0/*"
# first pass attempt at sequence from public.genomics.org.cn
# 2009-12-17
# NCBI has cleaned the sequence and removed a couple of scaffolds that
# are cloning vector sequence
mkdir download
cd download
wget --timestamping \
'ftp://public.genomics.org.cn/BGI/panda/genome_sequence/panda.scafSeq.gapFilled.noMito.GeneScaffold.fa.gz'
wget --timestamping \
'ftp://public.genomics.org.cn/BGI/panda/genome_sequence/panda_contig100.mapping_scaffolds_nucmer.list.clean.tab.sort.gff.gz'
wget --timestamping \
'ftp://panda.genomics.org.cn/pub/panda/panda.scafSeq.gapFilled.noMito.GeneScaffold.fa.gz'
faCount panda.scafSeq.gapFilled.noMito.GeneScaffold.fa.gz > faCount.txt
grep "^scaff" faCount.txt | awk '{print $1}' > scaffold.list
faSomeRecords panda.scafSeq.gapFilled.noMito.GeneScaffold.fa.gz \
scaffold.list stdout | gzip > scaffolds.fa.gz
hgFakeAgp scaffolds.fa.gz stdout | sed -e "s/D/W/" > fake.agp
##########################################################################
# Initial genome build (DONE - 2009-12-17 - Hiram)
cd /hive/data/genomes/ailMel1
cat << '_EOF_' > ailMel1.config.ra
# Config parameters for makeGenomeDb.pl:
db ailMel1
clade mammal
genomeCladePriority 19
scientificName Ailuropoda melanoleuca
commonName Panda
assemblyDate Dec. 2009
assemblyLabel BGI-Shenzhen AilMel 1.0 Dec. 2009
orderKey 220
mitoAcc NC_009492
fastaFiles /hive/data/genomes/ailMel1/genbank/Primary_Assembly/unplaced_scaffolds/FASTA/unplaced.scaf.fa.gz
agpFiles /hive/data/genomes/ailMel1/genbank/Primary_Assembly/unplaced_scaffolds/AGP/unplaced.scaf.agp.gz
# qualFiles none
dbDbSpeciesDir panda
taxId 9646
makeGenomeDb.pl -workhorse=hgwdev -stop=seq ailMel1.config.ra > seq.out 2>&1
# real 4m4.783s
makeGenomeDb.pl -start=agp -stop=agp ailMel1.config.ra > agp.out 2>&1
# real 0m20.968s
makeGenomeDb.pl -start=db -stop=db ailMel1.config.ra > db.out 2>&1
# real 6m28.168s
makeGenomeDb.pl -start=dbDb -stop=dbDb ailMel1.config.ra > dbDb.out 2>&1
makeGenomeDb.pl -start=trackDb -stop=trackDb ailMel1.config.ra > trackDb.out 2>&1
##########################################################################
# running repeat masker (DONE - 2010-02-02 - Hiram)
mkdir /hive/data/genomes/ailMel1/bed/repeatMasker
cd /hive/data/genomes/ailMel1/bed/repeatMasker
doRepeatMasker.pl -buildDir=`pwd` -noSplit -bigClusterHub=swarm \
-workhorse=hgwdev ailMel1 > do.log 2>&1
cat faSize.rmsk.txt
# 2299509015 bases (54196184 N's 2245312831 real 1371210941 upper 874101890
# lower) in 81467 sequences in 1 files
# %38.01 masked total, %38.93 masked real
##########################################################################
# running simple repeat (DONE - 2010-02-02 - Hiram)
mkdir /hive/data/genomes/ailMel1/bed/simpleRepeat
cd /hive/data/genomes/ailMel1/bed/simpleRepeat
time doSimpleRepeat.pl -buildDir=`pwd` -smallClusterHub=swarm \
-workhorse=hgwdev ailMel1 > do.log 2>&1 &
# real 103m30.880s
cat fb.simpleRepeat
# 25064758 bases of 2245312831 (1.116%) in intersection
cd /hive/data/genomes/ailMel1
twoBitMask ailMel1.rmsk.2bit \
-add bed/simpleRepeat/trfMask.bed ailMel1.2bit
# you can safely ignore the warning about fields >= 13
twoBitToFa ailMel1.2bit stdout | faSize stdin > faSize.ailMel1.2bit.txt
cat faSize.ailMel1.2bit.txt
# 2299509015 bases (54196184 N's 2245312831 real 1370617891 upper
# 874694940 lower) in 81467 sequences in 1 files
# %38.04 masked total, %38.96 masked real
rm /gbdb/ailMel1/ailMel1.2bit
ln -s `pwd`/ailMel1.2bit /gbdb/ailMel1/ailMel1.2bit
##########################################################################
# BLATSERVERS ENTRY (DONE - 2009-12-23 - Hiram)
# After getting a blat server assigned by the Blat Server Gods,
ssh hgwdev
hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
VALUES ("ailMel1", "blat12", "17802", "1", "0"); \
INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
VALUES ("ailMel1", "blat12", "17803", "0", "1");' \
hgcentraltest
# test it with some sequence
#########################################################################
# MAKE 11.OOC FILES FOR BLAT (DONE - 2010-02-03 - Hiram)
ssh kolossus
# numerator is ailMel1 gapless bases as reported by faSize
# denominator is hg17 gapless bases as reported by featureBits,
# 1024 is threshold used for human -repMatch:
calc \( 2245312831 / 2897310462 \) \* 1024
# ( 2245312831 / 2897310462 ) * 1024 = 793.563675
# ==> use -repMatch=750 according to size scaled down from 1024 for human.
# and rounded down to nearest 50
cd /hive/data/genomes/ailMel1
blat ailMel1.2bit /dev/null /dev/null -tileSize=11 \
-makeOoc=jkStuff/ailMel1.11.ooc -repMatch=750
# Wrote 25467 overused 11-mers to jkStuff/ailMel1.11.ooc
mkdir /hive/data/staging/data/ailMel
cp -p ailMel1.2bit chrom.sizes jkStuff/ailMel1.11.ooc \
/hive/data/staging/data/ailMel1
#########################################################################
# genbank run (DONE - 2010-02-04 - Hiram)
ssh hgwdev
cd $HOME/kent/src/hg/makeDb/genbank
# edit etc/genbank.conf to add this section just after canFam2:
# ailMel1 (panda)
ailMel1.serverGenome = /hive/data/genomes/ailMel1/ailMel1.2bit
ailMel1.clusterGenome = /scratch/data/ailMel1/ailMel1.2bit
ailMel1.lift = no
ailMel1.ooc = /scratch/data/ailMel1/ailMel1.11.ooc
ailMel1.refseq.mrna.native.pslCDnaFilter = ${ordered.refseq.mrna.native.pslCDnaFilter}
ailMel1.refseq.mrna.xeno.pslCDnaFilter = ${ordered.refseq.mrna.xeno.pslCDnaFilter}
ailMel1.genbank.mrna.native.pslCDnaFilter = ${ordered.genbank.mrna.native.pslCDnaFilter}
ailMel1.genbank.mrna.xeno.pslCDnaFilter = ${ordered.genbank.mrna.xeno.pslCDnaFilter}
ailMel1.genbank.est.native.pslCDnaFilter = ${ordered.genbank.est.native.pslCDnaFilter}
ailMel1.refseq.mrna.native.load = yes
ailMel1.refseq.mrna.xeno.load = yes
ailMel1.genbank.mrna.xeno.load = yes
ailMel1.downloadDir = ailMel1
ailMel1.perChromTables = no
# Edit src/lib/gbGenome.c to add new species. With these four lines:
# static char *ailMelNames[] = {"Ailuropoda melanoleuca", "Canis familiaris",
# "Canis sp.", "Canis lupus familiaris",
# "Canis lupus", NULL};
# ... later ...
# {"ailMel", ailMelNames},
cvs ci -m "adding ailMel - Panda with Dog specifications" src/lib/gbGenome.c
make install-server
ssh genbank
screen # control this business with a screen since it takes a while
cd /cluster/data/genbank
time nice -n +19 bin/gbAlignStep -initial ailMel1 &
# var/build/logs/2010.02.03-11:16:16.ailMel1.initalign.log
# real 326m58.232s
ssh hgwdev
cd /cluster/data/genbank
time ./bin/gbDbLoadStep -drop -initialLoad ailMel1 &
# logFile: var/dbload/hgwdev/logs/2010.02.04-11:12:16.dbload.log
# real 34m59.832s
# enable daily alignment and update of hgwdev
cd ~/kent/src/hg/makeDb/genbank
cvsup
# add oryCun2 to:
etc/align.dbs
etc/hgwdev.dbs
cvs ci -m "Adding ailMel1 - Panda - Ailuropoda melanoleuca" \
etc/align.dbs etc/hgwdev.dbs
make etc-update
# done - 2010-01-19 - Hiram
############################################################################
# reset position to RHO location as found from blat of hg19 RHO gene
hgsql -e \
'update dbDb set defaultPos="GL192818.1:558576-566855" where name="ailMel1";' \
hgcentraltest
############################################################################
# LASTZ Opossum monDom5 (DONE - 2010-02-05 - Hiram)
# original alignment
cd /hive/data/genomes/monDom5/bed/lastzAilMel1.2010-02-04
cat fb.monDom5.chainAilMel1Link.txt
# 223510659 bases of 3501660299 (6.383%) in intersection
# and for the swap
mkdir /hive/data/genomes/ailMel1/bed/blastz.monDom5.swap
cd /hive/data/genomes/ailMel1/bed/blastz.monDom5.swap
time nice -n +19 doBlastzChainNet.pl -verbose=2 \
/hive/data/genomes/monDom5/bed/lastzAilMel1.2010-02-04/DEF \
-swap -noLoadChainSplit -syntenicNet \
-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
-chainMinScore=5000 -chainLinearGap=loose > swap.log 2>&1 &
# real 69m35.464s
cat fb.ailMel1.chainMonDom5Link.txt
# 211209682 bases of 2245312831 (9.407%) in intersection
#########################################################################
# LASTZ Mouse Mm9 (DONE - 2010-02-05 - Hiram)
# original alignment
cd /hive/data/genomes/mm9/bed/lastzAilMel1.2010-02-04
cat fb.mm9.chainAilMel1Link.txt
# 749595031 bases of 2620346127 (28.607%) in intersection
# and for the swap
mkdir /hive/data/genomes/ailMel1/bed/blastz.mm9.swap
cd /hive/data/genomes/ailMel1/bed/blastz.mm9.swap
time doBlastzChainNet.pl -verbose=2 \
/hive/data/genomes/mm9/bed/lastzAilMel1.2010-02-04/DEF \
-swap -noLoadChainSplit -bigClusterHub=swarm -smallClusterHub=memk \
-workhorse=hgwdev \
-chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
# real 54m57.140s
cat fb.ailMel1.chainMm9Link.txt
# 739076250 bases of 2245312831 (32.916%) in intersection
+#######################################################################
+# felCatV17e Cat BLASTZ/CHAIN/NET (working - 2010-03-22 - Chin)
+ screen # use a screen to manage this multi-day job
+ mkdir /hive/data/genomes/ailMel1/bed/lastzFelCatV17e.2010-03-22
+ cd /hive/data/genomes/ailMel1/bed/lastzFelCatV17e.2010-03-22
+
+ cat << '_EOF_' > DEF
+# human vs. cat
+# maximum M allowed with lastz is only 254
+BLASTZ_M=254
+
+# QUERY: Panda ailMel1
+SEQ2_DIR=/scratch/data/ailMel1/ailMel1.2bit
+SEQ2_LEN=/scratch/data/ailMel1/chrom.sizes
+SEQ2_CHUNK=10000000
+SEQ2_LAP=10000
+SEQ2_LIMIT=300
+
+# TARGET: Cat (felCatV17e)
+SEQ1_DIR=/scratch/data/felCatV17e/felCatV17e.2bit
+SEQ1_LEN=/scratch/data/felCatV17e/chrom.sizes
+SEQ1_LIMIT=50
+SEQ1_CHUNK=20000000
+SEQ1_LAP=10000
+
+BASE=/hive/data/genomes/ailMel1/bed/lastzFelCatV17e.2010-03-22
+TMPDIR=/scratch/tmp
+'_EOF_'
+ # << this line keeps emacs coloring happy
+
+ time nice -n +19 doBlastzChainNet.pl -verbose=2 \
+ `pwd`/DEF \
+ -syntenicNet -noDbNameCheck \
+ -chainMinScore=3000 -chainLinearGap=medium \
+ -workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
+ > do.log 2>&1 &
+ # Command failed:
+ # doClusterRun.csh
+ # manually run the doClusterRun.csh 03/28
+ time ./doClusterRun.csh > doClusterRun.log 2>&1 &
+ # real 93m42.604s
+ # check with counting file number in psl directory to make sure that
+ # all jobs were completed.
+ # manually start with "cat" step
+ time nice -n +19 doBlastzChainNet.pl -verbose=2 \
+ `pwd`/DEF \
+ -syntenicNet -continue=cat -noDbNameCheck \
+ -chainMinScore=3000 -chainLinearGap=medium \
+ -workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
+ > cat.log 2>&1 &
+ # real 346m8.431s
+ cat fb.felCatV17e.chainAilMel1Link.txt
+ # 1503647735 bases of 1990635005 (75.536%) in intersection
+
+
+ # need to change name back due to the seq1 and seq2 switch
+ # more TODO XXXX 03/29
+ # name ???cat fb.ailMel1.chainFelCatV17eLink.txt
+ # ???? 2047068864 bases of 2897316137 (70.654%) in intersection
+ mkdir /hive/data/genomes/felCatV17e/bed/blastz.ailMel1.swap
+ cd /hive/data/genomes/felCatV17e/bed/blastz.ailMel1.swap
+ time nice -n +19 doBlastzChainNet.pl -verbose=2 \
+ /hive/data/genomes/ailMel1/bed/lastzFelCatV17e.2010-03-22/DEF \
+ -swap -syntenicNet -noDbNameCheck \
+ -workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
+ -chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
+ # real 155m57.566s
+
+ # need to change name back due to the seq1 and seq2 switch
+ # more TODO XXXX 03/29
+ # so should be able to do
+ # cat fb.felCatV17e.chainAilMel1Link.txt
+ # without
+ # cd /hive/data/genomes/ailMel1/bed/blastz.felCatV17e.swap
+ cat fb.ailMel1.chainFelCatV17eLink.txt
+ # 1507273252 bases of 2245312831 (67.130%) in intersection
+
#########################################################################