Index: src/hg/makeDb/doc/ailMel1.txt
===================================================================
RCS file: /projects/compbio/cvsroot/kent/src/hg/makeDb/doc/ailMel1.txt,v
retrieving revision 1.7
retrieving revision 1.8
diff -b -B -U 1000000 -r1.7 -r1.8
--- src/hg/makeDb/doc/ailMel1.txt	7 Apr 2010 17:42:02 -0000	1.7
+++ src/hg/makeDb/doc/ailMel1.txt	7 Apr 2010 17:52:44 -0000	1.8
@@ -1,359 +1,360 @@
 # for emacs: -*- mode: sh; -*-
 
 #	$Id$
 
 # Panda sequence: http://panda.genomics.org.cn/page/panda/download.jsp
 #	Ailuropoda melanoleuca
 
 ##########################################################################
 # Download sequence (DONE - 2009-12-17 - Hiram)
     mkdir /hive/data/genomes/ailMel1
     cd /hive/data/genomes/ailMel1
     mkdir genbank
     cd genbank
 
     wget --timestamping -r --cut-dirs=6 --level=0 -nH -x \
 	--no-remove-listing -np \
 "ftp.ncbi.nlm.nih.gov:genbank/genomes/Eukaryotes/vertebrates_mammals/Ailuropoda_melanoleuca/AilMel_1.0/*"
 
     #	first pass attempt at sequence from public.genomics.org.cn
     #	2009-12-17
     #	NCBI has cleaned the sequence and removed a couple of scaffolds that
     #	are cloning vector sequence
     mkdir download
     cd download
 
 wget --timestamping \
 'ftp://public.genomics.org.cn/BGI/panda/genome_sequence/panda.scafSeq.gapFilled.noMito.GeneScaffold.fa.gz'
 
 wget --timestamping \
 'ftp://public.genomics.org.cn/BGI/panda/genome_sequence/panda_contig100.mapping_scaffolds_nucmer.list.clean.tab.sort.gff.gz'
 
 wget --timestamping \
 'ftp://panda.genomics.org.cn/pub/panda/panda.scafSeq.gapFilled.noMito.GeneScaffold.fa.gz'
 
 
     faCount panda.scafSeq.gapFilled.noMito.GeneScaffold.fa.gz > faCount.txt
 
     grep "^scaff" faCount.txt | awk '{print $1}' > scaffold.list
 
     faSomeRecords panda.scafSeq.gapFilled.noMito.GeneScaffold.fa.gz \
 	scaffold.list stdout | gzip > scaffolds.fa.gz
 
     hgFakeAgp scaffolds.fa.gz stdout | sed -e "s/D/W/" > fake.agp
 
 ##########################################################################
 # Initial genome build (DONE - 2009-12-17 - Hiram)
     cd /hive/data/genomes/ailMel1
 
     cat << '_EOF_' > ailMel1.config.ra
 # Config parameters for makeGenomeDb.pl:
 db ailMel1
 clade mammal
 genomeCladePriority 19
 scientificName Ailuropoda melanoleuca
 commonName Panda
 assemblyDate Dec. 2009
 assemblyLabel BGI-Shenzhen AilMel 1.0 Dec. 2009
 orderKey 220
 mitoAcc NC_009492
 fastaFiles /hive/data/genomes/ailMel1/genbank/Primary_Assembly/unplaced_scaffolds/FASTA/unplaced.scaf.fa.gz
 agpFiles /hive/data/genomes/ailMel1/genbank/Primary_Assembly/unplaced_scaffolds/AGP/unplaced.scaf.agp.gz
 # qualFiles none
 dbDbSpeciesDir panda
 taxId 9646
 
     makeGenomeDb.pl -workhorse=hgwdev -stop=seq ailMel1.config.ra > seq.out 2>&1
 #	real    4m4.783s
     makeGenomeDb.pl -start=agp -stop=agp ailMel1.config.ra > agp.out 2>&1
 #	real    0m20.968s
     makeGenomeDb.pl -start=db -stop=db ailMel1.config.ra > db.out 2>&1
 #	real    6m28.168s
     makeGenomeDb.pl -start=dbDb -stop=dbDb ailMel1.config.ra > dbDb.out 2>&1
     makeGenomeDb.pl -start=trackDb -stop=trackDb ailMel1.config.ra > trackDb.out 2>&1
 
 ##########################################################################
 # running repeat masker (DONE - 2010-02-02 - Hiram)
     mkdir /hive/data/genomes/ailMel1/bed/repeatMasker
     cd /hive/data/genomes/ailMel1/bed/repeatMasker
     doRepeatMasker.pl -buildDir=`pwd` -noSplit -bigClusterHub=swarm \
 	-workhorse=hgwdev ailMel1 > do.log 2>&1
     cat faSize.rmsk.txt
 # 2299509015 bases (54196184 N's 2245312831 real 1371210941 upper 874101890
 #	lower) in 81467 sequences in 1 files
 # %38.01 masked total, %38.93 masked real
 
 ##########################################################################
 # running simple repeat (DONE - 2010-02-02 - Hiram)
     mkdir /hive/data/genomes/ailMel1/bed/simpleRepeat
     cd /hive/data/genomes/ailMel1/bed/simpleRepeat
     time doSimpleRepeat.pl -buildDir=`pwd` -smallClusterHub=swarm \
 	-workhorse=hgwdev ailMel1 > do.log 2>&1 &
 #	real    103m30.880s
     cat fb.simpleRepeat 
 #	25064758 bases of 2245312831 (1.116%) in intersection
 
     cd /hive/data/genomes/ailMel1
     twoBitMask ailMel1.rmsk.2bit \
 	-add bed/simpleRepeat/trfMask.bed ailMel1.2bit
     #	you can safely ignore the warning about fields >= 13
 
     twoBitToFa ailMel1.2bit stdout | faSize stdin > faSize.ailMel1.2bit.txt
     cat faSize.ailMel1.2bit.txt
 # 2299509015 bases (54196184 N's 2245312831 real 1370617891 upper
 #	874694940 lower) in 81467 sequences in 1 files
 #	%38.04 masked total, %38.96 masked real
 
     rm /gbdb/ailMel1/ailMel1.2bit
     ln -s `pwd`/ailMel1.2bit /gbdb/ailMel1/ailMel1.2bit
 
 ##########################################################################
 #  BLATSERVERS ENTRY (DONE - 2009-12-23 - Hiram)
 #	After getting a blat server assigned by the Blat Server Gods,
     ssh hgwdev
 
     hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
 	VALUES ("ailMel1", "blat12", "17802", "1", "0"); \
 	INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
 	VALUES ("ailMel1", "blat12", "17803", "0", "1");' \
 	    hgcentraltest
     #	test it with some sequence
 
 #########################################################################
 # MAKE 11.OOC FILES FOR BLAT (DONE - 2010-02-03 - Hiram)
     ssh kolossus
     # numerator is ailMel1 gapless bases as reported by faSize 
     # denominator is hg17 gapless bases as reported by featureBits,
     # 1024 is threshold used for human -repMatch:
     calc \( 2245312831 / 2897310462 \) \* 1024
 #	( 2245312831 / 2897310462 ) * 1024 = 793.563675
     # ==> use -repMatch=750 according to size scaled down from 1024 for human.
     #	and rounded down to nearest 50
     cd /hive/data/genomes/ailMel1
     blat ailMel1.2bit /dev/null /dev/null -tileSize=11 \
       -makeOoc=jkStuff/ailMel1.11.ooc -repMatch=750
     #	Wrote 25467 overused 11-mers to jkStuff/ailMel1.11.ooc
 
     mkdir /hive/data/staging/data/ailMel
     cp -p ailMel1.2bit chrom.sizes jkStuff/ailMel1.11.ooc \
 	/hive/data/staging/data/ailMel1
 
 
 #########################################################################
 # genbank run (DONE - 2010-02-04 - Hiram)
     ssh hgwdev
     cd $HOME/kent/src/hg/makeDb/genbank
     # edit etc/genbank.conf to add this section just after canFam2:
 # ailMel1 (panda)
 ailMel1.serverGenome = /hive/data/genomes/ailMel1/ailMel1.2bit
 ailMel1.clusterGenome = /scratch/data/ailMel1/ailMel1.2bit
 ailMel1.lift = no
 ailMel1.ooc = /scratch/data/ailMel1/ailMel1.11.ooc
 ailMel1.refseq.mrna.native.pslCDnaFilter  = ${ordered.refseq.mrna.native.pslCDnaFilter}
 ailMel1.refseq.mrna.xeno.pslCDnaFilter    = ${ordered.refseq.mrna.xeno.pslCDnaFilter}
 ailMel1.genbank.mrna.native.pslCDnaFilter = ${ordered.genbank.mrna.native.pslCDnaFilter}
 ailMel1.genbank.mrna.xeno.pslCDnaFilter   = ${ordered.genbank.mrna.xeno.pslCDnaFilter}
 ailMel1.genbank.est.native.pslCDnaFilter  = ${ordered.genbank.est.native.pslCDnaFilter}
 ailMel1.refseq.mrna.native.load = yes
 ailMel1.refseq.mrna.xeno.load = yes
 ailMel1.genbank.mrna.xeno.load = yes
 ailMel1.downloadDir = ailMel1
 ailMel1.perChromTables = no
 
     # Edit src/lib/gbGenome.c to add new species.  With these four lines:
 # static char *ailMelNames[] = {"Ailuropoda melanoleuca", "Canis familiaris",
 #                                "Canis sp.", "Canis lupus familiaris",
 #                                "Canis lupus", NULL};
 #   ... later ...
 # {"ailMel", ailMelNames},
 
     cvs ci -m "adding ailMel - Panda with Dog specifications" src/lib/gbGenome.c
     make install-server
 
     ssh genbank
     screen  # control this business with a screen since it takes a while
     cd /cluster/data/genbank
     time nice -n +19 bin/gbAlignStep -initial  ailMel1 &
     #	var/build/logs/2010.02.03-11:16:16.ailMel1.initalign.log
     #	real    326m58.232s
 
     ssh hgwdev
     cd /cluster/data/genbank
     time ./bin/gbDbLoadStep -drop -initialLoad ailMel1 &
     #	logFile: var/dbload/hgwdev/logs/2010.02.04-11:12:16.dbload.log
     #	real    34m59.832s
 
     # enable daily alignment and update of hgwdev
     cd ~/kent/src/hg/makeDb/genbank
     cvsup
     # add oryCun2 to:
         etc/align.dbs
         etc/hgwdev.dbs
     cvs ci -m "Adding ailMel1 - Panda - Ailuropoda melanoleuca" \
 	etc/align.dbs etc/hgwdev.dbs
     make etc-update
     #	done - 2010-01-19 - Hiram
 
 ############################################################################
 # reset position to RHO location as found from blat of hg19 RHO gene
     hgsql -e \
 'update dbDb set defaultPos="GL192818.1:558576-566855" where name="ailMel1";' \
 	hgcentraltest
 
 ############################################################################
 # LASTZ Opossum monDom5 (DONE - 2010-02-05 - Hiram)
     #	original alignment
     cd /hive/data/genomes/monDom5/bed/lastzAilMel1.2010-02-04
     cat fb.monDom5.chainAilMel1Link.txt 
     #	223510659 bases of 3501660299 (6.383%) in intersection
 
     #	and for the swap
     mkdir /hive/data/genomes/ailMel1/bed/blastz.monDom5.swap
     cd /hive/data/genomes/ailMel1/bed/blastz.monDom5.swap
     time nice -n +19 doBlastzChainNet.pl -verbose=2 \
 	/hive/data/genomes/monDom5/bed/lastzAilMel1.2010-02-04/DEF \
 	-swap -noLoadChainSplit -syntenicNet \
 	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
 	-chainMinScore=5000 -chainLinearGap=loose > swap.log 2>&1 &
     #	real    69m35.464s
     cat fb.ailMel1.chainMonDom5Link.txt 
     #	211209682 bases of 2245312831 (9.407%) in intersection
 
 #########################################################################
 # LASTZ Mouse Mm9 (DONE - 2010-02-05 - Hiram)
     #	original alignment
     cd /hive/data/genomes/mm9/bed/lastzAilMel1.2010-02-04
     cat fb.mm9.chainAilMel1Link.txt 
     #	749595031 bases of 2620346127 (28.607%) in intersection
 
     #	and for the swap
     mkdir /hive/data/genomes/ailMel1/bed/blastz.mm9.swap
     cd /hive/data/genomes/ailMel1/bed/blastz.mm9.swap
     time doBlastzChainNet.pl -verbose=2 \
 	/hive/data/genomes/mm9/bed/lastzAilMel1.2010-02-04/DEF \
 	-swap -noLoadChainSplit -bigClusterHub=swarm -smallClusterHub=memk \
 	-workhorse=hgwdev \
 	-chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
     #	real    54m57.140s
     cat fb.ailMel1.chainMm9Link.txt 
     #	739076250 bases of 2245312831 (32.916%) in intersection
 
 #######################################################################
 # felCatV17e Cat BLASTZ/CHAIN/NET (working  - 2010-03-22 - Chin)
     screen # use a screen to manage this multi-day job
     mkdir /hive/data/genomes/ailMel1/bed/lastzFelCatV17e.2010-03-22
     cd /hive/data/genomes/ailMel1/bed/lastzFelCatV17e.2010-03-22
 
     cat << '_EOF_' > DEF
 # human vs. cat
 # maximum M allowed with lastz is only 254
 BLASTZ_M=254
 
 # QUERY: Panda ailMel1
 SEQ2_DIR=/scratch/data/ailMel1/ailMel1.2bit
 SEQ2_LEN=/scratch/data/ailMel1/chrom.sizes
 SEQ2_CHUNK=10000000
 SEQ2_LAP=10000
 SEQ2_LIMIT=300
 
 # TARGET: Cat (felCatV17e)
 SEQ1_DIR=/scratch/data/felCatV17e/felCatV17e.2bit
 SEQ1_LEN=/scratch/data/felCatV17e/chrom.sizes
 SEQ1_LIMIT=50
 SEQ1_CHUNK=20000000
 SEQ1_LAP=10000
 
 BASE=/hive/data/genomes/ailMel1/bed/lastzFelCatV17e.2010-03-22
 TMPDIR=/scratch/tmp
 '_EOF_'
     # << this line keeps emacs coloring happy
 
     time nice -n +19 doBlastzChainNet.pl -verbose=2 \
         `pwd`/DEF \
         -syntenicNet -noDbNameCheck \
         -chainMinScore=3000 -chainLinearGap=medium \
         -workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
         > do.log 2>&1 &
     #  Command failed:
     #     doClusterRun.csh
     # manually run the doClusterRun.csh  03/28
     time ./doClusterRun.csh > doClusterRun.log 2>&1 &
     # real    93m42.604s
     #  check with counting file number in psl directory to make sure that
     #  all jobs were completed.
     # manually start with "cat" step
     time nice -n +19 doBlastzChainNet.pl -verbose=2 \
         `pwd`/DEF \
         -syntenicNet -continue=cat -noDbNameCheck \
         -chainMinScore=3000 -chainLinearGap=medium \
         -workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
         > cat.log 2>&1 &
     # real    346m8.431s
     cat fb.felCatV17e.chainAilMel1Link.txt
     # 1503647735 bases of 1990635005 (75.536%) in intersection
 
 
     mkdir /hive/data/genomes/felCatV17e/bed/blastz.ailMel1.swap
     cd /hive/data/genomes/felCatV17e/bed/blastz.ailMel1.swap
     time nice -n +19 doBlastzChainNet.pl -verbose=2 \
         /hive/data/genomes/ailMel1/bed/lastzFelCatV17e.2010-03-22/DEF \
         -swap -syntenicNet -noDbNameCheck \
         -workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
         -chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
     # real    155m57.566s
 
     # need to change name back due to the seq1 and seq2 switch
     # more TODO  XXXX 03/29
     # so should be able to do
     #	cat fb.felCatV17e.chainAilMel1Link.txt
     # without 
     # cd /hive/data/genomes/ailMel1/bed/blastz.felCatV17e.swap
     cat fb.ailMel1.chainFelCatV17eLink.txt
     # 1507273252 bases of 2245312831 (67.130%) in intersection
 
 #########################################################################
 # all.joiner update, downloads and in pushQ - (DONE - 2010-04-01 - Hiram)
     cd $HOME/kent/src/hg/makeDb/schema
     # fixup all.joiner until this is a clean output
     joinerCheck -database=ailMel1 -all all.joiner
 
     mkdir /hive/data/genomes/ailMel1/goldenPath
     cd /hive/data/genomes/ailMel1/goldenPath
     makeDownloads.pl ailMel1 > do.log 2>&1
 
     #	now ready for pushQ entry
     mkdir /hive/data/genomes/ailMel1/pushQ
     cd /hive/data/genomes/ailMel1/pushQ
     makePushQSql.pl ailMel1 > ailMel1.pushQ.sql 2> stderr.out
     #	check for errors in stderr.out, some are OK, e.g.:
 # WARNING: ailMel1 does not have seq
 # WARNING: ailMel1 does not have extFile
 
 # WARNING: Could not tell (from trackDb, all.joiner and hardcoded lists of
 # supporting and genbank tables) which tracks to assign these tables to:
 #  bosTau4ChainPileUp
 
     #	copy it to hgwbeta
     scp -p ailMel1.pushQ.sql hgwbeta:/tmp
     ssh hgwbeta
     cd /tmp
     hgsql qapushq < ailMel1.pushQ.sql
     #	in that pushQ entry walk through each entry and see if the
     #	sizes will set properly
 
 ############################################################################
 # Marking *all* gaps - they are not all in the AGP file
 #	(DONE - 2010-03-25 - Hiram)
     mkdir /hive/data/genomes/ailMel1/bed/allGaps
     cd /hive/data/genomes/ailMel1/bed/allGaps
     time nice -n +19 findMotif -motif=gattaca -verbose=4 \
 	-strand=+ ../../ailMel1.2bit > findMotif.txt 2>&1
     #	real    1m12.153s
     grep "^#GAP " findMotif.txt | sed -e "s/^#GAP //" > allGaps.bed
     featureBits ailMel1 -not gap -bed=notGap.bed
     #	2245312831 bases of 2245312831 (100.000%) in intersection
     featureBits ailMel1 allGaps.bed notGap.bed -bed=new.gaps.bed
     #	0 bases of 2245312831 (0.000%) in intersection
+    #	real    84m58.382s
     #	no new gaps
 
 #########################################################################

