src/hg/makeDb/doc/calJac3.txt 1.3
1.3 2010/02/16 04:54:18 hiram
done with a number of lastz runs
Index: src/hg/makeDb/doc/calJac3.txt
===================================================================
RCS file: /projects/compbio/cvsroot/kent/src/hg/makeDb/doc/calJac3.txt,v
retrieving revision 1.2
retrieving revision 1.3
diff -b -B -U 4 -r1.2 -r1.3
--- src/hg/makeDb/doc/calJac3.txt 15 Feb 2010 18:08:23 -0000 1.2
+++ src/hg/makeDb/doc/calJac3.txt 16 Feb 2010 04:54:18 -0000 1.3
@@ -60,9 +60,385 @@
echo ">${chrN}" > ${chrN}.fa
zcat ${AC}/FASTA/${chrN}.fa.gz | grep -v "^>" >> ${chrN}.fa
done
+##########################################################################
+# Initial genome build (DONE - 2009-12-17 - Hiram)
+ cd /hive/data/genomes/calJac3
+
+ cat << '_EOF_' > calJac3.config.ra
+# Config parameters for makeGenomeDb.pl:
+db calJac3
+clade mammal
+genomeCladePriority 16
+scientificName Callithrix jacchus
+commonName Marmoset
+assemblyDate Mar. 2009
+assemblyLabel WUSTL 3.2 (GCA_000004665.1)
+orderKey 40
+mitoAcc none
+fastaFiles /hive/data/genomes/calJac3/genbank/ucscChr/*.fa
+agpFiles /hive/data/genomes/calJac3/genbank/ucscChr/*.agp
+# qualFiles none
+dbDbSpeciesDir marmoset
+taxId 9483
+'_EOF_'
+
+ makeGenomeDb.pl -workhorse=hgwdev -stop=seq calJac3.config.ra > seq.out 2>&1
+# real 4m5.924s
+ makeGenomeDb.pl -continue=agp -stop=agp calJac3.config.ra > agp.out 2>&1
+# real 0m20.968s
+ makeGenomeDb.pl -continue=db -stop=db calJac3.config.ra > db.out 2>&1
+# real 5m39.181s
+XXX - chromInfo doesn't have large enough fields for the name keys
+ makeGenomeDb.pl -continue=dbDb -stop=dbDb calJac3.config.ra > dbDb.out 2>&1
+ makeGenomeDb.pl -continue=trackDb -stop=trackDb calJac3.config.ra > trackDb.out 2>&1
+
+##########################################################################
+# running repeat masker (DONE - 2010-02-02 - Hiram)
+ mkdir /hive/data/genomes/calJac3/bed/repeatMasker
+ cd /hive/data/genomes/calJac3/bed/repeatMasker
+ time doRepeatMasker.pl -buildDir=`pwd` -noSplit \
+ -bigClusterHub=swarm -dbHost=hgwdev -workhorse=hgwdev \
+ -smallClusterHub=memk calJac3 > do.log 2>&1 &
+ # real 443m55.891s
+ cat faSize.rmsk.txt
+# 2914958544 bases (162452744 N's 2752505800 real 1439949519 upper 1312556281
+# lower) in 14205 sequences in 1 files
+# %45.03 masked total, %47.69 masked real
+
+##########################################################################
+# running simple repeat (DONE - 2010-02-02 - Hiram)
+ mkdir /hive/data/genomes/calJac3/bed/simpleRepeat
+ cd /hive/data/genomes/calJac3/bed/simpleRepeat
+ time doSimpleRepeat.pl -buildDir=`pwd` -bigClusterHub=swarm \
+ -dbHost=hgwdev -workhorse=hgwdev -smallClusterHub=memk \
+ calJac3 > do.log 2>&1 &
+ # real 198m33.953s
+ cat fb.simpleRepeat
+ # 66211508 bases of 2752505800 (2.405%) in intersection
+
+ cd /hive/data/genomes/calJac3
+ twoBitMask calJac3.rmsk.2bit \
+ -add bed/simpleRepeat/trfMask.bed calJac3.2bit
+ # you can safely ignore the warning about fields >= 13
+
+ twoBitToFa calJac3.2bit stdout | faSize stdin > faSize.calJac3.2bit.txt
+ cat faSize.calJac3.2bit.txt
+# 2914958544 bases (162452744 N's 2752505800 real 1439244378 upper
+# 1313261422 lower) in 14205 sequences in 1 files
+# %45.05 masked total, %47.71 masked real
+
+ rm /gbdb/calJac3/calJac3.2bit
+ ln -s `pwd`/calJac3.2bit /gbdb/calJac3/calJac3.2bit
+
#########################################################################
+# MAKE 11.OOC FILES FOR BLAT (DONE - 2010-02-03 - Hiram)
+ ssh kolossus
+ # numerator is calJac3 gapless bases "real" as reported by faSize
+ # denominator is hg17 gapless bases as reported by featureBits,
+ # 1024 is threshold used for human -repMatch:
+ calc \( 2752505800 / 2897310462 \) \* 1024
+ # ( 2752505800 / 2897310462 ) * 1024 = 972.821510
+ # ==> use -repMatch=950 according to size scaled down from 1024 for human.
+ # and rounded down to nearest 50
+ cd /hive/data/genomes/calJac3
+ blat calJac3.2bit /dev/null /dev/null -tileSize=11 \
+ -makeOoc=jkStuff/calJac3.11.ooc -repMatch=950
+ # Wrote 32908 overused 11-mers to jkStuff/calJac3.11.ooc
+
+ mkdir /hive/data/staging/data/calJac3
+ cp -p calJac3.2bit chrom.sizes jkStuff/calJac3.11.ooc \
+ /hive/data/staging/data/calJac3
+
+ gapToLift -bedFile=jkStuff/nonBridgedGaps.bed calJac3 \
+ jkStuff/calJac3.nonBridged.lft
+
+##########################################################################
+# BLATSERVERS ENTRY (DONE - 2009-12-23 - Hiram)
+# After getting a blat server assigned by the Blat Server Gods,
+ ssh hgwdev
+
+ hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
+ VALUES ("calJac3", "blat12", "17804", "1", "0"); \
+ INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
+ VALUES ("calJac3", "blat12", "17805", "0", "1");' \
+ hgcentraltest
+ # test it with some sequence
+
+############################################################################
+# reset position to RHO location as found from blat of hg19 RHO gene
+ hgsql -e \
+'update dbDb set defaultPos="chr15:67439781-67471107" where name="calJac3";' \
+ hgcentraltest
+
+############################################################################
+# genbank run (DONE - 2010-02-04 - Hiram)
+ ssh hgwdev
+ cd $HOME/kent/src/hg/makeDb/genbank
+ # edit etc/genbank.conf to add this section just before calJac1:
+# Marmoset
+calJac3.serverGenome = /hive/data/genomes/calJac3/calJac3.2bit
+calJac3.clusterGenome = /scratch/data/calJac3/calJac3.2bit
+calJac3.ooc = /scratch/data/calJac3/calJac3.11.ooc
+calJac3.lift = no
+calJac3.perChromTables = no
+calJac3.refseq.mrna.native.pslCDnaFilter = ${ordered.refseq.mrna.native.pslCDnaFilter}
+calJac3.refseq.mrna.xeno.pslCDnaFilter = ${ordered.refseq.mrna.xeno.pslCDnaFilter}
+calJac3.genbank.mrna.native.pslCDnaFilter = ${ordered.genbank.mrna.native.pslCDnaFilter}
+calJac3.genbank.mrna.xeno.pslCDnaFilter = ${ordered.genbank.mrna.xeno.pslCDnaFilter}
+calJac3.genbank.est.native.pslCDnaFilter = ${ordered.genbank.est.native.pslCDnaFilter}
+calJac3.genbank.est.xeno.pslCDnaFilter = ${ordered.genbank.est.xeno.pslCDnaFilter}
+calJac3.downloadDir = calJac3
+calJac3.refseq.mrna.native.load = yes
+calJac3.refseq.mrna.xeno.load = yes
+calJac3.refseq.mrna.xeno.loadDesc = yes
+
+ cvs ci -m "adding marmoset calJac3" etc/genbank.conf
+ make etc-update
+
+ ssh genbank
+ screen # control this business with a screen since it takes a while
+ cd /cluster/data/genbank
+ time nice -n +19 bin/gbAlignStep -initial calJac3 &
+ # var/build/logs/2010.02.10-09:19:20.calJac3.initalign.log
+ # real 238m59.390s
+
+ ssh hgwdev
+ cd /cluster/data/genbank
+ time ./bin/gbDbLoadStep -drop -initialLoad calJac3 &
+ # logFile: var/dbload/hgwdev/logs/2010.02.11-10:41:22.dbload.log
+ # real 25m55.474s
+
+ # enable daily alignment and update of hgwdev
+ cd ~/kent/src/hg/makeDb/genbank
+ cvsup
+ # add calJac3 to:
+ etc/align.dbs
+ etc/hgwdev.dbs
+ cvs ci -m "Adding calJac3 - Marmoset - Callithrix jacchus" \
+ etc/align.dbs etc/hgwdev.dbs
+ make etc-update
+ # done - 2010-02-11 - Hiram
+
+############################################################################
+# running cpgIsland business (DONE - 2010-02-11 - Hiram)
+ mkdir /hive/data/genomes/calJac3/bed/cpgIsland
+ cd /hive/data/genomes/calJac3/bed/cpgIsland
+ cvs -d /projects/compbio/cvsroot checkout -P hg3rdParty/cpgIslands
+ cd hg3rdParty/cpgIslands
+ # needed to fixup this source, adding include to readseq.c:
+#include "string.h"
+ # and to cpg_lh.c:
+#include "unistd.h"
+#include "stdlib.h"
+ # and fixing a declaration in cpg_lh.c
+ sed -e "s#\(extern char\* malloc\)#// \1#" cpg_lh.c > tmp.c
+ mv tmp.c cpg_lh.c
+ make
+ cd ../../
+ ln -s hg3rdParty/cpgIslands/cpglh.exe
+ mkdir -p hardMaskedFa
+ cut -f1 ../../chrom.sizes | while read C
+do
+ echo ${C}
+ twoBitToFa ../../calJac3.2bit:$C stdout \
+ | maskOutFa stdin hard hardMaskedFa/${C}.fa
+done
+
+ ssh swarm
+ cd /hive/data/genomes/calJac3/bed/cpgIsland
+ mkdir results
+ cut -f1 ../../chrom.sizes > chr.list
+ cat << '_EOF_' > template
+#LOOP
+./runOne $(root1) {check exists results/$(root1).cpg}
+#ENDLOOP
+'_EOF_'
+ # << happy emacs
+
+ # the faCount business is to make sure there is enough sequence to
+ # work with in the fasta. cpglh.exe does not like files with too many
+ # N's - it gets stuck
+ cat << '_EOF_' > runOne
+#!/bin/csh -fe
+set C = `faCount hardMaskedFa/$1.fa | grep ^chr | awk '{print $2 - $7 }'`
+if ( $C > 200 ) then
+ ./cpglh.exe hardMaskedFa/$1.fa > /scratch/tmp/$1.$$
+ mv /scratch/tmp/$1.$$ $2
+else
+ touch $2
+endif
+'_EOF_'
+ # << happy emacs
+
+ gensub2 chr.list single template jobList
+ para create jobList
+ para try
+ para check ... etc
+ para time
+# Completed: 14205 of 14205 jobs
+# CPU time in finished jobs: 205s 3.41m 0.06h 0.00d 0.000 y
+# IO & Wait Time: 38701s 645.02m 10.75h 0.45d 0.001 y
+# Average job time: 3s 0.05m 0.00h 0.00d
+# Longest finished job: 23s 0.38m 0.01h 0.00d
+# Submission to last job: 202s 3.37m 0.06h 0.00d
+
+ # Transform cpglh output to bed +
+ catDir results | awk '{
+$2 = $2 - 1;
+width = $3 - $2;
+printf("%s\t%d\t%s\t%s %s\t%s\t%s\t%0.0f\t%0.1f\t%s\t%s\n",
+ $1, $2, $3, $5,$6, width,
+ $6, width*$7*0.01, 100.0*2*$6/width, $7, $9);
+}' > cpgIsland.bed
+
+ cd /hive/data/genomes/calJac3/bed/cpgIsland
+ hgLoadBed calJac3 cpgIslandExt -tab \
+ -sqlTable=$HOME/kent/src/hg/lib/cpgIslandExt.sql cpgIsland.bed
+
+# Reading cpgIsland.bed
+# Loaded 32732 elements of size 10
+# Sorted
+# Saving bed.tab
+# Loading calJac3
+
+ # cleanup
+ rm -fr hardMaskedFa
+
+############################################################################
+# LASTZ Human Swap (DONE - 2010-02-11 - Hiram)
+ # original alignment on hg19:
+ cd /hive/data/genomes/hg19/bed/lastzCalJac3.2010-02-11
+ cat fb.hg19.chainCalJac3Link.txt
+ # 2047068864 bases of 2897316137 (70.654%) in intersection
+
+ # and for this swap
+ mkdir /hive/data/genomes/calJac3/bed/blastz.hg19.swap
+ cd /hive/data/genomes/calJac3/bed/blastz.hg19.swap
+ time nice -n +19 doBlastzChainNet.pl -verbose=2 \
+ /hive/data/genomes/hg19/bed/lastzCalJac3.2010-02-11/DEF \
+ -swap -syntenicNet \
+ -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
+ -chainMinScore=5000 -chainLinearGap=medium > swap.log 2>&1 &
+ # real 120m42.991s
+ cat fb.calJac3.chainHg19Link.txt
+ # 2030475813 bases of 2752505800 (73.768%) in intersection
+
+#####################################################################
+# LASTZ Rhesus Swap (DONE - 2010-02-11 - Hiram)
+ # original alignment to rheMac2
+ cd /hive/data/genomes/rheMac2/bed/lastzCalJac3.2010-02-11
+ cat fb.rheMac2.chainCalJac3Link.txt
+ # 1871513554 bases of 2646704109 (70.711%) in intersection
+
+ # and for this swap
+ mkdir /hive/data/genomes/calJac3/bed/blastz.rheMac2.swap
+ cd /hive/data/genomes/calJac3/bed/blastz.rheMac2.swap
+ time nice -n +19 doBlastzChainNet.pl -verbose=2 \
+ /hive/data/genomes/rheMac2/bed/lastzCalJac3.2010-02-11/DEF \
+ -swap -syntenicNet \
+ -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
+ -chainMinScore=5000 -chainLinearGap=medium > swap.log 2>&1 &
+ # real 142m34.894s
+ cat fb.calJac3.chainHg19Link.txt
+ # 1916431926 bases of 2752505800 (69.625%) in intersection
+
+############################################################################
+# LASTZ Chimp Swap (DONE - 2010-02-11 - Hiram)
+ # original alignment to panTro2
+ cd /hive/data/genomes/panTro2/bed/lastzCalJac3.2010-02-11
+ cat fb.panTro2.chainCalJac3Link.txt
+ # 2016331285 bases of 2909485072 (69.302%) in intersection
+
+ # and this swap run
+ mkdir /hive/data/genomes/calJac3/bed/blastz.panTro2.swap
+ cd /hive/data/genomes/calJac3/bed/blastz.panTro2.swap
+ time nice -n +19 doBlastzChainNet.pl -verbose=2 \
+ /hive/data/genomes/panTro2/bed/lastzCalJac3.2010-02-11/DEF \
+ -swap -noLoadChainSplit -syntenicNet \
+ -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
+ -chainMinScore=5000 -chainLinearGap=medium > swap.log 2>&1 &
+ # real 118m42.203s
+ cat fb.calJac3.chainHg19Link.txt
+ # 1990168262 bases of 2752505800 (72.304%) in intersection
+
+############################################################################
+# LASTZ Orangutan Swap (DONE - 2010-02-11 - Hiram)
+ # original alignment to ponAbe2
+ cd /hive/data/genomes/ponAbe2/bed/lastzCalJac3.2010-02-11
+ cat fb.ponAbe2.chainCalJac3Link.txt
+ # 2086557592 bases of 3093572278 (67.448%) in intersection
+
+ # and this swap run
+ mkdir /hive/data/genomes/calJac3/bed/blastz.ponAbe2.swap
+ cd /hive/data/genomes/calJac3/bed/blastz.ponAbe2.swap
+ time nice -n +19 doBlastzChainNet.pl -verbose=2 \
+ /hive/data/genomes/ponAbe2/bed/lastzCalJac3.2010-02-11/DEF \
+ -swap -syntenicNet \
+ -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
+ -chainMinScore=5000 -chainLinearGap=medium > swap.log 2>&1 &
+ # real 146m12.301s
+ cat fb.calJac3.chainHg19Link.txt
+ # 1978857628 bases of 2752505800 (71.893%) in intersection
+#####################################################################
+# lastz Dog CanFam2 Swap (DONE - 2010-02-14 - Hiram)
+ # original alignment
+ cd /hive/data/genomes/canFam2/bed/lastzCalJac3.2010-02-12
+ cat fb.canFam2.chainCalJac3Link.txt
+ # 1363307334 bases of 2384996543 (57.162%) in intersection
+
+ # and for the swap
+ mkdir /hive/data/genomes/calJac3/bed/blastz.canFam2.swap
+ cd /hive/data/genomes/calJac3/bed/blastz.canFam2.swap
+ time nice -n +19 doBlastzChainNet.pl -verbose=2 \
+ /hive/data/genomes/canFam2/bed/lastzCalJac3.2010-02-12/DEF \
+ -swap -syntenicNet \
+ -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
+ -chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
+ # real 129m56.144s
+ cat fb.calJac3.chainHg19Link.txt
+ # 1397333116 bases of 2752505800 (50.766%) in intersection
+
+#########################################################################
+# lastz Mouse Mm9 Swap (DONE - 2010-02-14 - Hiram)
+ # original alignment to mouse
+ cd /hive/data/genomes/mm9/bed/lastzCalJac3.2010-02-12
+ cat fb.mm9.chainCalJac3Link.txt
+ # 859869647 bases of 2620346127 (32.815%) in intersection
+
+ # and for the swap
+ mkdir /hive/data/genomes/calJac3/bed/blastz.mm9.swap
+ cd /hive/data/genomes/calJac3/bed/blastz.mm9.swap
+ time nice -n +19 doBlastzChainNet.pl -verbose=2 \
+ /hive/data/genomes/mm9/bed/lastzCalJac3.2010-02-12/DEF \
+ -swap -syntenicNet \
+ -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
+ -chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
+ # real 90m38.739s
+ cat fb.calJac3.chainHg19Link.txt
+ # 861811978 bases of 2752505800 (31.310%) in intersection
+
+#####################################################################
+# lastz Opossum monDom5 Swap (DONE - 2010-02-14 - Hiram)
+ # original alignment to Opossum
+ cd /hive/data/genomes/monDom5/bed/lastzCalJac3.2010-02-12
+ cat fb.monDom5.chainCalJac3Link.txt
+ # 216197506 bases of 3501660299 (6.174%) in intersection
+
+ # and for the swap
+ mkdir /hive/data/genomes/calJac3/bed/blastz.monDom5.swap
+ cd /hive/data/genomes/calJac3/bed/blastz.monDom5.swap
+ time nice -n +19 doBlastzChainNet.pl -verbose=2 \
+ /hive/data/genomes/monDom5/bed/lastzCalJac3.2010-02-12/DEF \
+ -swap -noLoadChainSplit -syntenicNet \
+ -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
+ -chainMinScore=5000 -chainLinearGap=loose > swap.log 2>&1 &
+ # real 110m13.435s
+ cat fb.calJac3.chainMonDom5Link.txt
+ # 217614612 bases of 2752505800 (7.906%) in intersection
+
+##############################################################################
# HUMAN (hg18) PROTEINS TRACK (working braney...)
# bash if not using bash shell already
cd /cluster/data/calJac3
@@ -225,4 +601,6 @@
# blastHg18KG 5.611%, ensGene 9.520%, both 4.993%, cover 88.97%, enrich 9.35x
rm -rf blastOut
#end tblastn
+
+##############################################################################