src/hg/makeDb/doc/tetNig2.txt 1.4
1.4 2009/08/07 14:42:33 braney
make Human Proteins track. Needs re-doing to deal with chrUn correctly
Index: src/hg/makeDb/doc/tetNig2.txt
===================================================================
RCS file: /projects/compbio/cvsroot/kent/src/hg/makeDb/doc/tetNig2.txt,v
retrieving revision 1.3
retrieving revision 1.4
diff -b -B -U 1000000 -r1.3 -r1.4
--- src/hg/makeDb/doc/tetNig2.txt 6 Aug 2009 22:59:01 -0000 1.3
+++ src/hg/makeDb/doc/tetNig2.txt 7 Aug 2009 14:42:33 -0000 1.4
@@ -1,213 +1,379 @@
# for emacs: -*- mode: sh; -*-
# Tetraodon Nigirividis from Genoscope, version v8 (released Mar 2007)
# Project website:
# http://www.genoscope.cns.fr/externe/Download/Projets/Projet_C/genomique/goldenpath_v2/
##########################################################################
# Download sequence (DONE - 2009-08-06 - Hiram)
mkdir /hive/data/genomes/tetNig2
cd /hive/data/genomes/tetNig2
mkdir genoscope
cd genoscope
D="http://www.genoscope.cns.fr/externe/Download/Projets/Projet_C/genomique/golde
npath_v2/"
for F in chr.agp chr.agp.info
do
wget --timestamping "${D}/${F}"
done
for C in 1 1_random 2 2_random 3 4 5 6 7 8 9 10 11 12 13 14 15 \
15_random 16 17 18 19 20 21 21_random Un_random
do
wget --timestamping "${D}/unmasked/chr${C}.fa.bz2"
done
for F in Tetraodon_mRNAs_v8.2.fa Tetraodon_peptide_v8.2.fa \
Tetraodon_peptide_v8.2.fa.compo annotation_tetraodon_v8.2.gff
do
wget --timestamping "${D}/annotation_v8.2/${F}"
done
bunzip *.fa.bz2
gzip *.fa
##########################################################################
# Initial browser (DONE - 2009-08-06 - Hiram)
cd /hive/data/genomes/tetNig2
cat << '_EOF_' > tetNig2.config.ra
# Config parameters for makeGenomeDb.pl:
db tetNig2
clade vertebrate
scientificName Tetraodon nigroviridis
commonName Tetraodon
assemblyDate Mar. 2007
assemblyLabel Genoscope Tetraodon v8.0 (NCBI project 12350, CAAE01000000)
orderKey 459
mitoAcc NC_007176
fastaFiles /hive/data/genomes/tetNig2/genoscope/chr*.fa.gz
agpFiles /hive/data/genomes/tetNig2/genoscope/chr.agp
# qualFiles none
dbDbSpeciesDir tetraodon
taxId 99883
'_EOF_'
# << happy emacs
time nice -n +19 makeGenomeDb.pl -verbose=2 -workhorse=hgwdev \
-noGoldGapSplit -stop=agp tetNig2.config.ra > agp.log 2>&1
time nice -n +19 makeGenomeDb.pl -verbose=2 -workhorse=hgwdev \
-noGoldGapSplit -continue=db -stop=db tetNig2.config.ra > db.log 2>&1
time nice -n +19 makeGenomeDb.pl -verbose=2 -workhorse=hgwdev \
-noGoldGapSplit -continue=dbDb tetNig2.config.ra > dbDb.log 2>&1
# add the trackDb files to the source tree and entry to trackDb/makefile
##########################################################################
# Repeat Masker (DONE - 2009-08-06 - Hiram)
mkdir /hive/data/genomes/tetNig2/bed/repeatMasker
cd /hive/data/genomes/tetNig2/bed/repeatMasker
doRepeatMasker.pl -verbose=2 -workhorse=hgwdev \
-noSplit -buildDir=`pwd` tetNig2 > do.log 2>&1
cat faSize.rmsk.txt
# 358618246 bases (56303458 N's 302314788 real 292078336 upper 10236452 lower)
# in 27 sequences in 1 files
# %2.85 masked total, %3.39 masked real
# since this doesn't mask very much, use windowmasker instead
hgsql -e "drop table rmsk;" tetNig2
# this leaves the interrupted repeats track showing on genome-test
########################################################################
# Simple Repeats (DONE - 2009-08-06 - Hiram)
mkdir /hive/data/genomes/tetNig2/bed/simpleRepeat
cd /hive/data/genomes/tetNig2/bed/simpleRepeat
doSimpleRepeat.pl -workhorse=hgwdev \
-buildDir=`pwd` tetNig2 > do.log 2>&1 &
# fails on the job for chrM, make an empty result:
touch /hive/data/genomes/tetNig2/TrfPart/009/009.lst.bed
doSimpleRepeat.pl -workhorse=hgwdev -continue=filter \
-buildDir=`pwd` tetNig2 > filter.log 2>&1 &
cat fb.simpleRepeat
# 11549259 bases of 332311746 (3.475%) in intersection
########################################################################
# WindowMasker (DONE - 2009-08-06 - Hiram)
mkdir /hive/data/genomes/tetNig2/bed/windowMasker
cd /hive/data/genomes/tetNig2/bed/windowMasker
doWindowMasker.pl -workhorse=hgwdev -buildDir=`pwd` tetNig2 > do.log 2>&1
twoBitToFa tetNig2.wmsk.sdust.2bit stdout | faSize stdin
# 358618246 bases (56303458 N's 302314788 real 241249522 upper
# 61065266 lower) in 27 sequences in 1 files
# %17.03 masked total, %20.20 masked real
# load this initial data to get ready to clean it
ssh hgwdev
cd /hive/data/genomes/tetNig2/bed/windowMasker
hgLoadBed tetNig2 windowmaskerSdust windowmasker.sdust.bed.gz
# Loaded 1649525 elements of size 3
featureBits -countGaps tetNig2 windowmaskerSdust
# 117367586 bases of 358618246 (32.728%) in intersection
# eliminate the gaps from the masking
featureBits tetNig2 -not gap -bed=notGap.bed
time nice -n +19 featureBits tetNig2 windowmaskerSdust notGap.bed \
-bed=stdout | gzip -c > cleanWMask.bed.gz
# 91061086 bases of 332311746 (27.402%) in intersection
# reload track to get it clean
hgLoadBed tetNig2 windowmaskerSdust cleanWMask.bed.gz
# Loaded 1647549 elements of size 4
featureBits -countGaps tetNig2 windowmaskerSdust
# 91061086 bases of 358618246 (25.392%) in intersection
# mask the sequence with this clean mask
zcat cleanWMask.bed.gz \
| twoBitMask ../../tetNig2.unmasked.2bit stdin \
-type=.bed tetNig2.cleanWMSdust.2bit
twoBitToFa tetNig2.cleanWMSdust.2bit stdout | faSize stdin \
> tetNig2.cleanWMSdust.faSize.txt
cat tetNig2.cleanWMSdust.faSize.txt
# 358618246 bases (56303458 N's 302314788 real 241249522 upper 61065266
# lower) in 27 sequences in 1 files
# %17.03 masked total, %20.20 masked real
########################################################################
# MASK SEQUENCE WITH WM+TRF (DONE - 2009-08-06 - Hiram)
cd /hive/data/genomes/tetNig2
twoBitMask -add bed/windowMasker/tetNig2.cleanWMSdust.2bit \
bed/simpleRepeat/trfMask.bed tetNig2.2bit
# safe to ignore the warnings about BED file with >=13 fields
twoBitToFa tetNig2.2bit stdout | faSize stdin > faSize.tetNig2.txt
cat faSize.tetNig2.wmskSdust.TRF.txt
# 358618246 bases (56303458 N's 302314788 real 241039472 upper 61275316
# lower) in 27 sequences in 1 files
# %17.09 masked total, %20.27 masked real
# create symlink to gbdb
ssh hgwdev
rm /gbdb/tetNig2/tetNig2.2bit
ln -s `pwd`/tetNig2.2bit /gbdb/tetNig2/tetNig2.2bit
#########################################################################
# MAKE 11.OOC FILE FOR BLAT/GENBANK (DONE - 2009-08-06 - Hiram)
# Use -repMatch=130 (based on size -- for human we use 1024, and
# Tetraodon size is ~12.7% of human judging by gapless tetNig2 vs.
# hg18 genome size from featureBits.
# genome. Bump that up a bit to be more conservative.
# 100*302314788/2897310462 = 10.434324
cd /hive/data/genomes/tetNig2
blat tetNig2.2bit /dev/null /dev/null -tileSize=11 \
-makeOoc=jkStuff/tetNig2.11.ooc -repMatch=130
# Wrote 8132 overused 11-mers to jkStuff/tetNig2.11.ooc
# copy all of this stuff to the klusters:
cd /hive/data/genomes/tetNig2
mkdir /hive/data/staging/data/tetNig2
cp -p jkStuff/tetNig2.11.ooc chrom.sizes tetNig2.2bit \
/hive/data/staging/data/tetNig2
#########################################################################
# Ensembl genes v55 (DONE - 2009-08-06 - Hiram)
cd /hive/data/genomes/tetNig2
cat << '_EOF_' > tetNig2.ensGene.ra
# required db variable
db tetNig2
# optional nameTranslation, the sed command that will transform
# Ensemble names to UCSC names. With quotes just to make sure.
nameTranslation "s/^\([0-9XY][0-9]*\)/chr\1/; s/^MT/chrM/; s/^Un/chrUn/"
'_EOF_'
# << happy emacs
doEnsGeneUpdate.pl -verbose=2 -workhorse=hgwdev \
-ensVersion=55 -stop=process tetNig2.ensGene.ra > tetNig2.55.log 2>&1
doEnsGeneUpdate.pl -verbose=2 -workhorse=hgwdev \
-ensVersion=55 -continue=load tetNig2.ensGene.ra >> tetNig2.55.log 2>&1
featureBits tetNig2 ensGene
# 31637658 bases of 332311746 (9.520%) in intersection
#########################################################################
# BLATSERVERS ENTRY (DONE - 2009-08-06 - Hiram)
Thu Aug 6 15:58:09 PDT 2009
# After getting a blat server assigned by the Blat Server Gods,
ssh hgwdev
hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
VALUES ("tetNig2", "blat15", "17794", "1", "0"); \
INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
VALUES ("tetNig2", "blat15", "17795", "0", "1");' \
hgcentraltest
# test it with some sequence
#########################################################################
# Set default position same as tetNig1 after blating tetNig1 to here
# (DONE - 2009-08-06 - Hiram)
hgsql -e \
'update dbDb set defaultPos="chr10:2797091-2804214" where name="tetNig2";' \
hgcentraltest
#########################################################################
+# HUMAN (hg18) PROTEINS TRACK (DONE 2009-08-07 braney )
+
+### !!!! needs re-doing to deal with chrUn correctly !!!!
+
+ # bash if not using bash shell already
+
+ cd /cluster/data/tetNig2
+ mkdir /cluster/data/tetNig2/blastDb
+
+ awk '{if ($2 > 1000000) print $1}' chrom.sizes > 1meg.lst
+ twoBitToFa -seqList=1meg.lst tetNig2.2bit temp.fa
+ faSplit gap temp.fa 1000000 blastDb/x -lift=blastDb.lft
+ rm temp.fa 1meg.lst
+
+ awk '{if ($2 <= 1000000) print $1}' chrom.sizes > less1meg.lst
+ twoBitToFa -seqList=less1meg.lst tetNig2.2bit temp.fa
+ faSplit about temp.fa 1000000 blastDb/y
+ rm temp.fa less1meg.lst
+
+ cd blastDb
+ for i in *.fa
+ do
+ /hive/data/outside/blast229/formatdb -i $i -p F
+ done
+ rm *.fa
+ ls *.nsq | wc -l
+# 394
+
+
+ mkdir -p /cluster/data/tetNig2/bed/tblastn.hg18KG
+ cd /cluster/data/tetNig2/bed/tblastn.hg18KG
+ echo ../../blastDb/*.nsq | xargs ls -S | sed "s/\.nsq//" > query.lst
+ wc -l query.lst
+# 394 query.lst
+
+ # we want around 100000 jobs
+ calc `wc /cluster/data/hg18/bed/blat.hg18KG/hg18KG.psl | awk '{print $1}'`/\(100000/`wc query.lst | awk '{print $1}'`\)
+
+# 36727/(100000/394) = 144.704380
+
+ mkdir -p kgfa
+ split -l 145 /cluster/data/hg18/bed/blat.hg18KG/hg18KG.psl kgfa/kg
+ cd kgfa
+ for i in *; do
+ nice pslxToFa $i $i.fa;
+ rm $i;
+ done
+ cd ..
+ ls -1S kgfa/*.fa > kg.lst
+ wc kg.lst
+# 254 254 3302 kg.lst
+
+ mkdir -p blastOut
+ for i in `cat kg.lst`; do mkdir blastOut/`basename $i .fa`; done
+ tcsh
+ cd /cluster/data/tetNig2/bed/tblastn.hg18KG
+ cat << '_EOF_' > blastGsub
+#LOOP
+blastSome $(path1) {check in line $(path2)} {check out exists blastOut/$(root2)/q.$(root1).psl }
+#ENDLOOP
+'_EOF_'
+
+ cat << '_EOF_' > blastSome
+#!/bin/sh
+BLASTMAT=/hive/data/outside/blast229/data
+export BLASTMAT
+g=`basename $2`
+f=/tmp/`basename $3`.$g
+for eVal in 0.01 0.001 0.0001 0.00001 0.000001 1E-09 1E-11
+do
+if /hive/data/outside/blast229/blastall -M BLOSUM80 -m 0 -F no -e $eVal -p tblastn -d $1 -i $2 -o $f.8
+then
+ mv $f.8 $f.1
+ break;
+fi
+done
+if test -f $f.1
+then
+ if /cluster/bin/i386/blastToPsl $f.1 $f.2
+ then
+ liftUp -nosort -type=".psl" -nohead $f.3 /cluster/data/tetNig2/blastDb.lft carry $f.2
+ liftUp -nosort -type=".psl" -pslQ -nohead $3.tmp /cluster/data/hg18/bed/blat.hg18KG/protein.lft warn $f.3
+ if pslCheck -prot $3.tmp
+ then
+ mv $3.tmp $3
+ rm -f $f.1 $f.2 $f.3 $f.4
+ fi
+ exit 0
+ fi
+fi
+rm -f $f.1 $f.2 $3.tmp $f.8 $f.3 $f.4
+exit 1
+'_EOF_'
+ # << happy emacs
+ chmod +x blastSome
+ exit
+
+ ssh swarm
+ cd /cluster/data/tetNig2/bed/tblastn.hg18KG
+ gensub2 query.lst kg.lst blastGsub blastSpec
+ para create blastSpec
+# para try, check, push, check etc.
+
+ para time
+# Completed: 100076 of 100076 jobs
+# CPU time in finished jobs: 2219419s 36990.31m 616.51h 25.69d 0.070 y
+# IO & Wait Time: 445463s 7424.39m 123.74h 5.16d 0.014 y
+# Average job time: 27s 0.44m 0.01h 0.00d
+# Longest finished job: 319s 5.32m 0.09h 0.00d
+# Submission to last job: 2791s 46.52m 0.78h 0.03d
+
+ ssh swarm
+ cd /cluster/data/tetNig2/bed/tblastn.hg18KG
+ mkdir chainRun
+ cd chainRun
+ tcsh
+ cat << '_EOF_' > chainGsub
+#LOOP
+chainOne $(path1)
+#ENDLOOP
+'_EOF_'
+
+ cat << '_EOF_' > chainOne
+(cd $1; cat q.*.psl | simpleChain -prot -outPsl -maxGap=12000 stdin ../c.`basename $1`.psl)
+'_EOF_'
+ chmod +x chainOne
+ ls -1dS ../blastOut/kg?? > chain.lst
+ gensub2 chain.lst single chainGsub chainSpec
+ # do the cluster run for chaining
+ para create chainSpec
+ para try, check, push, check etc.
+
+# Completed: 254 of 254 jobs
+# CPU time in finished jobs: 590077s 9834.62m 163.91h 6.83d 0.019 y
+# IO & Wait Time: 20749s 345.81m 5.76h 0.24d 0.001 y
+# Average job time: 2405s 40.08m 0.67h 0.03d
+# Longest finished job: 36592s 609.87m 10.16h 0.42d
+# Submission to last job: 36604s 610.07m 10.17h 0.42d
+
+ cd /cluster/data/tetNig2/bed/tblastn.hg18KG/blastOut
+ for i in kg??
+ do
+ cat c.$i.psl | awk "(\$13 - \$12)/\$11 > 0.6 {print}" > c60.$i.psl
+ sort -rn c60.$i.psl | pslUniq stdin u.$i.psl
+ awk "((\$1 / \$11) ) > 0.60 { print }" c60.$i.psl > m60.$i.psl
+ echo $i
+ done
+ sort u.*.psl m60* | uniq | sort -T /tmp -k 14,14 -k 16,16n -k 17,17n > ../blastHg18KG.psl
+ cd ..
+ pslCheck blastHg18KG.psl
+# checked: 38517 failed: 0 errors: 0
+
+ # load table
+ ssh hgwdev
+ cd /cluster/data/tetNig2/bed/tblastn.hg18KG
+ hgLoadPsl tetNig2 blastHg18KG.psl
+
+ # check coverage
+ featureBits tetNig2 blastHg18KG
+# 18646943 bases of 332311746 (5.611%) in intersection
+
+ featureBits tetNig2 blastHg18KG ensGene -enrichment
+# blastHg18KG 5.611%, ensGene 9.520%, both 4.993%, cover 88.97%, enrich 9.35x
+
+ rm -rf blastOut
+#end tblastn