src/hg/makeDb/doc/tetNig2.txt 1.9
1.9 2010/02/11 23:43:29 hiram
add cpgIslandExt track
Index: src/hg/makeDb/doc/tetNig2.txt
===================================================================
RCS file: /projects/compbio/cvsroot/kent/src/hg/makeDb/doc/tetNig2.txt,v
retrieving revision 1.8
retrieving revision 1.9
diff -b -B -U 1000000 -r1.8 -r1.9
--- src/hg/makeDb/doc/tetNig2.txt 9 Jan 2010 00:28:46 -0000 1.8
+++ src/hg/makeDb/doc/tetNig2.txt 11 Feb 2010 23:43:29 -0000 1.9
@@ -1,803 +1,811 @@
# for emacs: -*- mode: sh; -*-
# Tetraodon Nigirividis from Genoscope, version v8 (released Mar 2007)
# Project website:
# http://www.genoscope.cns.fr/externe/Download/Projets/Projet_C/genomique/goldenpath_v2/
##########################################################################
# Download sequence (DONE - 2009-08-06 - Hiram)
mkdir /hive/data/genomes/tetNig2
cd /hive/data/genomes/tetNig2
mkdir genoscope
cd genoscope
D="http://www.genoscope.cns.fr/externe/Download/Projets/Projet_C/genomique/golde
npath_v2/"
for F in chr.agp chr.agp.info
do
wget --timestamping "${D}/${F}"
done
for C in 1 1_random 2 2_random 3 4 5 6 7 8 9 10 11 12 13 14 15 \
15_random 16 17 18 19 20 21 21_random Un_random
do
wget --timestamping "${D}/unmasked/chr${C}.fa.bz2"
done
for F in Tetraodon_mRNAs_v8.2.fa Tetraodon_peptide_v8.2.fa \
Tetraodon_peptide_v8.2.fa.compo annotation_tetraodon_v8.2.gff
do
wget --timestamping "${D}/annotation_v8.2/${F}"
done
bunzip *.fa.bz2
gzip *.fa
##########################################################################
# Initial browser (DONE - 2009-08-06 - Hiram)
cd /hive/data/genomes/tetNig2
cat << '_EOF_' > tetNig2.config.ra
# Config parameters for makeGenomeDb.pl:
db tetNig2
clade vertebrate
scientificName Tetraodon nigroviridis
commonName Tetraodon
assemblyDate Mar. 2007
assemblyLabel Genoscope Tetraodon v8.0 (NCBI project 12350, CAAE01000000)
orderKey 459
mitoAcc NC_007176
fastaFiles /hive/data/genomes/tetNig2/genoscope/chr*.fa.gz
agpFiles /hive/data/genomes/tetNig2/genoscope/chr.agp
# qualFiles none
dbDbSpeciesDir tetraodon
taxId 99883
'_EOF_'
# << happy emacs
time nice -n +19 makeGenomeDb.pl -verbose=2 -workhorse=hgwdev \
-noGoldGapSplit -stop=agp tetNig2.config.ra > agp.log 2>&1
time nice -n +19 makeGenomeDb.pl -verbose=2 -workhorse=hgwdev \
-noGoldGapSplit -continue=db -stop=db tetNig2.config.ra > db.log 2>&1
time nice -n +19 makeGenomeDb.pl -verbose=2 -workhorse=hgwdev \
-noGoldGapSplit -continue=dbDb tetNig2.config.ra > dbDb.log 2>&1
# add the trackDb files to the source tree and entry to trackDb/makefile
##########################################################################
# Repeat Masker (DONE - 2009-08-06 - Hiram)
mkdir /hive/data/genomes/tetNig2/bed/repeatMasker
cd /hive/data/genomes/tetNig2/bed/repeatMasker
doRepeatMasker.pl -verbose=2 -workhorse=hgwdev \
-noSplit -buildDir=`pwd` tetNig2 > do.log 2>&1
cat faSize.rmsk.txt
# 358618246 bases (56303458 N's 302314788 real 292078336 upper 10236452 lower)
# in 27 sequences in 1 files
# %2.85 masked total, %3.39 masked real
# since this doesn't mask very much, use windowmasker instead
hgsql -e "drop table rmsk;" tetNig2
# this leaves the interrupted repeats track showing on genome-test
# from the do.log:
# Repeat masker version: June 4 2009 (open-3-2-8)
# Library release 20090604
########################################################################
# Simple Repeats (DONE - 2009-08-06 - Hiram)
mkdir /hive/data/genomes/tetNig2/bed/simpleRepeat
cd /hive/data/genomes/tetNig2/bed/simpleRepeat
doSimpleRepeat.pl -workhorse=hgwdev \
-buildDir=`pwd` tetNig2 > do.log 2>&1 &
# fails on the job for chrM, make an empty result:
touch /hive/data/genomes/tetNig2/TrfPart/009/009.lst.bed
doSimpleRepeat.pl -workhorse=hgwdev -continue=filter \
-buildDir=`pwd` tetNig2 > filter.log 2>&1 &
cat fb.simpleRepeat
# 11549259 bases of 332311746 (3.475%) in intersection
########################################################################
# Marking *all* gaps - they are not all in the AGP file
# (DONE - 2009-08-07 - Hiram)
mkdir /hive/data/genomes/tetNig2/bed/allGaps
time nice -n +19 findMotif -motif=gattaca -verbose=4 \
-strand=+ ../../tetNig2.2bit > findMotif.txt 2>&1
# real 0m7.967s
grep "^#GAP " findMotif.txt | sed -e "s/^#GAP //" > allGaps.bed
featureBits tetNig2 -not gap -bed=notGap.bed
featureBits tetNig2 allGaps.bed notGap.bed -bed=new.gaps.bed
# what is the last index in the existing gap table:
hgsql -N -e "select ix from gap;" tetNig2 | sort -n | tail -1
# 34284
cat << '_EOF_' > mkGap.pl
#!/usr/bin/env perl
use strict;
use warnings;
my $ix=`hgsql -N -e "select ix from gap;" tetNig2 | sort -n | tail -1`;
chomp $ix;
open (FH,"<new.gaps.bed") or die "can not read new.gaps.bed";
while (my $line = <FH>) {
my ($chrom, $chromStart, $chromEnd, $rest) = split('\s+', $line);
++$ix;
printf "%s\t%d\t%d\t%d\tN\t%d\tother\tyes\n", $chrom, $chromStart,
$chromEnd, $ix, $chromEnd-$chromStart;
}
close (FH);
'_EOF_'
# << happy emacs
chmod +x ./mkGap.pl
./mkGap.pl > other.gap
hgLoadBed -sqlTable=$HOME/kent/src/hg/lib/gap.sql \
-noLoad tetNig2 otherGap other.gap
# Loaded 17051
# adding this many:
wc -l bed.tab
# 17051
# starting with this many
hgsql -e "select count(*) from gap;" tetNig2
# 17140
hgsql tetNig2 -e 'load data local infile "bed.tab" into table gap;'
# result count:
hgsql -e "select count(*) from gap;" tetNig2
# 34191
# == 17140 + 17051
########################################################################
# WindowMasker (DONE - 2009-08-06 - Hiram)
mkdir /hive/data/genomes/tetNig2/bed/windowMasker
cd /hive/data/genomes/tetNig2/bed/windowMasker
doWindowMasker.pl -workhorse=hgwdev -buildDir=`pwd` tetNig2 > do.log 2>&1
twoBitToFa tetNig2.wmsk.sdust.2bit stdout | faSize stdin
# 358618246 bases (56303458 N's 302314788 real 241249522 upper
# 61065266 lower) in 27 sequences in 1 files
# %17.03 masked total, %20.20 masked real
# load this initial data to get ready to clean it
ssh hgwdev
cd /hive/data/genomes/tetNig2/bed/windowMasker
hgLoadBed tetNig2 windowmaskerSdust windowmasker.sdust.bed.gz
# Loaded 1649525 elements of size 3
featureBits -countGaps tetNig2 windowmaskerSdust
# 117367586 bases of 358618246 (32.728%) in intersection
# eliminate the gaps from the masking
featureBits tetNig2 -not gap -bed=notGap.bed
# 302314788 bases of 302314788 (100.000%) in intersection
time nice -n +19 featureBits tetNig2 windowmaskerSdust notGap.bed \
-bed=stdout | gzip -c > cleanWMask.bed.gz
# 61065266 bases of 302314788 (20.199%) in intersection
# reload track to get it clean
hgLoadBed tetNig2 windowmaskerSdust cleanWMask.bed.gz
# Loaded 1644977 elements of size 4
featureBits -countGaps tetNig2 windowmaskerSdust
# 61065266 bases of 358618246 (17.028%) in intersection
# mask the sequence with this clean mask
zcat cleanWMask.bed.gz \
| twoBitMask ../../tetNig2.unmasked.2bit stdin \
-type=.bed tetNig2.cleanWMSdust.2bit
twoBitToFa tetNig2.cleanWMSdust.2bit stdout | faSize stdin \
> tetNig2.cleanWMSdust.faSize.txt
cat tetNig2.cleanWMSdust.faSize.txt
# 358618246 bases (56303458 N's 302314788 real 241249522 upper 61065266
# lower) in 27 sequences in 1 files
# %17.03 masked total, %20.20 masked real
########################################################################
# MASK SEQUENCE WITH WM+TRF (DONE - 2009-08-06 - Hiram)
cd /hive/data/genomes/tetNig2
twoBitMask -add bed/windowMasker/tetNig2.cleanWMSdust.2bit \
bed/simpleRepeat/trfMask.bed tetNig2.2bit
# safe to ignore the warnings about BED file with >=13 fields
twoBitToFa tetNig2.2bit stdout | faSize stdin > faSize.tetNig2.txt
cat faSize.tetNig2.txt
# 358618246 bases (56303458 N's 302314788 real 241039472 upper 61275316
# lower) in 27 sequences in 1 files
# %17.09 masked total, %20.27 masked real
# create symlink to gbdb
ssh hgwdev
rm /gbdb/tetNig2/tetNig2.2bit
ln -s `pwd`/tetNig2.2bit /gbdb/tetNig2/tetNig2.2bit
#########################################################################
# MAKE 11.OOC FILE FOR BLAT/GENBANK (DONE - 2009-08-06 - Hiram)
# Use -repMatch=130 (based on size -- for human we use 1024, and
# Tetraodon size is ~12.7% of human judging by gapless tetNig2 vs.
# hg18 genome size from featureBits.
# genome. Bump that up a bit to be more conservative.
# 100*302314788/2897310462 = 10.434324
cd /hive/data/genomes/tetNig2
blat tetNig2.2bit /dev/null /dev/null -tileSize=11 \
-makeOoc=jkStuff/tetNig2.11.ooc -repMatch=130
# Wrote 8132 overused 11-mers to jkStuff/tetNig2.11.ooc
# copy all of this stuff to the klusters:
cd /hive/data/genomes/tetNig2/jkStuff
gapToLift tetNig2 nonBridged.lift -bedFile=nonBridged.bed
cd /hive/data/genomes/tetNig2
mkdir /hive/data/staging/data/tetNig2
cp -p jkStuff/tetNig2.11.ooc jkStuff/nonBridged.lift \
chrom.sizes tetNig2.2bit /hive/data/staging/data/tetNig2
#########################################################################
# Ensembl genes v55 (DONE - 2009-08-06 - Hiram)
cd /hive/data/genomes/tetNig2
cat << '_EOF_' > tetNig2.ensGene.ra
# required db variable
db tetNig2
# optional nameTranslation, the sed command that will transform
# Ensemble names to UCSC names. With quotes just to make sure.
nameTranslation "s/^\([0-9XY][0-9]*\)/chr\1/; s/^MT/chrM/; s/^Un/chrUn/"
'_EOF_'
# << happy emacs
doEnsGeneUpdate.pl -verbose=2 -workhorse=hgwdev \
-ensVersion=55 -stop=process tetNig2.ensGene.ra > tetNig2.55.log 2>&1
doEnsGeneUpdate.pl -verbose=2 -workhorse=hgwdev \
-ensVersion=55 -continue=load tetNig2.ensGene.ra >> tetNig2.55.log 2>&1
featureBits tetNig2 ensGene
# 31637658 bases of 332311746 (9.520%) in intersection
#########################################################################
# BLATSERVERS ENTRY (DONE - 2009-08-06 - Hiram)
Thu Aug 6 15:58:09 PDT 2009
# After getting a blat server assigned by the Blat Server Gods,
ssh hgwdev
hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
VALUES ("tetNig2", "blat15", "17794", "1", "0"); \
INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
VALUES ("tetNig2", "blat15", "17795", "0", "1");' \
hgcentraltest
# test it with some sequence
#########################################################################
# Set default position same as tetNig1 after blating tetNig1 to here
# (DONE - 2009-08-06 - Hiram)
hgsql -e \
'update dbDb set defaultPos="chr10:2797091-2804214" where name="tetNig2";' \
hgcentraltest
#########################################################################
# HUMAN (hg18) PROTEINS TRACK (DONE 2009-08-07 braney )
### !!!! needs re-doing to deal with chrUn correctly !!!!
# bash if not using bash shell already
cd /cluster/data/tetNig2
mkdir /cluster/data/tetNig2/blastDb
awk '{if ($2 > 1000000) print $1}' chrom.sizes > 1meg.lst
twoBitToFa -seqList=1meg.lst tetNig2.2bit temp.fa
faSplit gap temp.fa 1000000 blastDb/x -lift=blastDb.lft
rm temp.fa 1meg.lst
awk '{if ($2 <= 1000000) print $1}' chrom.sizes > less1meg.lst
twoBitToFa -seqList=less1meg.lst tetNig2.2bit temp.fa
faSplit about temp.fa 1000000 blastDb/y
rm temp.fa less1meg.lst
cd blastDb
for i in *.fa
do
/hive/data/outside/blast229/formatdb -i $i -p F
done
rm *.fa
ls *.nsq | wc -l
# 394
mkdir -p /cluster/data/tetNig2/bed/tblastn.hg18KG
cd /cluster/data/tetNig2/bed/tblastn.hg18KG
echo ../../blastDb/*.nsq | xargs ls -S | sed "s/\.nsq//" > query.lst
wc -l query.lst
# 394 query.lst
# we want around 100000 jobs
calc `wc /cluster/data/hg18/bed/blat.hg18KG/hg18KG.psl | awk '{print $1}'`/\(100000/`wc query.lst | awk '{print $1}'`\)
# 36727/(100000/394) = 144.704380
mkdir -p kgfa
split -l 145 /cluster/data/hg18/bed/blat.hg18KG/hg18KG.psl kgfa/kg
cd kgfa
for i in *; do
nice pslxToFa $i $i.fa;
rm $i;
done
cd ..
ls -1S kgfa/*.fa > kg.lst
wc kg.lst
# 254 254 3302 kg.lst
mkdir -p blastOut
for i in `cat kg.lst`; do mkdir blastOut/`basename $i .fa`; done
tcsh
cd /cluster/data/tetNig2/bed/tblastn.hg18KG
cat << '_EOF_' > blastGsub
#LOOP
blastSome $(path1) {check in line $(path2)} {check out exists blastOut/$(root2)/q.$(root1).psl }
#ENDLOOP
'_EOF_'
cat << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/hive/data/outside/blast229/data
export BLASTMAT
g=`basename $2`
f=/tmp/`basename $3`.$g
for eVal in 0.01 0.001 0.0001 0.00001 0.000001 1E-09 1E-11
do
if /hive/data/outside/blast229/blastall -M BLOSUM80 -m 0 -F no -e $eVal -p tblastn -d $1 -i $2 -o $f.8
then
mv $f.8 $f.1
break;
fi
done
if test -f $f.1
then
if /cluster/bin/i386/blastToPsl $f.1 $f.2
then
liftUp -nosort -type=".psl" -nohead $f.3 /cluster/data/tetNig2/blastDb.lft carry $f.2
liftUp -nosort -type=".psl" -pslQ -nohead $3.tmp /cluster/data/hg18/bed/blat.hg18KG/protein.lft warn $f.3
if pslCheck -prot $3.tmp
then
mv $3.tmp $3
rm -f $f.1 $f.2 $f.3 $f.4
fi
exit 0
fi
fi
rm -f $f.1 $f.2 $3.tmp $f.8 $f.3 $f.4
exit 1
'_EOF_'
# << happy emacs
chmod +x blastSome
exit
ssh swarm
cd /cluster/data/tetNig2/bed/tblastn.hg18KG
gensub2 query.lst kg.lst blastGsub blastSpec
para create blastSpec
# para try, check, push, check etc.
para time
# Completed: 100076 of 100076 jobs
# CPU time in finished jobs: 2219419s 36990.31m 616.51h 25.69d 0.070 y
# IO & Wait Time: 445463s 7424.39m 123.74h 5.16d 0.014 y
# Average job time: 27s 0.44m 0.01h 0.00d
# Longest finished job: 319s 5.32m 0.09h 0.00d
# Submission to last job: 2791s 46.52m 0.78h 0.03d
ssh swarm
cd /cluster/data/tetNig2/bed/tblastn.hg18KG
mkdir chainRun
cd chainRun
tcsh
cat << '_EOF_' > chainGsub
#LOOP
chainOne $(path1)
#ENDLOOP
'_EOF_'
cat << '_EOF_' > chainOne
(cd $1; cat q.*.psl | simpleChain -prot -outPsl -maxGap=12000 stdin ../c.`basename $1`.psl)
'_EOF_'
chmod +x chainOne
ls -1dS ../blastOut/kg?? > chain.lst
gensub2 chain.lst single chainGsub chainSpec
# do the cluster run for chaining
para create chainSpec
para try, check, push, check etc.
# Completed: 254 of 254 jobs
# CPU time in finished jobs: 590077s 9834.62m 163.91h 6.83d 0.019 y
# IO & Wait Time: 20749s 345.81m 5.76h 0.24d 0.001 y
# Average job time: 2405s 40.08m 0.67h 0.03d
# Longest finished job: 36592s 609.87m 10.16h 0.42d
# Submission to last job: 36604s 610.07m 10.17h 0.42d
cd /cluster/data/tetNig2/bed/tblastn.hg18KG/blastOut
for i in kg??
do
cat c.$i.psl | awk "(\$13 - \$12)/\$11 > 0.6 {print}" > c60.$i.psl
sort -rn c60.$i.psl | pslUniq stdin u.$i.psl
awk "((\$1 / \$11) ) > 0.60 { print }" c60.$i.psl > m60.$i.psl
echo $i
done
sort u.*.psl m60* | uniq | sort -T /tmp -k 14,14 -k 16,16n -k 17,17n > ../blastHg18KG.psl
cd ..
pslCheck blastHg18KG.psl
# checked: 38517 failed: 0 errors: 0
# load table
ssh hgwdev
cd /cluster/data/tetNig2/bed/tblastn.hg18KG
hgLoadPsl tetNig2 blastHg18KG.psl
# check coverage
featureBits tetNig2 blastHg18KG
# 18646943 bases of 332311746 (5.611%) in intersection
featureBits tetNig2 blastHg18KG ensGene -enrichment
# blastHg18KG 5.611%, ensGene 9.520%, both 4.993%, cover 88.97%, enrich 9.35x
rm -rf blastOut
#end tblastn
#########################################################################
# Creating contigs 2bit file (DONE - 2009-08-10 - Hiram)
cd /hive/data/genomes/tetNig2/jkStuff
gapToLift tetNig2 nonBridged.lift -bedFile=nonBridged.bed
# there aren't any gaps in chrM and it doesn't show up in
# the result ! Add chrM to this lift file:
echo -e "0\tchrM.00\t16462\tchrM\t16462" >> nonBridged.lift
echo -e "chrM\t0\t16462\tchrM.00" >> nonBridged.bed
# take a look at the nonBridged.bed as a custom track
mkdir /hive/data/genomes/tetNig2/contigs
cd /hive/data/genomes/tetNig2/contigs
ln -s ../jkStuff/nonBridged.lift .
~/kent/src/hg/utils/lft2BitToFa.pl ../tetNig2.2bit nonBridged.lift \
| gzip -c > tetNig2.contigs.fa.gz
# make sure nothing was destroyed:
faCount *.fa.gz > faCount.contigs.txt 2>&1
twoBitToFa ../tetNig2.2bit stdout | faCount stdin > faCount.2bit.txt 2>&1
tail -1 faCount.contigs.txt
# total 333797246 81050307 70114488 70126181
# 81023812 31482458 9799877
tail -1 faCount.2bit.txt
# total 358618246 81050307 70114488 70126181
# 81023812 56303458 9799877
# only the total size and N count are different
faSize tetNig2.contigs.fa.gz
# 333797246 bases (31482458 N's 302314788 real 241039472 upper 61275316
# lower) in 14352 sequences in 1 files
# %18.36 masked total, %20.27 masked real
twoBitToFa ../tetNig2.2bit stdout | faSize stdin
# 358618246 bases (56303458 N's 302314788 real 241039472 upper 61275316
# lower) in 27 sequences in 1 files
# %17.09 masked total, %20.27 masked real
faToTwoBit tetNig2.contigs.fa.gz tetNig2.contigs.2bit
twoBitInfo tetNig2.contigs.2bit stdout | sort -k2nr > tetNig2.contigs.sizes
cp -p tetNig2.contigs.2bit tetNig2.contigs.sizes \
/hive/data/staging/data/tetNig2
cp -p nonBridged.lift /hive/data/staging/data/tetNig2/tetNig2.contigs.lift
#########################################################################
# GENBANK AUTO UPDATE (DONE - 2009-08-11 - Hiram)
# align with latest genbank process.
ssh hgwdev
cd ~/kent/src/hg/makeDb/genbank
cvsup
# edit etc/genbank.conf to add tetNig2 just before tetNig1
# tetNig2 (Tetraodon)
tetNig2.serverGenome = /hive/data/genomes/tetNig2/tetNig2.2bit
tetNig2.clusterGenome = /scratch/data/tetNig2/tetNig2.2bit
tetNig2.ooc = /scratch/data/tetNig2/tetNig2.11.ooc
tetNig2.lift = /scratch/data/tetNig2/tetNig2.contigs.lift
tetNig2.align.unplacedChroms = chr*_random
tetNig2.refseq.mrna.native.pslCDnaFilter = ${lowCover.refseq.mrna.native.pslCDnaFilter}
tetNig2.refseq.mrna.xeno.pslCDnaFilter = ${lowCover.refseq.mrna.xeno.pslCDnaFilter}
tetNig2.genbank.mrna.native.pslCDnaFilter = ${lowCover.genbank.mrna.native.pslCDnaFilter}
tetNig2.genbank.mrna.xeno.pslCDnaFilter = ${lowCover.genbank.mrna.xeno.pslCDnaFilter}
tetNig2.genbank.est.native.pslCDnaFilter = ${lowCover.genbank.est.native.pslCDnaFilter}
tetNig2.downloadDir = tetNig2
tetNig2.genbank.mrna.xeno.loadDesc = yes
tetNig2.refseq.mrna.native.load = no
cvs ci -m "Added tetNig2" etc/genbank.conf
# update /cluster/data/genbank/:
make etc-update
ssh genbank
screen # use a screen to manage this job
cd /cluster/data/genbank
time nice -n +19 bin/gbAlignStep -initial tetNig2 &
# logFile: var/build/logs/2009.08.10-16:42:06.tetNig2.initalign.log
# real 578m42.777s
# load database when finished
ssh hgwdev
cd /cluster/data/genbank
time nice -n +19 ./bin/gbDbLoadStep -drop -initialLoad tetNig2
# logFile: var/dbload/hgwdev/logs/2009.08.11-09:22:29.dbload.log
# real 31m29.282s
# enable daily alignment and update of hgwdev
cd ~/kent/src/hg/makeDb/genbank
cvsup
# add tetNig2 to:
etc/align.dbs
etc/hgwdev.dbs
cvs ci -m "Added tetNig2 - Tetraodon Nigirividis" \
etc/align.dbs etc/hgwdev.dbs
make etc-update
#########################################################################
# lastz swap Human hg19 (DONE - 2009-08-11 - Hiram)
# result of human alignment (date stamp on directory is incorrect)
cd /hive/data/genomes/hg19/bed/lastzTetNig2.2009-10-10
cat fb.hg19.chainTetNig2Link.txt
# 49611132 bases of 2897316137 (1.712%) in intersection
# running the swap
mkdir /hive/data/genomes/tetNig2/bed/blastz.hg19.swap
cd /hive/data/genomes/tetNig2/bed/blastz.hg19.swap
time nice -n +19 doBlastzChainNet.pl -verbose=2 \
/hive/data/genomes/hg19/bed/lastzTetNig2.2009-10-10/DEF \
-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
-swap > swap.log 2>&1 &
# real 13m21.591s
# forgot the qRepeats for tetNig2
rm axtChain/tetNig2.hg19.net
time nice -n +19 doBlastzChainNet.pl -verbose=2 \
/hive/data/genomes/hg19/bed/lastzTetNig2.2009-10-10/DEF \
-continue=load -qRepeats=windowmaskerSdust \
-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
-swap > load.log 2>&1 &
# real 4m7.559s
cat fb.tetNig2.chainHg19Link.txt
# 42910930 bases of 302314788 (14.194%) in intersection
##############################################################################
# BLASTZ/CHAIN/NET oryLat2 swap (DONE - 2009-09-15 - Hiram)
# original
cd /hive/data/genomes/oryLat2/bed/blastzTetNig2.2009-09-14
cat fb.oryLat2.chainTetNig2Link.txt
# 162783854 bases of 700386597 (23.242%) in intersection
# And, for the swap:
mkdir /hive/data/genomes/tetNig2/bed/blastz.oryLat2.swap
cd /hive/data/genomes/tetNig2/bed/blastz.oryLat2.swap
time doBlastzChainNet.pl -chainMinScore=3000 -chainLinearGap=medium \
/hive/data/genomes/oryLat2/bed/blastzTetNig2.2009-09-14/DEF \
-swap -tRepeats=windowmaskerSdust -qRepeats=windowmaskerSdust \
-noLoadChainSplit -verbose=2 -smallClusterHub=pk -bigClusterHub=pk \
> swap.log 2>&1 &
# real 49m49.335s
cat fb.tetNig2.chainOryLat2Link.txt
# 136115939 bases of 302314788 (45.025%) in intersection
#########################################################################
# BLASTZ/CHAIN/NET gasAcu1 swap (DONE - 2009-09-15 - Hiram)
# original
cd /hive/data/genomes/gasAcu1/bed/lastzTetNig2.2009-08-10
featureBits gasAcu1 chainTetNig2Link >&fb.gasAcu1.chainTetNig2Link.txt
cat fb.gasAcu1.chainTetNig2Link.txt
# 134497679 bases of 446627861 (30.114%) in intersection
mkdir /hive/data/genomes/tetNig2/bed/blastz.gasAcu1.swap
cd /hive/data/genomes/tetNig2/bed/blastz.gasAcu1.swap
time nice -n +19 doBlastzChainNet.pl -verbose=2 \
/hive/data/genomes/gasAcu1/bed/lastzTetNig2.2009-08-10/DEF \
-swap -qRepeats=windowmaskerSdust -qRepeats=windowmaskerSdust \
-noLoadChainSplit -chainMinScore=2000 -chainLinearGap=medium \
-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
> swap.log 2>&1 &
# real 33m6.310s
cat fb.tetNig2.chainGasAcu1Link.txt
# 123285586 bases of 302314788 (40.781%) in intersection
#########################################################################
# LASTZ/CHAIN/NET danRer6 (DONE - 2009-09-15,18 - Hiram)
mkdir /hive/data/genomes/tetNig2/bed/lastzDanRer6.2009-09-15
cd /hive/data/genomes/tetNig2/bed/lastzDanRer6.2009-09-15
cat << '_EOF_' > DEF
# Tetraodon vs. Zebrafish
# using the "close" genome alignment parameters
# see also: http://genomewiki.ucsc.edu/index.php/Mm9_multiple_alignment
BLASTZ_Y=9400
BLASTZ_L=3000
BLASTZ_K=3000
BLASTZ_M=50
BLASTZ_Q=/scratch/data/blastz/HoxD55.q
# TARGET: Tetraodon TetNig2 - single chunk big enough to run single largest item
SEQ1_DIR=/scratch/data/tetNig2/tetNig2.2bit
SEQ1_LEN=/scratch/data/tetNig2/chrom.sizes
SEQ1_CTGDIR=/scratch/data/tetNig2/tetNig2.contigs.2bit
SEQ1_CTGLEN=/scratch/data/tetNig2/tetNig2.contigs.sizes
SEQ1_LIFT=/scratch/data/tetNig2/tetNig2.contigs.lift
SEQ1_CHUNK=20000000
SEQ1_LAP=10000
SEQ1_LIMIT=50
# QUERY: Zebrafish danRer6, chunk large enough to run largest piece
SEQ2_DIR=/scratch/data/danRer6/danRer6.2bit
SEQ2_LEN=/scratch/data/danRer6/chrom.sizes
SEQ2_CTGDIR=/hive/data/genomes/danRer6/contigs/danRer6.contigs.2bit
SEQ2_CTGLEN=/hive/data/genomes/danRer6/contigs/danRer6.contigs.sizes
SEQ2_LIFT=/hive/data/genomes/danRer6/contigs/danRer6.contigs.lift
SEQ2_CHUNK=10000000
SEQ2_LIMIT=100
SEQ2_LAP=0
BASE=/hive/data/genomes/tetNig2/bed/lastzDanRer6.2009-09-15
TMPDIR=/scratch/tmp
'_EOF_'
# << this line keeps emacs coloring happy
time nice -n +19 doBlastzChainNet.pl -verbose=2 \
`pwd`/DEF \
-tRepeats=windowmaskerSdust \
-noLoadChainSplit -chainMinScore=2000 -chainLinearGap=medium \
-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
> do.log 2>&1 &
# real 754m56.563s
cat fb.tetNig2.chainDanRer6Link.txt
# 70626082 bases of 302314788 (23.362%) in intersection
# and the swap to danRer6:
mkdir /hive/data/genomes/danRer6/bed/blastz.tetNig2.swap
cd /hive/data/genomes/danRer6/bed/blastz.tetNig2.swap
time nice -n +19 doBlastzChainNet.pl -verbose=2 \
/hive/data/genomes/tetNig2/bed/lastzDanRer6.2009-09-15/DEF \
-swap -tRepeats=windowmaskerSdust \
-noLoadChainSplit -chainMinScore=2000 -chainLinearGap=medium \
-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
> swap.log 2>&1 &
# real 26m40.310s
cat fb.danRer6.chainTetNig2Link.txt
# 93349443 bases of 1506896106 (6.195%) in intersection
#########################################################################
# LASTZ/CHAIN/NET fr2 (DONE - 2009-09-15,18 - Hiram)
mkdir /hive/data/genomes/tetNig2/bed/lastzFr2.2009-09-15
cd /hive/data/genomes/tetNig2/bed/lastzFr2.2009-09-15
cat << '_EOF_' > DEF
# Tetraodon vs. Zebrafish
# using the "close" genome alignment parameters
# see also: http://genomewiki.ucsc.edu/index.php/Mm9_multiple_alignment
BLASTZ_Y=9400
BLASTZ_L=3000
BLASTZ_K=3000
BLASTZ_M=50
BLASTZ_Q=/scratch/data/blastz/HoxD55.q
# TARGET: Tetraodon TetNig2 - single chunk big enough to run single largest item
SEQ1_DIR=/scratch/data/tetNig2/tetNig2.2bit
SEQ1_LEN=/scratch/data/tetNig2/chrom.sizes
SEQ1_CTGDIR=/scratch/data/tetNig2/tetNig2.contigs.2bit
SEQ1_CTGLEN=/scratch/data/tetNig2/tetNig2.contigs.sizes
SEQ1_LIFT=/scratch/data/tetNig2/tetNig2.contigs.lift
SEQ1_CHUNK=20000000
SEQ1_LAP=10000
SEQ1_LIMIT=50
# QUERY: Zebrafish fr2, chunk large enough to run largest piece
SEQ2_DIR=/scratch/data/fr2/fr2.2bit
SEQ2_LEN=/scratch/data/fr2/chrom.sizes
SEQ2_CTGDIR=/hive/data/genomes/fr2/noUn/fr2.scaffolds.2bit
SEQ2_CTGLEN=/hive/data/genomes/fr2/noUn/fr2.scaffolds.sizes
SEQ2_LIFT=/hive/data/genomes/fr2/jkStuff/liftAll.lft
SEQ2_CHUNK=10000000
SEQ2_LIMIT=100
SEQ2_LAP=0
BASE=/hive/data/genomes/tetNig2/bed/lastzFr2.2009-09-15
TMPDIR=/scratch/tmp
'_EOF_'
# << this line keeps emacs coloring happy
time nice -n +19 doBlastzChainNet.pl -verbose=2 \
`pwd`/DEF \
-tRepeats=windowmaskerSdust \
-noLoadChainSplit -chainMinScore=2000 -chainLinearGap=medium \
-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
> do.log 2>&1 &
# real 1226m51.875s
# broken down during lastz due to pk problems, finished the lastz
# cluster run manually, then continuing:
time nice -n +19 doBlastzChainNet.pl -verbose=2 \
`pwd`/DEF \
-continue=cat -tRepeats=windowmaskerSdust \
-noLoadChainSplit -chainMinScore=2000 -chainLinearGap=medium \
-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
> cat.log 2>&1 &
# real 63m23.911s
cat fb.tetNig2.chainFr2Link.txt
# 243965150 bases of 302314788 (80.699%) in intersection
# and the swap to fr2:
mkdir /hive/data/genomes/fr2/bed/blastz.tetNig2.swap
cd /hive/data/genomes/fr2/bed/blastz.tetNig2.swap
time nice -n +19 doBlastzChainNet.pl -verbose=2 \
/hive/data/genomes/tetNig2/bed/lastzFr2.2009-09-15/DEF \
-swap -tRepeats=windowmaskerSdust \
-noLoadChainSplit -chainMinScore=2000 -chainLinearGap=medium \
-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
> swap.log 2>&1 &
# real 21m56.576s
cat fb.fr2.chainTetNig2Link.txt
# 248984008 bases of 393312790 (63.304%) in intersection
#########################################################################
# BLASTZ/CHAIN/NET mm9/Mouse swap (DONE - 2009-09-15 - Hiram)
# the original alignment to the mouse sequence
cd /hive/data/genomes/mm9/bed/lastzTetNig2.2009-09-15
cat fb.mm9.chainTetNig2Link.txt
# 45642112 bases of 2620346127 (1.742%) in intersection
# running the swap to here, tetNig2:
mkdir /hive/data/genomes/tetNig2/bed/blastz.mm9.swap
cd /hive/data/genomes/tetNig2/bed/blastz.mm9.swap
time nice -n +19 doBlastzChainNet.pl -verbose=2 \
/hive/data/genomes/mm9/bed/lastzTetNig2.2009-09-15/DEF \
-qRepeats=windowmaskerSdust \
-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
-swap > swap.log 2>&1 &
# real 10m34.797s
cat fb.tetNig2.chainMm9Link.txt
# 41176381 bases of 302314788 (13.620%) in intersection
#########################################################################
############################################################################
# TRANSMAP vertebrate.2009-09-13 build (2009-09-20 markd)
vertebrate-wide transMap alignments were built Tracks are created and loaded
by a single Makefile. This is available from:
svn+ssh://hgwdev.cse.ucsc.edu/projects/compbio/usr/markd/svn/projs/transMap/tags/vertebrate.2009-09-13
see doc/builds.txt for specific details.
############################################################################
# GAZE - predictions from Genoscope - (DONE - 2010-01-06 - Hiram)
mkdir /hive/data/genomes/tetNig2/bed/gaze
sed -e "s/ ; /\t/g; s/CDS G/G/;" \
../../genoscope/annotation_tetraodon_v8.2.gff > gaze.gff
ldHgGene -nobin -genePredExt tetNig2 genscan gaze.gff -out=gaze.gp
Read 90337 transcripts in 284179 lines in 1 files
90337 groups 26 seqs 1 sources 4 feature types
26681 gene predictions
hgLoadGenePred tetNig2 gaze gaze.gp
zcat ../../genoscope/Tetraodon_peptide_v8.2.fa.gz \
| sed -e "s/ assembled CDS//;" \
| ~/kent/src/utils/faToTab/faToTab.pl /dev/null /dev/stdin \
| sed -e "s/*$//; s/^GSTENP/GSTENT/;" > gaze.pep.tab
hgPepPred tetNig2 tab gazePep gaze.pep.tab
############################################################################
# all.joiner update - (DONE - 2010-01-06,08 - Hiram)
cd $HOME/kent/src/hg/makeDb/schema
# fixup all.joiner until this is a clean output
joinerCheck -database=tetNig2 -all all.joiner
mkdir /hive/data/genomes/tetNig2/goldenPath
cd /hive/data/genomes/tetNig2/goldenPath
makeDownloads.pl tetNig2 > do.log 2>&1
# now ready for pushQ entry
mkdir /hive/data/genomes/tetNig2/pushQ
cd /hive/data/genomes/tetNig2/pushQ
makePushQSql.pl tetNig2 > tetNig2.pushQ.sql 2> stderr.out
# copy it to hgwbeta
scp -p tetNig2.pushQ.sql hgwbeta:/tmp
ssh hgwbeta
hgsql qapushq < tetNig2.pushQ.sql
# in that pushQ entry walk through each entry and see if the
# sizes will set properly
############################################################################
+# Create cpgIslandExt (DONE - 2010-02-11 - Hiram)
+ mkdir /hive/data/genomes/tetNig2/bed/cpgIslands
+ cd /hive/data/genomes/tetNig2/bed/cpgIslands
+ doCpgIslands.pl -dbHost=hgwdev -workhorse=hgwdev -buildDir=`pwd` tetNig2
+ featureBits tetNig2 cpgIslandExt
+ # 22165067 bases of 302314788 (7.332%) in intersection
+
+############################################################################