Index: src/hg/makeDb/doc/hg17.txt
===================================================================
RCS file: /projects/compbio/cvsroot/kent/src/hg/makeDb/doc/hg17.txt,v
retrieving revision 1.122
retrieving revision 1.123
diff -b -B -U 1000000 -r1.122 -r1.123
--- src/hg/makeDb/doc/hg17.txt	12 Aug 2009 18:00:01 -0000	1.122
+++ src/hg/makeDb/doc/hg17.txt	20 Sep 2009 17:16:43 -0000	1.123
@@ -1,24700 +1,24709 @@
 # for emacs: -*- mode: sh; -*-
 
 
 # This file describes how we made the browser database on 
 # NCBI build 35 (May 2004 freeze)
 
 # HOW TO BUILD AN ASSEMBLY FROM NCBI FILES 
 # ---------------------------------------
 
     # Make gs.18 directory, gs.18/build35 directory, and gs.18/ffa directory.
     ssh eieio
     mkdir /cluster/store5/gs.18
     mkdir /cluster/store5/gs.18/build35
     mkdir /cluster/store5/gs.18/agp
     mkdir /cluster/store5/gs.18/ffa
 
     #    Make a symbolic link from /cluster/store1 to this location
     #	(I assume there is some use for this later ?)
 	
     cd /cluster/store1
     ln -s /cluster/store5/gs.18 ./gs.18
     ln -s /cluster/store5/gs.18/build35 /cluster/data/hg17
 
     #    Make a symbolic link from your home directory to the build dir:
     #	(Investigate what this is used for, may no longer be necessary)
 
     ln -s /cluster/store5/gs.18/build35 ~/oo
 
 # NCBI download site, fetch everything into this one directory:
 
     #	with the machine and password in your $HOME/.netrc file, this
     #	wget command will require no login.  Your $HOME/.netrc file
     #	is set to 'chmod 600 .netrc' to prevent anyone from finding
     #	the data.  (There were some early files that later moved
     #		into an OLD subdirectory.  They were broken.)
     mkdir /cluster/store5/gs.18/ncbi
     cd /cluster/store5/gs.18/ncbi
     wget --timestamping ftp://ftp.ncbi.nlm.nih.gov/build_35/*
 
     # FYI: agp file format documented at:
     #	http://www.ncbi.nlm.nih.gov/Genbank/WGS.agpformat.html    # fixup a couple of names for our own purposes here
     cd /cluster/store5/gs.18/agp
     ln -s ../ncbi/chr*.agp ../ncbi/chr*.fa.gz .
     sed -e "s#MT/NC_001807.4#NC_001807#" ../ncbi/chrMT.agp > chrM.agp
     sed -e "s/NG_002392.2/NG_002392/" ../ncbi/DR52.agp > chr6_hla_hap1.agp
     sed -e "s/NG_002433.1/NG_002433/" ../ncbi/DR53.agp > chr6_hla_hap2.agp
     zcat ../ncbi/DR52.fa.gz | \
 	sed -e "s/gi|29124352|ref|NG_002392.2/ref|NG_002392/" | \
 	gzip > chr6_hla_hap1.fa.gz
     zcat ../ncbi/DR53.fa.gz | \
 	sed -e "s/gi|28212470|ref|NG_002433.1/ref|NG_002433/" | \
 	gzip > chr6_hla_hap2.fa.gz
     zcat ../ncbi/chrMT.fa.gz | \
 	sed -e "s/gi|17981852|ref|NC_001807.4/ref|NC_001807/" | \
 	gzip > chrM.fa.gz
     
 
     #  Put all the agp files together into one.
     cd /cluster/store5/gs.18/build35
     #	The chrM sequence now has its own agp, remove it from
     #	ref_placed.agp
     sed -e "/^NC_001807/d" ../ncbi/ref_placed.agp > ref_placed.agp
     cat ref_placed.agp ../agp/chrM.agp ../ncbi/ref_unplaced.agp \
 	../agp/chr6_hla_hap1.agp ../agp/chr6_hla_hap2.agp \
 	../ncbi/PAR.agp > ncbi_build35.agp
 
     #	and into ffa
     cd /cluster/store5/gs.18/ffa
     #	There is a single bogus line at the end of ref_placed.fa.gz
     #	declaring the NC_001807 MT sequence, this was later replaced by
     #	chrMT.fa.gz, so remove that one line:
     zcat ../ncbi/ref_placed.fa.gz | sed -e "/^>ref|NC_001807/d" | \
 	gzip > ref_placed.fa.gz
     #	(That's a 40 minute job)
 
     #	sequence.inf is usually here, symlink it
     ln -s ../ncbi/sequence.inf
     #	put all the fa.gz files together in one big fa.gz
     time zcat ref_placed.fa.gz ../agp/chrM.fa.gz ../ncbi/ref_unplaced.fa.gz \
 	../agp/chr6_hla_hap?.fa.gz ../ncbi/PAR.fa.gz | gzip \
 	> ncbi_build35.fa.gz
     #	real    37m42.208s
     #	user    37m3.490s
     #	sys     0m31.430s
 
     #	Make a listing of all the fasta record headers, just FYI:
     cd /cluster/store5/gs.18/ffa
     zcat ffa/ncbi_build35.fa.gz | grep "^>" > ncbi.fa.headers
 
 
     #	New to this build is the sequence: NC_001807 which is the
     #	mitochondria sequence.  This prefix NC_ is new to the process
     #	and will have to be accounted for below.  The other two special
     #	prefixes are similar to what was seen before:
     #	from DR52.agp NG_002392
     #	Homo sapiens major histocompatibility complex, class II,
     #		DR52 haplotype (DR52) on chromosome 6
     #	and from DR53.agp NG_002433
     #	Homo sapiens major histocompatibility complex, class II,
     #		DR53 haplotype (DR53) on chromosome 6
 
     #	Fixup seq_contig.md
     #
     #	It has a bunch of stuff belonging to the Celera
     #	genome assembly.  Filter those out.  I don't know what the
     #	NT_07959[0-7] items are, but there are no definitions for them
     #	in the agp files and no sequence in any fa.gz file.
     #	Fixup the names for the NG_ items, and change chrom MT to be M
     cd /cluster/store5/gs.18/build35
     egrep -v "Celera|NT_07959[0-7]" ../ncbi/seq_contig.md | \
 	sed -e "s/6|NG_002392/6_hla_hap1/" \
 	-e "s/6|NG_002433/6_hla_hap2/" \
 	-e "s/^9606\tMT|NC_001807/9606\tM/" \
 	> temp_contig.md
 
     #	get the randoms sorted in proper order.  The createNcbiLifts
     #	does not work correctly if the randoms are not grouped together
     #	by chromosome
     grep -v "|" temp_contig.md  > seq_contig.md
     #	This pulls out all the randoms and groups them within the
     #	same chrom but leaving them in the same order as they orginally
     #	were  (warning this is BASH code ...)
     grep "|" temp_contig.md | awk -F"|" '{print $1}' | \
         awk '{print $2}' | sort -n -u | while read CHR
 do
         grep "[^0-9]${CHR}|" temp_contig.md
 done >> seq_contig.md
 
 
     # Sanity check, checkYbr was updated to handle the NC_ identifier
     time zcat ../ffa/ncbi_build35.fa.gz | $HOME/bin/i386/checkYbr \
 	ncbi_build35.agp stdin seq_contig.md > check.seq_contig
     #	real    2m34.143s
     #	user    2m24.970s
     #	sys     0m8.900s
     #	result should be clean:
     cat check.seq_contig
     #	Read 380 contigs from ncbi_build35.agp
     #	Verifying sequence sizes in stdin
     #	0 problems detected
 
 
     # Convert fa files into UCSC style fa files and place in "contigs"
     # directory inside the gs.18/build35 directory 
     #	(a check that can be done here is make a list of the contigs
     #	in this ./contigs directory before and compare it with the
     #	list of distributed contigs created after they have been
     #	disbursed.)
     #	faNcbiToUcsc was fixed to handle the NC_ identifier
 
     cd /cluster/store5/gs.18/build35
     #	We've been through this often
     mv contigs contigs.0
     time zcat ../ffa/ncbi_build35.fa.gz | $HOME/bin/i386/faNcbiToUcsc \
 	-split -ntLast stdin contigs
     #	real    5m10.938s
     #	user    2m20.070s
     #	sys     0m51.020s
     #	If you want to compare anything to previous work, check now, then:
     rm -fr contigs.0
 
     # Determine the chromosome sizes from agps
     #	Watch carefully how chrY gets constructed.  I'm not sure
     #	this chrom_sizes represents the whole length of chrY with
     #	the PAR added.  We will see about that.
     #	Script updated to handle new chrom names:
     #	my @chroms = (1 .. 22, 'X', 'Y', 'M', '6_hla_hap1', '6_hla_hap2');
 
     cd /cluster/store5/gs.18/build35
     /cluster/bin/scripts/getChromSizes ../agp
     #	Create chrom.lst list for use in foreach() loops
     awk '{print $1}' chrom_sizes | sed -e "s/chr//" > chrom.lst
 
     # Create lift files (this will create chromosome directory structure) and
     #	inserts file
   
     /cluster/bin/scripts/createNcbiLifts -s chrom_sizes seq_contig.md .
 
     # Create contig agp files (will create contig directory structure)
 	
     /cluster/bin/scripts/createNcbiCtgAgp seq_contig.md ncbi_build35.agp .
 
     # Create chromsome random agp files.
 
     /cluster/bin/scripts/createNcbiChrAgp -randomonly .
 
     # Copy the original chrN.agp files from the gs.18/agp directory 
     #    into each of the chromosome directories since they contain better 
     #    gap information. Delete the comments at top from these.
     cd /cluster/store5/gs.18/build35
     foreach c ( `cat chrom.lst` )
 	sed -e "/^#.*/d" ../agp/chr${c}.agp > ./${c}/chr${c}.agp
     end
     #	chrM needs a name fixup
     sed -e "s#NC_001807#chrM#" ../agp/chrM.agp > M/chrM.agp
 
     # Distribute contig .fa to appropriate directory (assumes all files
     # are in "contigs" directory).
 
     # Create inserts file from agp and lift files (new - added by Terry, 2004-07-12)
     /cluster/bin/scripts/createInserts /cluster/data/hg17 > /cluster/data/hg17/inserts
 
     # create global data link for everyone.  No more home directory
     # links required.
     ln -s /cluster/store5/gs.18/build35 /cluster/data/hg17
     cd /cluster/data/hg17
     /cluster/bin/scripts/distNcbiCtgFa contigs .
     #	Verify that everything was moved properly, the contigs directory
     #	should be empty:
     ls contigs
     #	Nothing there, then remove it
     rmdir  contigs
 
     #	Make a list of the contigs for use later
     rm contig.lst
     touch contig.lst
     foreach chrom ( `cat chrom.lst` )
 	foreach c ( $chrom/N{C,G,T}_?????? )
 	    set contig = $c:t
 	    echo "${chrom}/${contig}/${contig}.fa" >> contig.lst
 	end
     end
     #   For later comparisons, this is how many contigs we have:
     wc -l contig.lst
     #	380
 
     #	Note 2004-06-30 - there are some clone numbers left in some of
     #	the NCBI files that are incorrect.  Due to version number
     #	changes, more than one version is listed.  Namely for accession
     #	numbers: AC004491 AC004921 AC004983 AC005088 AC006014 AC099654
     #	The AGP files are correct, the sequence.inf file lists these
     #	twice: AC004491.1 AC004491.2
     #	AC004921.1 AC004921.2 AC004983.2 AC004983.3
     #	AC005088.2 AC005088.3 AC006014.2 AC006014.3
     #	AC099654.4 AC099654.5 
 
     # FILES ARE NOW READY FOR REPEAT MASKING - start that process as
     #	other steps here can proceed in parallel.
 
     #	Previous practice used to copy everything over for jkStuff from a
     #	previous build.  Rather than do that, pick up whatever is needed
     #	at the time it is needed and verify that it is going to do what
     #	you expect.
 
     cd /cluster/data/hg17
     mkdir jkStuff
 
     # Create the contig.gl files - XXX - NCBI doesn't deliver
     # contig_overlaps.agp - 2004-06-18 - this is beginning to come
     # together and there is now a contig_overlaps.agp file
 
     #	This is properly done below with a combination of psLayout
     #	alignments to create the contig_overlaps.agp file
     # /cluster/bin/i386/agpToGl contig_overlaps.agp . -md=seq_contig.md
     # Create chromosome gl files
     # jkStuff/liftGl.csh contig.gl
 
 # CREATING DATABASE  (DONE - 2004-05-20 - Hiram)
     #	RE-DONE for new NIBS - 2004-06-03
     ssh hgwdev
     # Make sure there is at least 5 gig free on hgwdev:/var/lib/mysql
     df -h /var/lib/mysql
 #	Filesystem            Size  Used Avail Use% Mounted on
 #	/dev/sdc1             1.8T  303G  1.4T  19% /var/lib/mysql
 
     # Create the database.
     hgsql -e 'create database hg17' mysql
     # Copy over grp table (for track grouping) from another database:
     hgsql -e "create table grp (PRIMARY KEY(NAME)) select * from hg16.grp" hg17
 
     # ENCODE groups
     # Added 2005-08016 kate
     echo 'UPDATE grp SET priority=7 WHERE name="varRep"'| hgsql hg17
     echo 'UPDATE grp SET priority=8 WHERE name="encode"'| hgsql hg17
     echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeGenes", "ENCODE Regions and Genes", 8.1)' | hgsql hg17
     echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeTxLevels", "ENCODE Transcript Levels", 8.2)' | hgsql hg17
     echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeChip", "ENCODE Chromatin Immunoprecipitation", 8.3)' | hgsql hg17
     echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeChrom", "ENCODE Chromosome, Chromatin and DNA Structure", 8.4)' | hgsql hg17
     echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeCompGeno", "ENCODE Comparative Genomics", 8.5)' | hgsql hg17
     echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeVariation", "ENCODE Variation", 8.6)' | hgsql hg17
     echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeAnalysis", "ENCODE Analysis", 8.9)' | hgsql hg17
     
 
 # MAKE CHROMINFO TABLE WITH (TEMPORARILY UNMASKED) NIBS
 #	(DONE - 2004-05-21 - Hiram)
     #	RE-DONE with new NIBS - 2004-06-03
     # Make nib/, unmasked until RepeatMasker and TRF steps are done.
     # Do this now so that the chromInfo table will exist and thus the
     #	trackDb tables can be built in the next step.
     #	These unmasked nibs will be replaced by the masked nibs after
     #	repeat mask and trf are done.
     ssh eieio
     cd /cluster/data/hg17
     # Make chr*.fa from contig .fa
     #  Copied chrFa.sh from hg16/jkStuff, renamed it to chrFa.csh
     time ./jkStuff/chrFa.csh
     #	real    13m24.710s
     #	user    9m0.360s
     #	sys     1m15.820s
 
     mkdir nib
     foreach c (`cat chrom.lst`)
       foreach f ($c/chr${c}{,_random}.fa)
         if (-e $f) then
           echo "nibbing $f"
           /cluster/bin/i386/faToNib $f nib/$f:t:r.nib
         endif
       end
     end
 
     # Make symbolic links from /gbdb/hg17/nib to the real nibs.
     ssh hgwdev
     mkdir -p /gbdb/hg17/nib
     ln -s /cluster/data/hg17/nib/chr*.nib /gbdb/hg17/nib
     # Load /gbdb/hg17/nib paths into database and save size info.
     cd /cluster/data/hg17
     hgsql hg17  < $HOME/kent/src/hg/lib/chromInfo.sql
     hgNibSeq -preMadeNib hg17 /gbdb/hg17/nib */chr*.fa
     hgsql -N -e "select chrom,size from chromInfo order by chrom" hg17 \
 	> chrom.sizes
     # You can compare this chrom.sizes with the previously created
     # chrom_sizes.  Should be no difference
     sort chrom_sizes > s0
     sort chrom.sizes | grep -v random > s1
     diff s0 s1
     rm s0 s1
 
 # MAKE HGCENTRALTEST ENTRY AND TRACKDB TABLE (DONE - 2004-05-21 - Hiram)
     #	dbDb orderKey updated 2004-06-08 - Hiram
     ssh hgwdev
     #	reset dbDb orderKey - these have never been ordered properly
     #	before, this will get them on the program.
     hgsql -e 'update dbDb set orderKey=11 where name = "hg16";' \
 	-h genome-testdb hgcentraltest
     hgsql -e 'update dbDb set orderKey=12 where name = "hg15";' \
 	-h genome-testdb hgcentraltest
     hgsql -e 'update dbDb set orderKey=13 where name = "hg13";' \
 	-h genome-testdb hgcentraltest
 
     # Enter hg17 into hgcentraltest.dbDb so test browser knows about it:
     hgsql -e 'INSERT INTO dbDb (name, description, nibPath, organism, \
 	defaultPos, active, orderKey, genome, scientificName, \
 	htmlPath, hgNearOk, hgPbOk, sourceName) \
 	VALUES("hg17", "May 2004", "/gbdb/hg17/nib", "Human", \
 	"chr4:56214201-56291736", 1, 10, "Human", "Homo sapiens", \
 	"/gbdb/hg17/html/description.html", 0, 0, "NCBI Build 35");' \
 	-h genome-testdb hgcentraltest
     # Make trackDb table so browser knows what tracks to expect:
     cd ~/kent/src/hg/makeDb/trackDb
     cvs up -d -P .
     # Edit the makefile to add hg17 in all the right places and do
     make update
     make alpha
     cvs commit makefile
 
 # MAKE LIFTALL.LFT, NCBI.LFT (DONE - 2004-05-21 - Hiram)
     #	Re-DONE with new randoms - 2004-06-03 - Hiram)
     cd /cluster/data/hg17
     mkdir -p jkStuff
     cat */lift/{ordered,random}.lft > jkStuff/liftAll.lft
     # Create jkStuff/ncbi.lft for lifting stuff built with the NCBI assembly.
     # Note: this ncbi.lift will not lift floating contigs to chr_random coords,
     # but it will show the strand orientation of the floating contigs 
     # (grep for '|').
     #   mdToNcbiLift seq_contig.md jkStuff/ncbi.lft 
     #	XXXX - appears to be unused, not done - Hiram
 
 # REPEAT MASKING (DONE - 2004-05-24 - Hiram)
     #	The randoms were rearranged after this was first done,
     #	they are re-made below 2004-06-02)
     #	Record the RM version here:
     #	RepBase Update 8.12, RM database version 20040130
     #	as this changes over time and there is no record in the results
 
     # Split contigs, run RepeatMasker, lift results
 
     #	This split takes about 8 minutes
     ssh eieio
     cd /cluster/data/hg17
     foreach chrom ( `cat chrom.lst` )
 	foreach c ( $chrom/N{C,G,T}_?????? )
 	    set contig = $c:t
 	    echo "splitting ${chrom}/${contig}/${contig}.fa"
 	    faSplit size ${chrom}/${contig}/$contig.fa 500000 \
 		${chrom}/${contig}/${contig}_ \
 		-lift=${chrom}/${contig}/$contig.lft -maxN=500000
 	end
     end
 
     #- Make the run directory and job list:
     cd /cluster/data/hg17
     mkdir -p jkStuff
     #  According to RepeatMasker help file, no arguments are required to
     #	specify species because its default is set for primate (human)
     #  This run script saves the .tbl file to be sent to Arian.  He uses
     # those for his analysis.  Sometimes he needs the .cat and .align files for
     # checking problems.  Krish needs the .align files, they are large.
 
     cat << '_EOF_' > jkStuff/RMHuman
 #!/bin/csh -fe
 
 cd $1
 pushd .
 /bin/mkdir -p /tmp/hg17/$2
 /bin/cp $2 /tmp/hg17/$2/
 cd /tmp/hg17/$2
 /cluster/bluearc/RepeatMasker/RepeatMasker -ali -s $2
 popd
 /bin/cp /tmp/hg17/$2/$2.out ./
  if (-e /tmp/hg17/$2/$2.align) /bin/cp /tmp/hg17/$2/$2.align ./
 if (-e /tmp/hg17/$2/$2.tbl) /bin/cp /tmp/hg17/$2/$2.tbl ./
 # if (-e /tmp/hg17/$2/$2.cat) /bin/cp /tmp/hg17/$2/$2.cat ./
 /bin/rm -fr /tmp/hg17/$2/*
 /bin/rmdir --ignore-fail-on-non-empty /tmp/hg17/$2
 /bin/rmdir --ignore-fail-on-non-empty /tmp/hg17
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod +x jkStuff/RMHuman
 
     ssh eieio
     cd /cluster/data/hg17
     mkdir RMRun
     rm -f RMRun/RMJobs
     touch RMRun/RMJobs
     foreach d ( `cat chrom.lst` )
      foreach c ( ${d}/N{C,G,T}_*/N{C,G,T}_*_*.fa )
         set f = $c:t
         set cc = $c:h
         set contig = $cc:t
         echo /cluster/store5/gs.18/build35/jkStuff/RMHuman \
    		/cluster/store5/gs.18/build35/${d}/${contig} $f \
    '{'check out line+ /cluster/store5/gs.18/build35/${d}/${contig}/$f.out'}' \
           >> RMRun/RMJobs
       end
     end
 
     # We have 5971 jobs in RMJobs:
     wc RMRun/RMJobs
     #	5970   41790 1105804 RMRun/RMJobs
 
     #- Do the run
     ssh kk
     cd /cluster/data/hg17/RMRun
     para create RMJobs
     para try, para check, para check, para push, para check,...
 
     #- While that is running, you can run TRF (simpleRepeat) on the small
     # cluster.  See SIMPLE REPEAT section below
 # Completed: 5970 of 5970 jobs
 # CPU time in finished jobs:   45189516s  753158.60m 12552.64h  523.03d  1.433 y
 # IO & Wait Time:                141333s    2355.55m    39.26h    1.64d  0.004 y
 # Average job time:                7593s     126.55m     2.11h    0.09d
 # Longest job:                    10268s     171.13m     2.85h    0.12d
 # Submission to last job:         81484s    1358.07m    22.63h    0.94d
 
     #	Lift up the split-contig .out's to contig-level .out's
     #
     #	If a mistake is made in the following it would be possible to
     #	destroy all the RM output.  So, just to be paranoid, save all
     #	the RM output in bluearc for the time being:
     ssh eieio
 
     cd /cluster/data/hg17
     mkdir /cluster/bluearc/hg17/RMOutput
     foreach c ( `cat chrom.lst` )
      foreach d ( ${c}/N{C,G,T}_* )
 	set T = /cluster/bluearc/hg17/RMOutput/${d}
 	mkdir -p ${T}
         cd ${d}
         set contig = $d:t
         cp -p ${contig}_?{,?,??}.fa.out ${T}
         cd ../..
 	echo "${d} done"
      end
     end
     #	Make sure we got them all:
     #	(this doesn't work later since there are more *.fa.out files
     #	after the lifting.  More explicitly to find just these:
     #		find . -name "N?_*_*.fa.out" -print | wc -l
     find . -name "*.fa.out" -print | wc -l
     #	5970
     find /cluster/bluearc/hg17/RMOutput -type f | wc -l
     #	5970
     #	same count
 
     #	OK, now you can try this operation, do it in a script like this
     #	and save the output of the script for a record of what happened.
 
     cat << '_EOF_' > jkStuff/liftRM.csh
 #!/bin/csh -fe
 foreach c ( `cat chrom.lst` )
  foreach d ( ${c}/N{C,G,T}_* )
     cd $d
     set contig = $d:t
     liftUp $contig.fa.out $contig.lft warn ${contig}_?{,?,??}.fa.out 
     cd ../..
  end
 end
 '_EOF_'
     chmod +x jkStuff/liftRM.csh
     mkdir scriptsOutput
     time jkStuff/liftRM.csh > scriptsOutput/liftRM.1 2>&1
     #	real    4m37.572s
     #	user    1m19.130s
     #	sys     0m32.950s
     #	Check that they all were done:
     grep "fa.out" scriptsOutput/liftRM.1 | wc -l
     #	5959
     #	same count as above
 
     #- Lift up RepeatMask .out files to chromosome coordinates via
     # picked up jkStuff/liftOut2.sh from the hg16 build.  Renamed to
     # liftOut2.csh, changed the line that does the chrom listing
 
     time ./jkStuff/liftOut2.csh > scriptsOutput/liftOut2 2>&1
     #	real    9m46.780s
     #	user    1m18.900s
     #	sys     7m33.990s
 
     #- By this point, the database should have been created (above):
     ssh hgwdev
     cd /cluster/data/hg17
     time hgLoadOut hg17 ?/*.fa.out ??/*.fa.out 6_hla_hap?/*.fa.out > \
 	scriptsOutput/hgLoadOut 2>&1
     #	real    5m59.137s
     #	user    1m47.550s
     #	sys     0m15.410s
 
     # errors during this load:  (there are always a couple of these)
     #	Strange perc. field -6.1 line 243543 of 2/chr2.fa.out
     #	Strange perc. field -5.6 line 243543 of 2/chr2.fa.out
     #	Strange perc. field -6.1 line 243545 of 2/chr2.fa.out
     #	Strange perc. field -5.6 line 243545 of 2/chr2.fa.out
     #	Strange perc. field -0.2 line 30322 of 17/chr17.fa.out
     #	Strange perc. field -0.2 line 30324 of 17/chr17.fa.out
     #	Strange perc. field -0.2 line 30326 of 17/chr17.fa.out
     #	Strange perc. field -0.2 line 30328 of 17/chr17.fa.out
     #	Strange perc. field -18.6 line 77034 of 19/chr19.fa.out
 
     #	Verify we have similar results to previous assembly:
     #	featureBits hg17 rmsk
     #	1391378842 bases of 2867328468 (48.525%) in intersection
     #	featureBits hg16 rmsk
     #	1388770568 bases of 2865248791 (48.469%) in intersection
     #	Now proceed to MASK SEQUENCE BOTH REPEATMASKER AND SIMPLE REPEAT/TRF
     #	following the SIMPLE REPEAT sections below
 
 # Re-Running REPEAT_MASKER on the new Randoms (DONE - 2004-06-02 - Hiram)
     ssh eieio
     cd /cluster/data/hg17
     grep "|" seq_contig.md | awk '{print $2}' | sed -e "s#|#/#" > randoms.lst
 
     mkdir /cluster/data/hg17/RMRandoms
     foreach r ( `cat randoms.lst` )
 	set d = $r:h
 	set contig = $r:t
 	foreach c ( ${r}/N{C,G,T}_*_*.fa )
 	    set f = $c:t
 	    echo /cluster/store5/gs.18/build35/jkStuff/RMHuman \
    		/cluster/store5/gs.18/build35/${d}/${contig} $f \
    '{'check out line+ /cluster/store5/gs.18/build35/${d}/${contig}/$f.out'}' \
           >> RMRandoms/RMJobs
 	end
     end
 
     ssh kk
     cd /cluster/data/hg17/RMRandoms
     para create RMJobs
     para try, para check, para check, para push, para check,...
 # Completed: 94 of 94 jobs
 # CPU time in finished jobs:     221454s    3690.91m    61.52h    2.56d  0.007 y
 # IO & Wait Time:                   866s      14.43m     0.24h    0.01d  0.000 y
 # Average job time:                2365s      39.42m     0.66h    0.03d
 # Longest job:                     9062s     151.03m     2.52h    0.10d
 # Submission to last job:          9106s     151.77m     2.53h    0.11d
 
     #	Continuing with the paranoia theme, let's backup all the RM output
     #
     ssh eieio
 
     cd /cluster/data/hg17
     mkdir /cluster/bluearc/hg17/RMRandoms
     foreach c ( `cat chrom.lst` )
      foreach d ( ${c}/N{C,G,T}_* )
 	set T = /cluster/bluearc/hg17/RMRandoms/${d}
 	mkdir -p ${T}
         cd ${d}
         set contig = $d:t
         cp -p ${contig}_?{,?,??}.fa.out ${T}
         cd ../..
 	echo "${d} done"
      end
     end
     #	Make sure we got them all:
     find . -name "N?_*_*.fa.out" -print | wc -l
     #	5959
     find /cluster/bluearc/hg17/RMRandoms -type f | wc -l
     #	5959
     #	same count
 
 
     time jkStuff/liftRM.csh > scriptsOutput/liftRM2.1 2>&1
     #	real    4m46.302s
     #	user    1m18.260s
     #	sys     0m18.000s
     #	Check that they all were done:
     grep "fa.out" scriptsOutput/liftRM2.1 | wc -l
     #	5959
     #	same count as above
 
     #- Lift up RepeatMask .out files to chromosome coordinates via
     # picked up jkStuff/liftOut2.sh from the hg16 build.  Renamed to
     # liftOut2.csh, changed the line that does the chrom listing
 
     time ./jkStuff/liftOut2.csh > scriptsOutput/liftOut2.1 2>&1
     #	real    2m46.347s
     #	user    1m18.650s
     #	sys     0m15.990s
 
     #- By this point, the database should have been created (above):
     ssh hgwdev
     cd /cluster/data/hg17
     time hgLoadOut hg17 ?/*.fa.out ??/*.fa.out 6_hla_hap?/*.fa.out > \
 	scriptsOutput/hgLoadOut 2>&1
     #	real    5m59.137s
     #	user    1m47.550s
     #	sys     0m15.410s
 
     # errors during this load:  (there are always a couple of these)
     #	Strange perc. field -6.1 line 243543 of 2/chr2.fa.out
     #	Strange perc. field -5.6 line 243543 of 2/chr2.fa.out
     #	Strange perc. field -6.1 line 243545 of 2/chr2.fa.out
     #	Strange perc. field -5.6 line 243545 of 2/chr2.fa.out
     #	Strange perc. field -0.2 line 30322 of 17/chr17.fa.out
     #	Strange perc. field -0.2 line 30324 of 17/chr17.fa.out
     #	Strange perc. field -0.2 line 30326 of 17/chr17.fa.out
     #	Strange perc. field -0.2 line 30328 of 17/chr17.fa.out
     #	Strange perc. field -18.6 line 77034 of 19/chr19.fa.out
 
     #	Verify we have similar results to previous assembly:
     #	featureBits hg17 rmsk
     #	1390952984 bases of 2866216770 (48.529%) in intersection
     #	featureBits hg17 rmsk  #with previous randoms:
     #	1391378842 bases of 2867328468 (48.525%) in intersection
     #	featureBits hg16 rmsk
     #	1388770568 bases of 2865248791 (48.469%) in intersection
     #	Now proceed to MASK SEQUENCE BOTH REPEATMASKER AND SIMPLE REPEAT/TRF
     #	following the SIMPLE REPEAT sections below
 
 # SIMPLE REPEAT [TRF] TRACK (DONE - 2004-05-21 - Hiram)
     #	Re-done with new randoms, 2004-06-02 - Hiram
     #	Copy the contigs, first to the bluearc, then to /iscratch/i
     ssh eieio
     mkdir /cluster/bluearc/hg17
     mkdir /cluster/bluearc/hg17/contigs
 
     cd /cluster/data/hg17
     foreach ctg ( `cat contig.lst` )
 	set c = $ctg:t
  	echo "$ctg > /cluster/bluearc/hg17/contigs/$c"
 	cp -p $ctg /cluster/bluearc/hg17/contigs/$c
     end
     #	Check how much is there:
     #	du -hsc /cluster/bluearc/hg17/contigs
     #	2.8G    /cluster/bluearc/hg17/contigs
 
     # Distribute contigs to /iscratch/i
     ssh kkr1u00
     mkdir /iscratch/i/gs.18/build35/unmaskedContigs
     cd /iscratch/i/gs.18/build35/unmaskedContigs
     cp -p /cluster/bluearc/hg17/contigs/* .
 
     # Verify same amount made it there:
     #	du -hsc /iscratch/i/gs.18/build35/unmaskedContigs
     #	2.8G    /iscratch/i/gs.18/build35/unmaskedContigs
     #	Then send them to the other 7 Iservers
     /cluster/bin/iSync
 
     #	Go to the small cluster for this business:
     ssh kki
 
     mkdir -p /cluster/data/hg17/bed/simpleRepeat
     cd /cluster/data/hg17/bed/simpleRepeat
     mkdir trf
     cat << '_EOF_' > runTrf
 #!/bin/csh -fe
 #
 set path1 = $1
 set inputFN = $1:t
 set outpath = $2
 set outputFN = $2:t
 mkdir -p /tmp/$outputFN
 cp $path1 /tmp/$outputFN
 pushd .
 cd /tmp/$outputFN
 /cluster/bin/i386/trfBig -trf=/cluster/bin/i386/trf $inputFN /dev/null -bedAt=$outputFN -tempDir=/tmp
 popd
 rm -f $outpath
 cp -p /tmp/$outputFN/$outputFN $outpath
 rm -fr /tmp/$outputFN/*
 rmdir --ignore-fail-on-non-empty /tmp/$outputFN
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod +x runTrf
 
     cat << '_EOF_' > gsub
 #LOOP
 ./runTrf {check in line+ $(path1)}  {check out line trf/$(root1).bed}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     ls -1S /iscratch/i/gs.18/build35/unmaskedContigs/*.fa > genome.lst
     gensub2 genome.lst single gsub jobList
     para create jobList
     para try
     para check
     para push
     para check
 # Completed: 380 of 380 jobs
 # CPU time in finished jobs:      13230s     220.49m     3.67h    0.15d  0.000 y
 # IO & Wait Time:                  2078s      34.64m     0.58h    0.02d  0.000 y
 # Average job time:                  40s       0.67m     0.01h    0.00d
 # Longest job:                     1590s      26.50m     0.44h    0.02d
 # Submission to last job:          2504s      41.73m     0.70h    0.03d
 
     liftUp simpleRepeat.bed /cluster/data/hg17/jkStuff/liftAll.lft \
 	warn trf/*.bed  > lu.out 2>&1
 
     # Load into the database:
     ssh hgwdev
     cd /cluster/data/hg17/bed/simpleRepeat
     /cluster/bin/i386/hgLoadBed hg17 simpleRepeat simpleRepeat.bed \
       -sqlTable=$HOME/kent/src/hg/lib/simpleRepeat.sql
     #	Loaded 629076 elements of size 16
     #	Compare with previous assembly
     featureBits hg17 simpleRepeat
     #	54952425 bases of 2866216770 (1.917%) in intersection
 
     #	with previous randoms
     featureBits hg17 simpleRepeat
     #	54964044 bases of 3096628158 (1.775%) in intersection
     featureBits hg16 simpleRepeat
     #	54320136 bases of 2865248791 (1.896%) in intersection
     #	GAPS weren't in hg17 yet at this point, after gaps added:
     #	featureBits hg17 simpleRepeat
     #	54964044 bases of 2867328468 (1.917%) in intersection
     #	featureBits -countGaps hg17 simpleRepeat
     #	54964044 bases of 3096628158 (1.775%) in intersection
 
 ###########################################################################
 # CREATE MICROSAT TRACK (done 2006-7-5 JK)
      ssh hgwdev
      cd /cluster/data/hg17/bed
      mkdir microsat
      cd microsat
      awk '($5==2 || $5==3) && $6 >= 15 && $8 == 100 && $9 == 0 {printf("%s\t%s\t%s\t%dx%s\n", $1, $2, $3, $6, $16);}' ../simpleRepeat/simpleRepeat.bed > microsat.bed 
     /cluster/bin/i386/hgLoadBed hg17 microsat microsat.bed
 
 
 # PROCESS SIMPLE REPEATS INTO MASK (DONE - 2004-05-21 - Hiram)
     #	re-done with new randoms - 2004-06-03 - Hiram
     # After the simpleRepeats track has been built, make a filtered version 
     # of the trf output: keep trf's with period <= 12:
     ssh eieio
     cd /cluster/data/hg17/bed/simpleRepeat
     mkdir -p trfMask
     foreach f (trf/*.bed)
       awk '{if ($5 <= 12) print;}' $f > trfMask/$f:t
     end
 
     #	EXPERIMENT, at a filter of <= 12, we have coverage:
     #	20904399 bases of 2867328468 (0.729%) in intersection
     #	at a filter of <= 9, we have coverage:
     #	19271270 bases of 2867328468 (0.672%) in intersection
 
 
     # Lift up filtered trf output to chrom coords as well:
     cd /cluster/data/hg17
     mkdir bed/simpleRepeat/trfMaskChrom
     foreach c ( `cat chrom.lst` )
       if (-e $c/lift/ordered.lst) then
         perl -wpe 's@(\S+)@bed/simpleRepeat/trfMask/$1.bed@' \
           $c/lift/ordered.lst > $c/lift/oTrf.lst
         liftUp bed/simpleRepeat/trfMaskChrom/chr$c.bed \
           jkStuff/liftAll.lft warn `cat $c/lift/oTrf.lst`
       endif
       if (-e $c/lift/random.lst) then
         perl -wpe 's@(\S+)@bed/simpleRepeat/trfMask/$1.bed@' \
            $c/lift/random.lst > $c/lift/rTrf.lst
         liftUp bed/simpleRepeat/trfMaskChrom/chr${c}_random.bed \
           jkStuff/liftAll.lft warn `cat $c/lift/rTrf.lst`
       endif
     end
 
 # MASK SEQUENCE BOTH REPEATMASKER AND SIMPLE REPEAT/TRF (DONE - 2004-05-25)
 #							 -Hiram
     #	re-done with new randoms - 2004-06-03 - Hiram
     # This used to be done right after RepeatMasking.  Now, we mask with 
     # TRF as well, so do this after the "PROCESS SIMPLE REPEATS" step above,
     #	and after Repeat Masker is complete.
     ssh eieio
     cd /cluster/data/hg17
 
     # copied these scripts from hg16 - reset the lines that make
     # the chrom list to work on, reset the wild cards that find all the
     # contig .fa's
 
     # Make chr*.fa from contig .fa
     #  Copied chrFa.sh from hg16/jkStuff, renamed it to chrFa.csh
     time ./jkStuff/chrFa.csh > scriptsOutput/chrFa.out 2>&1 &
     #	real    13m18.512s
     #	user    9m1.670s
     #	sys     1m7.290s
 
     #- Soft-mask (lower-case) the contig and chr .fa's
     time ./jkStuff/makeFaMasked.csh > scriptsOutput/maFaMasked.out 2>&1
     #	real    29m31.623s
     #	user    13m49.700s
     #	sys     5m58.750s
     #- Make hard-masked .fa.masked files as well:
     time ./jkStuff/makeHardMasked.csh > scriptsOutput/maHardMasked.out 2>&1
 
     #- Create the bothMasksNib/ directory
     time ./jkStuff/makeNib.csh > scriptsOutput/maNib.out 2>&1
     #	real    14m41.694s
     #	user    6m28.000s
     #	sys     1m42.500s
 
     # Make symbolic links from /gbdb/hg17/nib to the real nibs.
     ssh hgwdev
     mv nib nib.raw
     mv bothMasksNib nib
     rm /gbdb/hg17/nib/*.nib
     ln -s `pwd`/nib/* /gbdb/hg17/nib
 
     # Load /gbdb/hg17/nib paths into database and save size info.
     hgsql hg17  < ~/kent/src/hg/lib/chromInfo.sql
     cd /cluster/data/hg17
     hgNibSeq -preMadeNib hg17 /gbdb/hg17/nib */chr*.fa
     #	3096628158 total bases
 
     #	Should be the same size as before
     hgsql -N -e "select chrom,size from chromInfo order by chrom" hg17 \
 	> chrom.sizes.masked
     diff chrom.sizes chrom.sizes.masked
     #	should be no output at all, thus:
     rm chrom.sizes.masked
 
     # Copy the masked contig fa to /scratch and /iscratch
     #	And everything else we will need for blastz runs, etc ...
     #	Best to do this sequence first to /cluster/bluearc/scratch,
     #	which is going to be the source for the /scratch copy.
     #	And then from there to the /iscratch
     #	Make sure you are on the fileserver for the original source:
     ssh eieio
     mkdir -p /cluster/bluearc/scratch/hg/gs.18/build35
     cd /cluster/bluearc/scratch/hg/gs.18/build35
 
     #	these copies take less than 2 minutes each
     mkdir bothMaskedNibs
     cp -p /cluster/data/hg17/nib/*.nib ./bothMaskedNibs
     mkdir maskedContigs
     foreach chrom ( `cat /cluster/data/hg17/chrom.lst` )
 	cp -p /cluster/data/hg17/${chrom}/N{C,G,T}_*/N{C,G,T}_??????.fa \
 		./maskedContigs
 	echo "done ${chrom}"
     end
     #	make sure you have them all:
     ls maskedContigs | wc -l
     #	380
     wc -l /cluster/data/hg17/contig.lst
     #	380
     mkdir rmsk
     foreach chrom ( `cat /cluster/data/hg17/chrom.lst` )
 	cp -p /cluster/data/hg17/${chrom}/*.out ./rmsk
 	echo "done ${chrom}"
     end
 
     #	Now, go to the destination for /iscratch and copy from the
     #	bluearc
     ssh kkr1u00
     mkdir -p /iscratch/i/gs.18/build35
     cd /iscratch/i/gs.18/build35
     #	This takes about 5 minutes
     rsync -arlv /cluster/bluearc/scratch/hg/gs.18/build35/ .
 
     time /cluster/bin/iSync
     #	real    7m27.649s
 
     # request rsync of /cluster/bluearc/scratch to the KiloKluster /scratch
 
 # LOAD ctgPos table - Contig position track (DONE - 2004-06-08 - Hiram)
     #	After fixing up hgCtgPos to accept the -chromLst argument, simply:
     cd /cluster/data/hg17
     hgCtgPos -chromLst=chrom.lst hg17 .
 
 # GOLD AND GAP TRACKS (DONE - 2004-05-21 - Hiram)
     #	RE-DONE with new randoms - 2004-06-03 - Hiram
     ssh hgwdev
     cd /cluster/data/hg17
     hgGoldGapGl -noGl -chromLst=chrom.lst hg17 /cluster/data/hg17 .
     #	Disappointing to see this create so many tables ...
     #	_gap and _gold for each chrom
 
     # Create the contig.gl files - XXX - NCBI doesn't deliver
     # contig_overlaps.agp - 2004-06-18 - this is beginning to come
     # together and there is now a contig_overlaps.agp file
     cd /cluster/store5/gs.18/build35
     #	combine the various psLayout attempts on different sections of
     #	clones
     ./combineContigOverlaps.sh
     #	Turn contig_overlaps.agp into gl files
     ~hiram/bin/i386/agpToGl contig_overlaps.agp . -md=seq_contig.md
     # Create chromosome gl files  (had to fix liftUp to do the NC_ properly)
     jkStuff/liftGl.csh contig.gl
     #
     #	Need to remove these PAR clone names from chrY.gl
     rm -f /tmp/chrY.gl
     sed -e "/^AL954722.18/d; /^BX537334.4/d; /^BX000483.7/d; \
 	/^BX908402.3/d; /^BX649635.3/d; /^BX119919.5/d; \
 	/^AC079176.15/d; /^AC097314.27/d; /^AC006209.25/d; \
 	/^AJ271735.1/d; /^AJ271736.1/d" Y/chrY.gl > /tmp/chrY.gl
     rm -f Y/chrY.gl
     mv /tmp/chrY.gl Y/chrY.gl
 
 #  After contig.gl files have been made from contig_overlaps.agp
 #	The sed fixes the Celera clones that are marked phase W
 #	Call that phase 3 instead,
 #	Delete the Celera AACC clones, they are not in this assembly,
 #	And fix the line of AC018743 to add it to the assembly, it was a
 #	last minute addition by Terry that didn't get carried into the
 #	NCBI sequence.inf file.  And remove the older versions of five
 #	clones that got left in by mistake at NCBI
     #	AC004491.1=AC004491.2 AC004921.1=AC004921.2 AC004983.2=AC004983.3
     #	AC005088.2=AC005088.3 AC006014.2=AC006014.3 AC099654.4=AC099654.5 
 #	And finally the grep selects only those things for_assembly
     cd /cluster/data/hg17
     egrep "for_assembly|AC018743" /cluster/store5/gs.18/ncbi/sequence.inf | \
     sed -e "s/\tW\t/\t3\t/; /^AACC010000.*/d; /^AC004491.1.*/d; \
 	/^AC004921.1.*/d; /^AC004983.2.*/d; /^AC005088.2.*/d; \
 	/^AC006014.2.*/d; /^AC099654.4.*/d; \
 	s/AC018743.27\t31791062\t466818\t1\tD\tUn\t-\tBCM\tRP11-289M22\tSIZE:2big/AC018743.27\t31791062\t466818\t1\t-\t(12)\t-\tBCM\tRP11-289M22\tfor_assembly/" \
 	> sequence.inf
     cd /cluster/data/hg17
     hgGoldGapGl -chromLst=chrom.lst hg17 /cluster/store5/gs.18 build35
     $HOME/bin/i386/hgClonePos -chromLst=chrom.lst hg17 \
 	/cluster/data/hg17 ./sequence.inf /cluster/store5/gs.18 -maxErr=3 \
 	-maxWarn=2000 2> clone.pos.errors
     #	Extract all the PAR clones for chrX from clonePos, change the X
     #	to Y, fixup the coordinates on the last three, and load this
     #	data in on the clonePos table in addition to what is there
     #	already.
     cat << '_EOF_' > chrY.par.clonePos
 BX640545.2      34821   3       chrY    0       34250   F
 AL954722.18     37771   3       chrY    84821   122592  F
 BX537334.4      36872   3       chrY    120592  157464  F
 BX000483.7      15918   3       chrY    155466  171384  F
 AL954664.17     39200   3       chrY    251384  290307  F
 BX000476.5      33340   3       chrY    282188  315528  F
 AL732314.18     218723  3       chrY    313528  532251  F
 BX004827.18     119555  3       chrY    479050  600112  F
 AL683871.15     175765  3       chrY    598112  773877  F
 AL672311.26     115998  3       chrY    771877  887875  F
 AL672277.20     131682  3       chrY    885875  1017557 F
 BX908402.3      36556   3       chrY    1067557 1104113 F
 BX649635.3      43709   3       chrY    1154113 1197822 F
 BX649553.5      90286   3       chrY    1347822 1438108 F
 BX296563.3      21008   3       chrY    1488108 1509117 F
 BX119906.16     35666   3       chrY    1507116 1542782 F
 AL683870.15     162377  3       chrY    1541782 1704175 F
 AL691415.17     45085   3       chrY    1702175 1747265 F
 AL683807.22     189825  3       chrY    1745260 1935086 F
 AL672040.10     117297  3       chrY    1933086 2050383 F
 BX004859.8      63432   3       chrY    2048380 2111815 F
 BX119919.5      55442   3       chrY    2261815 2317257 F
 AC079176.15     186278  3       chrY    2311674 2497952 F
 AC097314.27     80501   3       chrY    2495948 2576449 F
 AC006209.25     141759  3       chrY    2551122 2692881 F
 AJ271735.1      240000  3       chrY    57302979        57543030        F
 AJ271736.1      158661  3       chrY    57543030        57701691        F
 '_EOF_'
 
 hgsql -e 'load data local infile "chrY.par.clonePos" into table clonePos;' hg17
 
     #	We have the following errors
 # Processing /cluster/data/hg17/Y/chrY.gl
 # Clone BX640545 is on chromosomes chrX and chrY.  Ignoring chrY
 # Clone AL954722 is on chromosomes chrX and chrY.  Ignoring chrY
 # ... etc for all the PAR clones
 # ... And there are an unknown number of these:
 # AB000359 is in ./sequence.inf but not in ooDir/*/*.gl
 # AB000360 is in ./sequence.inf but not in ooDir/*/*.gl
 
 #  gc5Base wiggle TRACK (DONE - 2004-05-22 - Hiram)
     #	This previously was a script that ran through each nib
     #	Recently transformed into a mini cluster run.
     #	Re-DONE with the new randoms - 2004-06-04
     ssh kki
     mkdir /cluster/data/hg17/bed/gc5Base
     cd /cluster/data/hg17/bed/gc5Base
 
     mkdir wigData5 dataLimits5 wigData5_1K dataLimits5_1K
 
     cat << '_EOF_' > kkRun.sh
 #!/bin/sh
 NIB=$1
 
 chr=${NIB/.nib/}
 chrom=${chr#chr}
 
 hgGcPercent -chr=${chr} -doGaps -file=stdout -win=5 hg17 \
         /iscratch/i/gs.18/build35/bothMaskedNibs | \
     grep -w GC | \
     awk '{if (($3-$2) >= 5) {printf "%d\t%.1f\n", $2+1, $5/10.0} }' | \
     wigAsciiToBinary -dataSpan=5 -chrom=${chr} \
         -wibFile=wigData5/gc5Base_${chrom} \
             -name=${chrom} stdin 2> dataLimits5/${chr}
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod +x kkRun.sh
 
     ls /iscratch/i/gs.18/build35/bothMaskedNibs > nibList
     cat << '_EOF_' > gsub
 #LOOP
 ./kkRun.sh $(path1)
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     gensub2 nibList single gsub jobList
     para create jobList
     para try, check, ... etc
 # Completed: 46 of 46 jobs
 # CPU time in finished jobs:       5251s      87.51m     1.46h    0.06d  0.000 y
 # IO & Wait Time:                   130s       2.17m     0.04h    0.00d  0.000 y
 # Average job time:                 117s       1.95m     0.03h    0.00d
 # Longest job:                      413s       6.88m     0.11h    0.00d
 # Submission to last job:           475s       7.92m     0.13h    0.01d
 
     # load the .wig files back on hgwdev:
     ssh hgwdev
     cd /cluster/data/hg17/bed/gc5Base
     hgLoadWiggle -pathPrefix=/gbdb/hg17/wib/gc5Base hg17 gc5Base wigData5/*.wig
     # and symlink the .wib files into /gbdb
     mkdir /gbdb/hg17/wib/gc5Base
     ln -s `pwd`/wigData5/*.wib /gbdb/hg17/wib/gc5Base
 
     #	And then the zoomed data view
     ssh kki
     cd /cluster/data/hg17/bed/gc5Base
     mkdir wigData5_1K dataLimits5_1K
 
     cat << '_EOF_' > kkRunZoom.sh
 #!/bin/sh
 NIB=$1
 
 chr=${NIB/.nib/}
 chrom=${chr#chr}
 
 hgGcPercent -chr=${chr} -doGaps -file=stdout -win=5 hg17 \
         /iscratch/i/gs.18/build35/bothMaskedNibs | \
     grep -w GC | \
     awk '{if (($3-$2) >= 5) {printf "%d\t%.1f\n", $2+1, $5/10.0} }' | \
     wigZoom -dataSpan=1000 stdin | wigAsciiToBinary -dataSpan=1000 \
 	-chrom=${chr} -wibFile=wigData5_1K/gc5Base_${chrom}_1K \
             -name=${chrom} stdin 2> dataLimits5_1K/${chr}
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod +x kkRunZoom.sh
 
     cat << '_EOF_' > gsubZoom
 #LOOP
 ./kkRunZoom.sh $(path1)
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     gensub2 nibList single gsubZoom jobListZoom
     para create jobListZoom
     para try ... check ... etc ...
 # Completed: 46 of 46 jobs
 # CPU time in finished jobs:       5216s      86.93m     1.45h    0.06d  0.000 y
 # IO & Wait Time:                    34s       0.57m     0.01h    0.00d  0.000 y
 # Average job time:                 114s       1.90m     0.03h    0.00d
 # Longest job:                      415s       6.92m     0.12h    0.00d
 # Submission to last job:           469s       7.82m     0.13h    0.01d
 
     #	Then load these .wig files into the same database as above
     ssh hgwdev
     hgLoadWiggle -pathPrefix=/gbdb/hg17/wib/gc5Base \
 	-oldTable hg17 gc5Base wigData5_1K/*.wig
     # and symlink these .wib files into /gbdb
     mkdir -p /gbdb/hg17/wib/gc5Base
     ln -s `pwd`/wigData5_1K/*.wib /gbdb/hg17/wib/gc5Base
 
 # AUTO UPDATE GENBANK MRNA RUN  (DONE - 2004-06-08 - Hiram)
     ssh eieio
     cd /cluster/data/genbank
     # This is a new organism, edit the etc/genbank.conf file and add:
 	# hg17
 	hg17.genome = /scratch/hg/gs.18/build35/bothMaskedNibs/chr*.nib
 	hg17.lift = /cluster/store5/gs.18/build35/jkStuff/liftAll.lft
 	hg17.genbank.est.xeno.load = yes
 	hg17.mgcTables.default = full
 	hg17.mgcTables.mgc = all
 	hg17.downloadDir = hg17
 
     #	Do the refseq's first, they are the quick ones
     ssh eieio
     cd /cluster/data/genbank
     nice bin/gbAlignStep -srcDb=refseq -type=mrna -verbose=1 -initial hg17
     #	 logFile: var/build/logs/2004.05.25-13:41:07.hg17.initalign.log
     #	checking that log, or watching the batch on kk, you can find
     #	where the batch is running and after it is done get the time:
     cd /cluster/store6/genbank/work/initial.hg17/align
     para time > time
     cat time
 # Completed: 9500 of 9500 jobs
 # CPU time in finished jobs:      62241s    1037.35m    17.29h    0.72d  0.002 y
 # IO & Wait Time:                 33719s     561.98m     9.37h    0.39d  0.001 y
 # Average job time:                  10s       0.17m     0.00h    0.00d
 # Longest job:                     1062s      17.70m     0.29h    0.01d
 # Submission to last job:          1063s      17.72m     0.30h    0.01d
 
     # Load the results from the above
     ssh hgwdev
     cd /cluster/data/genbank
     nice bin/gbDbLoadStep -verbose=1 -drop -initialLoad hg17
 
 #	To get the genbank started, the above results need to be
 #	moved out of the way.  These things can be removed if there are
 #	no problems to debug
     ssh eieio
     cd /cluster/data/genbank/work
     mv initial.hg17 initial.hg17.refseq.mrna
 
     cd /cluster/data/genbank
     nice bin/gbAlignStep -srcDb=genbank -type=mrna -verbose=1 -initial hg17
     #	logFile: var/build/logs/2004.06.04-10:47:21.hg17.initalign.log
     #	One job was hung up, after killing it on its node, the batch
     #	finished in a few minutes.
 # Completed: 35720 of 35720 jobs
 # CPU time in finished jobs:    5161424s   86023.74m  1433.73h   59.74d  0.164 y
 # IO & Wait Time:                144149s    2402.48m    40.04h    1.67d  0.005 y
 # Average job time:                 149s       2.48m     0.04h    0.00d
 # Longest job:                    18306s     305.10m     5.08h    0.21d
 # Submission to last job:         35061s     584.35m     9.74h    0.41d
 
     ssh hgwdev
     cd /cluster/data/genbank
     #	some kind of error happened here, had to remove a lock file to
     #	get this to proceed  (this same thing happened again the second
     #	time around)
     nice bin/gbDbLoadStep -verbose=1 -drop -initialLoad hg17
 
     ssh eieio
     cd /cluster/data/genbank/work
     mv initial.hg17 initial.hg17.genbank.mrna
     cd /cluster/data/genbank
     nice bin/gbAlignStep -srcDb=genbank -type=est -verbose=1 -initial hg17
 # Completed: 189240 of 189240 jobs
 # CPU time in finished jobs:   97172120s 1619535.33m 26992.26h 1124.68d  3.081 y
 # IO & Wait Time:               1507789s   25129.82m   418.83h   17.45d  0.048 y
 # Average job time:                 521s       8.69m     0.14h    0.01d
 # Longest job:                    33165s     552.75m     9.21h    0.38d
 # Submission to last job:        126988s    2116.47m    35.27h    1.47d
 
     ssh hgwdev
     cd /cluster/data/genbank
     time nice bin/gbDbLoadStep -verbose=1 -drop -initialLoad hg17
     #	real    440m42.750s
     #	user    69m7.810s
     #	sys     23m18.640s
     #	This is ~7.5 hours
 
     #	If the above is all OK, ask Mark to put this assembly on
     #	the daily updates.
 
 # CPGISLANDS (DONE - 2004-05-25 - Hiram)
     #	Re-DONE with new randoms - 2004-06-04 - Hiram
     ssh hgwdev
     mkdir -p /cluster/data/hg17/bed/cpgIsland
     cd /cluster/data/hg17/bed/cpgIsland
 
     # Build software from Asif Chinwalla (achinwal@watson.wustl.edu)
     cvs co hg3rdParty/cpgIslands
     cd hg3rdParty/cpgIslands
     make
     #	gcc readseq.c cpg_lh.c -o cpglh.exe
     mv cpglh.exe /cluster/data/hg17/bed/cpgIsland/
     
     # cpglh.exe requires hard-masked (N) .fa's.  
     # There may be warnings about "bad character" for IUPAC ambiguous 
     # characters like R, S, etc.  Ignore the warnings.  
     ssh eieio
     cd /cluster/data/hg17/bed/cpgIsland
     foreach f (../../*/chr*.fa.masked)
       set fout=$f:t:r:r.cpg
       echo running cpglh on $f to $fout
       ./cpglh.exe $f > $fout
     end
     #	the warnings:
     # Bad char 0x52 = 'R' at line 2046, base 102229, sequence chr16_random
     # Bad char 0x4d = 'M' at line 1216113, base 60805573, sequence chr3
     # Bad char 0x52 = 'R' at line 1216118, base 60805801, sequence chr3
     # Bad char 0x52 = 'R' at line 1216118, base 60805801, sequence chr3
     #	real    21m47.823s
     #	user    18m30.810s
     #	sys     1m13.420s
 
     # Transform cpglh output to bed +
     cat << '_EOF_' > filter.awk
 /* Input columns: */
 /* chrom, start, end, len, CpG: cpgNum, perGc, cpg:gpc, observed:expected */
 /* chr1\t 41776\t 42129\t 259\t CpG: 34\t 65.8\t 0.92\t 0.94 */
 /* Output columns: */
 /* chrom, start, end, name, length, cpgNum, gcNum, perCpg, perGc, obsExp */
 /* chr1\t41775\t42129\tCpG: 34\t354\t34\t233\t19.2\t65.8\to0.94 */
 {
 $2 = $2 - 1;
 width = $3 - $2;
 printf("%s\t%d\t%s\t%s %s\t%s\t%s\t%0.0f\t%0.1f\t%s\t%s\n",
        $1, $2, $3, $5,$6, width,
        $6, width*$7*0.01, 100.0*2*$6/width, $7, $9);
 }
 '_EOF_'
     # << this line makes emacs coloring happy
     awk -f filter.awk chr*.cpg > cpgIsland.bed
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/cpgIsland
     hgLoadBed hg17 cpgIslandExt -tab -noBin \
       -sqlTable=$HOME/kent/src/hg/lib/cpgIslandExt.sql cpgIsland.bed
     #	Reading cpgIsland.bed
     #	Loaded 27801 elements of size 10
     #	Sorted
     #	Saving bed.tab
     #	Loading hg17
 
 # MAKE HGCENTRALTEST BLATSERVERS ENTRY (DONE - 2004-05-25 - Heather)
     ssh hgwdev
     hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans) \
 	VALUES("hg17", "blat12", "17778", "1"); \
 	INSERT INTo blatServers (db, host, port, isTrans) \
 	VALUES("hg17", "blat12", "17779", "0");' \
 	-h genome-testdb hgcentraltest
 
 # PREPARE CLUSTER FOR BLASTZ RUNS (DONE - 2004-05-26 - Hiram)
     #	Re-DONE with new randoms - 2004-06-03 - Hiram
 
     ssh eieio
     mkdir /cluster/bluearc/scratch/hg/gs.18/build35/rmsk.spec
     cd /cluster/bluearc/scratch/hg/gs.18/build35/rmsk.spec
     ln -s ../rmsk/*.out .
     #	This takes 40 minutes run as a script, to hurry it up it has
     #	been converted to a mini cluster run
     cat << '_EOF_' > runArian.sh
 #!/bin/sh
 for FN in *.out
 do
     echo /cluster/bluearc/RepeatMasker030619/DateRepsinRMoutput.pl \
 	${FN} -query human -comp rat -comp mouse
 done
 '_EOF_'
     chmod +x runArian.sh
     ssh kki
     cd /cluster/bluearc/scratch/hg/gs.18/build35/rmsk.spec
     ./runArian.sh > jobList
     para create jobList
     para try, ... check ... push ... etc ...
 # Completed: 46 of 46 jobs
 # CPU time in finished jobs:        668s      11.14m     0.19h    0.01d  0.000 y
 # IO & Wait Time:                   514s       8.56m     0.14h    0.01d  0.000 y
 # Average job time:                  26s       0.43m     0.01h    0.00d
 # Longest job:                       86s       1.43m     0.02h    0.00d
 # Submission to last job:           108s       1.80m     0.03h    0.00d
 
     #	Now extract each one, 1 = Rat, 2 = Mouse
     ssh eieio
     cd /cluster/bluearc/scratch/hg/gs.18/build35
 
     mkdir linSpecRep.notInRat linSpecRep.notInMouse
     foreach f (rmsk.spec/*.out_rat_mus)
         set base = $f:t:r:r
         echo "$f -> $base.out.spec"
         /cluster/bin/scripts/extractLinSpecReps 1 $f > \
                         linSpecRep.notInRat/$base.out.spec
         /cluster/bin/scripts/extractLinSpecReps 2 $f > \
                         linSpecRep.notInMouse/$base.out.spec
     end
     #	There is actually no difference at all between these two results.
     #	copy to iscratch
     ssh kkr1u00
     cd /iscratch/i/gs.18/build35
     rsync -arlv /cluster/bluearc/scratch/hg/gs.18/build35/ .
     /cluster/bin/iSync
     # request rsync of /cluster/bluearc/scratch to the KiloKluster /scratch
 
 # COPY DATA TO GOLDEN PATH LOCATIONS (DONE - 2004-06-04 - Hiram)
     ssh hgwdev
     mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/chromosomes
     cd /cluster/data/hg17
     #	Beware, this backgrounding of the gzips can be hard on hgwdev.
     #	You could wait until after the copy then run one gzip to do them all
     foreach chrom ( `cat chrom.lst` )
 	cp -p ${chrom}/*.fa /usr/local/apache/htdocs/goldenPath/hg17/chromosomes
 	gzip \
 	/usr/local/apache/htdocs/goldenPath/hg17/chromosomes/chr${chrom}*.fa &
 	echo "done ${chrom}"
     end
     cd /usr/local/apache/htdocs/goldenPath/hg17/chromosomes
     gzip *.fa
 
 
 # FOSMID END PAIRS TRACK (2004-06-09 kate)
 # Corrected upper size limit to 50Kbp, reran pslPairs, 
 # and reloaded (2004-07-15 kate)
 
     # Use latest fosmid ends data prepared by Terry Furey.
     # He says there is no on-going work on fosmid ends, so this
     # should suffice indefinitely ?  Move/link this stuff into
     # central data area.
     ssh eieio
     cd /cluster/data/ncbi
     mkdir -p fosends/human
     ln -s /cluster/store1/fosends.3 fosends/human
     cd fosends/human/fosends.3
     faSize fosEnds.fa
        # 579735181 bases (369769 N's 579365412 real) in 1087670 sequences 
        # 580M bases in 1M sequences
     # create link in /gbdb/ncbi/fosends/human ?
 
     # use pre-split fosend files, and associated list for cluster run
     # Sequences are in /cluster/bluearc/hg/fosEnds
     cp /cluster/bluearc/booch/fosends/fosEnds.lst /cluster/bluearc/hg/fosEnds
     
     # run on rack9 since kilokluster is busy
     ssh kk9
     cd /cluster/data/hg17
     mkdir -p bed/fosends
     cd bed/fosends
     mkdir -p run
     cd run
     ls -1S /scratch/hg/gs.18/build35/maskedContigs/*.fa \
                 > contigs.lst
     cp /cluster/bluearc/hg/fosEnds/fosEnds.lst fosEnds.lst
         # 380 contigs vs 97 fosEnd files -> 40K jobs
     # send output to kksilo, as it can better handle the NFS load
     mkdir -p /cluster/store7/kate/hg17/fosends/out
     ln -s /cluster/store7/kate/hg17/fosends/out ../out
 cat > gsub << 'EOF'
 #LOOP
 /cluster/bin/i386/blat $(path1) $(path2) -ooc=/scratch/hg/h/11.ooc {check out line+ /cluster/data/hg17/bed/fosends/out/$(root2)/$(root1).$(root2).psl}
 #ENDLOOP
 'EOF'
     gensub2 contigs.lst fosEnds.lst gsub jobList
     foreach f (`cat fosEnds.lst`)
         set d = $f:r:t
         echo $d
         mkdir -p /cluster/data/hg17/bed/fosends/out/$d
     end
 
     para create jobList
         # 36860 jobs
     para try
     para check
     para push
 # CPU time in finished jobs:    1655943s   27599.05m   459.98h   19.17d  0.053 y
 # IO & Wait Time:                101145s    1685.75m    28.10h    1.17d  0.003 y
 # Average job time:                  48s       0.79m     0.01h    0.00d
 # Longest job:                     1294s      21.57m     0.36h    0.01d
 # Submission to last job:         19269s     321.15m     5.35h    0.22d
 
     # sort, filter, and lift alignments
     ssh eieio
     cd /cluster/data/hg17/bed/fosends
     pslSort dirs raw.psl temp out/fosEnds*
     pslReps  -nearTop=0.01 -minCover=0.70 -minAli=0.85 -noIntrons raw.psl \
                         fosEnds.psl /dev/null
         # Processed 84096767 alignments
 
     # cleanup
     rm -r temp
     rm raw.psl
     rm -fr out /cluster/store7/kate/hg17/fosends
 
     mkdir lifted
     liftUp lifted/fosEnds.lifted.psl \
                 /cluster/data/hg17/jkStuff/liftAll.lft warn fosEnds.psl
     pslSort dirs fosEnds.sorted.psl temp lifted
     rmdir temp
     wc -l *.sorted.psl
         # 1693693 fosEnds.sorted.psl
  
     set ncbiDir = /cluster/data/ncbi/fosends/human/fosends.3
     ~/bin/i386/pslPairs -tInsert=5000 -minId=0.94 -noBin -min=30000 -max=50000 -slop -short -long -orphan -mismatch -verbose fosEnds.sorted.psl $ncbiDir/fosEnds.pairs all_fosends fosEnds
 
     # create header required by "rdb" tools
     # TODO: replace w/ awk & sort
     echo 'chr\tstart\tend\tclone\tscore\tstrand\tall\tfeatures\tstarts\tsizes' > header
     echo '10\t10N\t10N\t10\t10N\t10\t10\t10N\t10\t10' >> header
     cat header fosEnds.pairs | row score ge 300 | sorttbl chr start | headchg -del > fosEndPairs.bed
     cat header fosEnds.slop fosEnds.short fosEnds.long fosEnds.mismatch \
          fosEnds.orphan \
     | row score ge 300 | sorttbl chr start | headchg -del > fosEndPairsBad.bed
 
     extractPslLoad -noBin fosEnds.sorted.psl fosEndPairs.bed \
                 fosEndPairsBad.bed | \
                         sorttbl tname tstart | headchg -del > fosEnds.load.psl
 
     # load into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/fosends
     hgLoadBed hg17 fosEndPairs fosEndPairs.bed \
                  -sqlTable=/cluster/home/kate/kent/src/hg/lib/fosEndPairs.sql 
         # Loaded 384558 elements
 
     # note - this track isn't pushed to RR, just used for assembly QA
     hgLoadBed hg17 fosEndPairsBad fosEndPairsBad.bed \
                  -sqlTable=/cluster/home/kate/kent/src/hg/lib/fosEndPairsBad.sql
         # Loaded  30830 elements
 
     #hgLoadPsl hg17 -nobin -table=all_fosends fosEnds.load.psl
     # NOTE: truncates file to 0 if -nobin is used
     hgLoadPsl hg17 -table=all_fosends fosEnds.load.psl
         # load of all_fosends did not go as planned: 1526991 record(s), 0 row(s) skipped, 156 warning(s) loading psl.tab
 
     # load sequences
     mkdir -p /gbdb/hg17/fosends
     ln -s /cluster/data/ncbi/fosends/human/fosends.3/fosEnds.fa \
                                 /gbdb/hg17/fosends/fosEnds.fa
     hgLoadSeq hg17 /gbdb/hg17/fosends/fosEnds.fa
         # 1087670 sequences
        # NOTE: extFile ID is 832625 (shouldn't be so large ??) 
        # may want to reset this.
 
 
 # BAC END PAIRS TRACK (DONE - 2004-06-09 kate)
     # Re-ran pslPairs with updated pairs file (2004-10-04 booch)
 
     # Use latest BAC ends data from NCBI
     # Checked  ftp.ncbi.nih.gov/genomes/BACENDS/homo_sapiens,
     #  and files were unchanged from Terry's last download
     #  (to /cluster/store1/bacends.4)
     # Link this stuff into central data area.
     ssh eieio
     cd /cluster/data/ncbi
     mkdir -p bacends/human
     ln -s /cluster/store1/bacends.4 bacends/human
     cd bacends/human/bacends.4
     faSize BACends.fa
         # 400230494 bases (2743171 N's 397487323 real) in 832614 sequences
         # 400M bases in 800K sequences
 
     # use pre-split bacends files, and associated list for cluster run
     ssh kk
     cd /cluster/data/hg17
     mkdir -p bed/bacends
     cd bed/bacends
     mkdir run
     cd run
     ls -1S /scratch/hg/gs.18/build35/maskedContigs/*.fa > contigs.lst
     ls -1S /cluster/bluearc/hg/bacEnds/hs/*.fa > bacends.lst
         # 380 contigs vs 98 bacends files -> 40K jobs
 
     # send output to kksilo, as it can better handle the NFS load
     # (these are quick jobs)
     mkdir -p /cluster/store7/kate/hg17/bacends/out
     ln -s /cluster/store7/kate/hg17/bacends/out ../out
 cat > gsub << 'EOF'
 #LOOP
 /cluster/bin/i386/blat $(path1) $(path2) -ooc=/scratch/hg/h/11.ooc {check out line+ /cluster/data/hg17/bed/bacends/out/$(root2)/$(root1).$(root2).psl}
 #ENDLOOP
 'EOF'
     gensub2 contigs.lst bacends.lst gsub jobList
     foreach f (`cat bacends.lst`)
         set d = $f:r:t
         echo $d
         mkdir -p /cluster/data/hg17/bed/bacends/out/$d
     end
 
     para create jobList
         # 37240 jobs written to batch
     para try
     para check
     para push
 # CPU time in finished jobs:    1573932s   26232.19m   437.20h   18.22d  0.050 y
 # IO & Wait Time:                122751s    2045.86m    34.10h    1.42d  0.004 y
 # Average job time:                  46s       0.76m     0.01h    0.00d
 # Longest job:                     3312s      55.20m     0.92h    0.04d
 # Submission to last job:          7148s     119.13m     1.99h    0.08d
 
     cd ../out/BACends000
     pslCheck *.psl
 #Error: invalid PSL: AZ519021:1-575 NT_004559:1306426-1608347 - NT_004559.BACends000.psl:1101
 #AZ519021 query block 3 start 283 < previous block end 575
     # NOTE: inquired with JK regarding these results
 
     # lift alignments
     ssh eieio
     cd /cluster/data/hg17/bed/bacends
     pslSort dirs raw.psl temp out/BACends*
     # takes hours ?
 
         # 37240 files in 98 dirs
         # Got 37240 files 193 files per mid file
     pslReps -nearTop=0.02 -minCover=0.60 -minAli=0.85 -noIntrons \
                 raw.psl  bacEnds.psl /dev/null
         # Processed 52291246 alignments
     mkdir lifted
     liftUp lifted/bacEnds.lifted.psl \
                 /cluster/data/hg17/jkStuff/liftAll.lft warn bacEnds.psl
     pslSort dirs bacEnds.sorted.psl temp lifted
 
     # cleanup
     rmdir temp
     rm -fr out /cluster/store7/kate/hg17/bacends
 
     wc -l *.sorted.psl
         # 2497227 bacEnds.sorted.psl
 
     set ncbiDir = /cluster/data/ncbi/bacends/human/bacends.4
     ~/bin/i386/pslPairs -tInsert=10000 -minId=0.91 -noBin -min=25000 -max=350000 -slopval=10000 -hardMax=500000 -slop -short -long -orphan -mismatch -verbose bacEnds.sorted.psl $ncbiDir/bacEndPairs.txt all_bacends bacEnds
 
     # create header required by "rdb" tools
     # TODO: replace w/ awk & sort
     echo 'chr\tstart\tend\tclone\tscore\tstrand\tall\tfeatures\tstarts\tsizes' > header
     echo '10\t10N\t10N\t10\t10N\t10\t10\t10N\t10\t10' >> header
     cat header bacEnds.pairs | row score ge 300 | sorttbl chr start | headchg -del > bacEndPairs.bed
     cat header  bacEnds.slop bacEnds.short bacEnds.long bacEnds.mismatch bacEnds.orphan \
         | row score ge 300 | sorttbl chr start | headchg -del > bacEndPairsBad.bed
 
     extractPslLoad -noBin bacEnds.sorted.psl bacEndPairs.bed \
                 bacEndPairsBad.bed | \
                         sorttbl tname tstart | headchg -del > bacEnds.load.psl
 
     # load into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/bacends
     hgLoadBed hg17 bacEndPairs bacEndPairs.bed \
                  -sqlTable=/cluster/home/kate/kent/src/hg/lib/bacEndPairs.sql 
         # Loaded 201380  
     # note - this track isn't pushed to RR, just used for assembly QA
     hgLoadBed hg17 bacEndPairsBad bacEndPairsBad.bed \
                  -sqlTable=/cluster/home/kate/kent/src/hg/lib/bacEndPairsBad.sql
         # Loaded 81773
     #hgLoadPsl hg17 -nobin -table=all_bacends bacEnds.load.psl
     # NOTE: truncates file to 0 if -nobin is used
     hgLoadPsl hg17 -table=all_bacends bacEnds.load.psl
         #load of all_bacends did not go as planned: 441072 record(s), 0 row(s) skipped, 30 warning(s) loading psl.tab
     # Reloaded table, 2004-07-21, and got more rows:
     # load of all_bacends did not go as planned: 1698790 record(s), 
     # 0 row(s) skipped, 63 warning(s) loading psl.tab
 
     # load BAC end sequences
 
     mkdir -p /gbdb/hg17/bacends
     ln -s /cluster/data/ncbi/bacends/human/bacends.4/BACends.fa \
                                 /gbdb/hg17/bacends/BACends.fa
     hgLoadSeq hg17 /gbdb/hg17/bacends/BACends.fa
         # 158588 sequences
 
     # Re-ran pslPairs with updated pairs file to take advantage of new 
     # feature of # allowing comma separated lists of end accessions for each
     #  end for a clone
     
     # First, create new pairs file (bacEndPairs.txt, bacEndSingles.txt)
     mkdir /cluster/data/ncbi/bacends/human/bacends.5
     cd /cluster/data/ncbi/bacends/human/bacends.5
     cp ../bacends.4/cl_ac_gi_len .
     /cluster/bin/scripts/convertEndPairInfo cl_ac_gi_len
 
     # Next, re-create the bed file
     mkdir /cluster/data/hg17/bed/bacends.update
     cd /cluster/data/hg17/bed/bacends.update
     ln -s /cluster/data/hg17/bed/bacends/bacEnds.sorted.psl ./bacEnds.sorted.psl
     set ncbiDir = /cluster/data/ncbi/bacends/human/bacends.5
     ~/bin/i386/pslPairs -tInsert=10000 -minId=0.91 -noBin -min=25000 -max=350000 -slopval=10000 -hardMax=500000 -slop -short -long -orphan -mismatch -verbose bacEnds.sorted.psl $ncbiDir/bacEndPairs.txt all_bacends bacEnds
 
     # create header required by "rdb" tools
     # TODO: replace w/ awk & sort
     echo 'chr\tstart\tend\tclone\tscore\tstrand\tall\tfeatures\tstarts\tsizes' > header
     echo '10\t10N\t10N\t10\t10N\t10\t10\t10N\t10\t10' >> header
     cat header bacEnds.pairs | row score ge 300 | sorttbl chr start | headchg -del > bacEndPairs.bed
     cat header  bacEnds.slop bacEnds.short bacEnds.long bacEnds.mismatch bacEnds.orphan \
         | row score ge 300 | sorttbl chr start | headchg -del > bacEndPairsBad.bed
 
     # wc *.bed
     # 204884 2253724 20612402 bacEndPairs.bed
     # 79401  873411 6527559 bacEndPairsBad.bed
 
     # previous
     # wc ../bacends/*.bed
     # 201380 2215180 20280578 ../bacends/bacEndPairs.bed
     # 81773  899503 6712402 ../bacends/bacEndPairsBad.bed
 
     extractPslLoad -noBin bacEnds.sorted.psl bacEndPairs.bed \
                 bacEndPairsBad.bed | \
                         sorttbl tname tstart | headchg -del > bacEnds.load.psl
 
     # load into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/bacends.update
     hgLoadBed hg17 bacEndPairs bacEndPairs.bed \
                  -sqlTable=/cluster/home/kate/kent/src/hg/lib/bacEndPairs.sql 
         # Loaded 204884  
     # note - this track isn't pushed to RR, just used for assembly QA
     hgLoadBed hg17 bacEndPairsBad bacEndPairsBad.bed \
                  -sqlTable=/cluster/home/kate/kent/src/hg/lib/bacEndPairsBad.sql
         # Loaded 79401
     #hgLoadPsl hg17 -nobin -table=all_bacends bacEnds.load.psl
     # NOTE: truncates file to 0 if -nobin is used
     hgLoadPsl hg17 -table=all_bacends bacEnds.load.psl
     # load of all_bacends did not go as planned: 1729146 record(s), 0 row(s) skipped, 70 warning(s) loading psl.tab
 
 
 # PLACE ASSEMBLY CLONES - misc instructions, only somewhat relevant
 #	See PLACE ASSEMBLY CLONES ON CONTIGS AND SEQUENCE below
 
 ######	 A second attempt at clone alignment###
     #	Split the clones into 3K pieces into about 1000 fa files
 
     #	Example:
 zcat Z99916.1.fa.gz Z99774.1.fa.gz Z99756.7.fa.gz | faSplit size stdin 3000 /tmp/name.fa -lift=/tmp/name.lft -oneFile
 
     #	Trying this idea in unPlacedBatch
     ssh kk0
     mkdir /cluster/data/hg17/bed/contig_overlaps/unPlacedBatch
     cd /cluster/data/hg17/bed/contig_overlaps/unPlacedBatch
     ls -1S /scratch/hg/gs.18/build35/bothMaskedNibs > nibList
     ls -1S /cluster/data/hg17/bed/contig_overlaps/blatClones > cloneList
 cat << '_EOF_' > gsub
 #LOOP
 /cluster/bin/i386/blat -fastMap -ooc=/scratch/hg/h/11.ooc -q=dna -t=dna {check in exists /scratch/hg/gs.18/build35/bothMaskedNibs/$(path1)} {check in exists+ /cluster/data/hg17/bed/contig_overlaps/blatClones/$(path2)} {check out line+ psl/$(root1)/$(root2).psl}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     mkdir psl
     cat nibList | sed -e "s/.nib//" | while read D
 do
 mkdir psl/$D
 done
 
     gensub2 nibList cloneList gsub jobList
     para create jobList
 
 # PLACE ASSEMBLY CLONES ON CONTIGS AND SEQUENCE (DONE - 2004-07-12 - Hiram)
     ssh eieio
     mkdir /cluster/data/hg17/bed/contig_overlaps
     cd /cluster/data/hg17/bed/contig_overlaps
     #	find all the clones that were used in the assembly
     sed -e "/^#.*/d" /cluster/data/hg17/ncbi_build35.agp | \
         awk '{if (!match($5,"N")) {print $6}}' | \
         sort -u > placed_in_assembly.list
     wc -l placed_in_assembly.list
     #	26872 placed_in_assembly.list
     #	These may be available from the phases files at:
     #	ftp://ftp.ncbi.nih.gov/genbank/genomes/H_sapiens
     #	Which are easily fetched with wget.  However I took a look
     #	at those and could not find all the clones in them.  There may
     #	be a versioning problem because these phases files are often
     #	updated.
     #	Fetch them from Genbank with the following three PERL scripts:
     #	[hiram@hgwdev /cluster/data/hg17/bed/contig_overlaps] ls -og *.pl
     #	-rwxrwxr-x    1     3047 May 24 18:43 bioPerlFetch.pl
     #	-rwxrwxr-x    1     2370 Jun  4 15:21 fetchGenbank.pl
     #	-rwxrwxr-x    1      700 May 24 21:47 foldEm.pl
 
     #	Which takes about 4 days ...
     #	Example, 
     cat << '_EOF_' > terrys.list
 AC011841.7
 AC018692.9
 AC018743.27
 AC037482.14
 AL163540.11
 '_EOF_'
     # << this line makes emacs coloring happy
     #	only works on hgwdev
     ssh hgwdev
     cd /cluster/data/hg17/bed/contig_overlaps
     mkdir fasta
     time ./fetchGenbank.pl terrys.list > fetchResult.out 2>&1
 
     #	There is a bit of behind the scenes hocus pocus going on here.
     #	This is a tedious task of comparing various lists with each
     #	other and making sure everything matches.  Manual fixups are
     #	done for the newly named 6_hla_hap* items, copies of the PAR
     #	business were duplicated so that X and Y both have the same set
     #	of clones for that.  The end result should be a directory hierarchy
     #	here with a directory for each chrom, each random, the 6_hla_hap?
     #	items and each directory contains the clones that belong to that
     #	chromosome.  The leftovers are the unplaced clones which end up
     #	in the directory called: unPlaced.  The instructions here are
     #	merely a guideline of possibilities.  Care should be taken to
     #	make sure all listings are correct and everything gets in the
     #	right place.
     ssh eieio
     #	And then make a list of all clones considered for assembly:
     sed -e "/^#.*/d" /cluster/store5/gs.18/ncbi/sequence.inf | \
 	grep for_assembly | awk '{print $1}' | sort -u > sequence.list
     wc -l sequence.list
     #	46733 sequence.list
     #	Verify overlaps are correct:
     comm -12 placed_in_assembly.list sequence.list > inBoth
     comm -23 placed_in_assembly.list sequence.list > inAssemblyNotSequence
     comm -13 placed_in_assembly.list sequence.list > inSequenceNotAssembly
     wc in*
     #	    1       1      12 inAssemblyNotSequence
     #	26871   26871  301709 inBoth
     #	19862   19862  219050 inSequenceNotAssembly
     #	46734   46734  520771 total
     #	This stray one is from Terry's five additions in the final fixup
     #	phase with Greg:
     cat inAssemblyNotSequence
     #	AC018743.27
     #	Terry added: AC011841.7 AC018692.9 AC018743.27 AC037482.14 AL163540.11
     #
     #	Generate a listing that relates clones to their contigs
     sed -e "/^#.*/d" /cluster/store5/gs.18/build35/ncbi_build35.agp | \
 	./contigAcc.pl > disburseEm.list
     #
     #	Using that list, sort the downloaded clones into their
     #	respective chrom directories:
     ./disburse.sh
 
     #	Check the number of sequences obtained:
     find ./? ./?? ./*_random ./6_hla* -type f | wc -l
     #	26872
     #	So, why is this number one more than the inBoth list ?
     #	Because, the official NCBI sequence.inf file is missing one of
     #	the clones that Terry added: AC018743.27
     #	And it shows up in our check list above as inAssemblyNotSequence
     #	It isn't exactly missing, it just isn't marked "for_assembly"
 
     #	OK, with everything in place, we are ready to try and find
     #	all these items in the assembly.  To run a Kluster job on one of
     #	the chroms, matching the items that are supposed to be included
     #	in that chrom.  We need to get things set up on the Iservers,
     #	psLayout is heavy into disk I/O and it brings everything down if
     #	allowed to work on any NFS filesystems for input.
 
     #	It appears that psLayout wants an ooc file of tile size 10
     #	I tried making one for the whole assembly but it seemed to
     #	include too much for some contigs and it caused a lot of
     #	alignments to be missed.  Thus, create an ooc file for each
     #	contig
 
     ssh eieio
     mkdir /cluster/bluearc/scratch/hg/gs.18/build35/contigOoc10
     cd /cluster/bluearc/scratch/hg/gs.18/build35/contigOoc10
     ls ../maskedContigs | sed -e "s/.fa//" | while read CONTIG
     do
 	blat -repMatch=256 -makeOoc=${CONTIG}.10.ooc -tileSize=10 \
 	    ../maskedContigs/${CONTIG}.fa \
 	    ../maskedContigs/${CONTIG}.fa /dev/null
 	echo "done: ${CONTIG}"
     done
 
     #	Copy that result to the Iservers:
     ssh kkr1u00
     mkdir /iscratch/i/gs.18/build35/contigOoc10
     cd /iscratch/i/gs.18/build35/contigOoc10
     rsync -arlv /cluster/bluearc/scratch/hg/gs.18/build35/contigOoc10/ .
     #	And, copy the clone sequences:
     mkdir /iscratch/i/gs.18/build35/clones
     cd /cluster/store5/gs.18/build35/bed/contig_overlaps
     for D in ? ?? *_random 6_hla_hap?
     do
 	rsync -arlv `pwd`/${D} /iscratch/i/gs.18/build35/clones
     done
     
     /cluster/bin/iSync
 
     ssh kk
     cd /cluster/data/hg17/bed/contig_overlaps
     mkdir psl
     cat << '_EOF_' > runPsLayout.sh
 #!/bin/sh
 #       kkiPsLayout.sh <chrom> <clone> <contig>
 #       where <chrom> is the chrom this contig is on
 #       <clone> is one of the .fa.gz files in
 #               /cluster/data/hg17/bed/contig_overlaps/*/<clone>.fa.gz
 #               without the .fa.gz extension
 #               This stuff has been mirrored to:
 #               /iscratch/i/gs.18/clones/*/<clone>.fa.gz
 #       <contig> is one of the contigs found in:
 #               /cluster/store5/gs.18/build35/<chrom>/<contig>/<contig>.fa
 #
 CHROM=$1
 CLONE=$2
 CONTIG=$3
 TARGET=/iscratch/i/gs.18/build35/maskedContigs/${CONTIG}.fa
 FAZ=/iscratch/i/gs.18/build35/clones/${CHROM}/${CLONE}.fa.gz
 OOC=/iscratch/i/gs.18/build35/contigOoc10/${CONTIG}.10.ooc
 mkdir -p psl/${CONTIG}
 if [ ! -s ${FAZ} ]; then
         echo "Can not find: ${FAZ}"
         exit 255
 fi
 if [ ! -s ${TARGET} ]; then
         echo "Can not find: ${TARGET}"
         exit 255
 fi
 if [ ! -s ${OOC} ]; then
         echo "Can not find: ${OOC}"
         exit 255
 fi
 zcat ${FAZ} > /tmp/${CLONE}.fa
 $HOME/bin/i386/psLayout ${TARGET} \
         /tmp/${CLONE}.fa genomic ${OOC} psl/${CONTIG}/${CLONE}.psl
 RET=$?
 rm -f /tmp/${CLONE}.fa
 exit ${RET}
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod +x runPsLayout.sh
 
     #	make up a listing of chrom, clone, contig from:
     grep -v "^#" disburseEm.list | sed -e "s/.fa.gz//" > chr.clone.contig.list
     wc -l chr.clone.contig.list
     #	26872 chr.clone.contig.list
     awk '{
 printf "./runPsLayout.sh %s %s %s {check out line+ psl/%s/%s.psl}\n",
         $1, $2, $3, $3, $2
 }' chr.clone.contig.list > jobList
     # << this line makes emacs coloring happy
     #	To do a quick test, run just chr22:
     grep -v "^22" chr.clone.contig.list | awk '{
 printf "./runPsLayout.sh %s %s %s {check out line+ psl/%s/%s.psl}\n",
         $1, $2, $3, $3, $2
 }' > jobList
     para create jobList
     para try ... check ... etc ...
     #	One run on chr22 took:
 # Completed: 561 of 561 jobs
 # CPU time in finished jobs:     927068s   15451.14m   257.52h   10.73d  0.029 y
 # IO & Wait Time:                  6295s     104.91m     1.75h    0.07d  0.000 y
 # Average job time:                1664s      27.73m     0.46h    0.02d
 # Longest job:                    69745s    1162.42m    19.37h    0.81d
 # Submission to last job:         69780s    1163.00m    19.38h    0.81d
 
 
     #	put the results together, filter, lift and load:
     cd /cluster/data/hg17/bed/contig_overlaps/psl
     pslSort dirs raw.psl tmp N*
     pslReps -singleHit raw.psl repsSingle.psl /dev/null
     liftUp chr22.psl /cluster/data/hg17/jkStuff/liftAll.lft \
 	warn repsSingle.psl
     hgLoadPsl -table=cloneTest hg17 chr22.psl
 
     #	There are a number of clones listed in the sequence.inf file
     #	as status W with names beginning AACC AADB AADC AADD
     #	These are the Whole shotgun assemblies for the Celera genome.
     #	A few of them were used in the assembly of the NCBI genome, namely:
 ./11/AADB01066164.1.fa.gz
 ./11/AADC01095577.1.fa.gz
 ./11/AADD01116830.1.fa.gz
 ./11/AADD01118406.1.fa.gz
 ./11/AADD01116787.1.fa.gz
 ./11/AADD01112371.1.fa.gz
 ./11/AADD01116788.1.fa.gz
 ./11/AADD01115518.1.fa.gz
 ./11/AADD01118410.1.fa.gz
 ./11/AADD01117999.1.fa.gz
 ./21/AADD01172789.1.fa.gz
 ./21/AADD01172788.1.fa.gz
 ./21/AADD01209098.1.fa.gz
 ./21/AADD01172902.1.fa.gz
     #	And these have been distributed properly in their corresponding
     #	chromosome.  The rest of them, 26, all with names starting AACC are in
     #	the directory here: celeraOnly
 
     #	To run the unPlaced alignments.
     #	Prepare scratch and iscratch
     ssh eieio
     mkdir /cluster/bluearc/scratch/hg/gs.18/build35/clones/unPlaced
     rsync -arlv /cluster/data/hg17/bed/contig_overlaps/unPlaced/ \
 	/cluster/bluearc/scratch/hg/gs.18/build35/clones/unPlaced
     #	request scratch sync to cluster admins
 
     ssh kkr1u00
     mkdir /iscratch/i/gs.18/build35/clones/unPlaced
     rsync -arlv /cluster/data/hg17/bed/contig_overlaps/unPlaced/ \
 	/iscratch/i/gs.18/build35/clones/unPlaced
     /cluster/bin/iSync
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/contig_overlaps/unPlacedBatch
     #	There are too many to try them all, obtain guildelines from hg16
     #	of clone to contig mapping:
     hgsql -N -e "select name,chrom from clonePos;" hg16 > hg16.clone.chrom
     hgsql -N -e "select contig,chrom from ctgPos;" hg16 > hg16.contig.chrom
 
     ssh kk
     mkdir /cluster/data/hg17/bed/contig_overlaps/unPlacedBatch
     cd /cluster/data/hg17/bed/contig_overlaps/unPlacedBatch
     ls ../unPlaced | sed -e "s/.fa.gz//" > unPlaced.clone.list
     wc -l unPlaced.clone.list
     #	19836 unPlaced.clone.list
     ls -1S /scratch/hg/gs.18/build35/maskedContigs > contig.list
     wc -l contig.list
     #	380 contig.list
 
     cat << '_EOF_' > runPsLayout.sh
 #!/bin/sh
 #       kkiPsLayout.sh <clone> <contig>
 #       <clone> is one of the .fa.gz files in
 #               /scratch/hg/gs.18/build35/clones/unPlaced
 #               without the .fa.gz extension
 #       <contig> is one of the contigs found in:
 #               /iscratch/i/gs.18/build35/maskedContigs
 #
 CLONE=$1
 CONTIG=$2
 TARGET=/iscratch/i/gs.18/build35/maskedContigs/${CONTIG}.fa
 FAZ=/scratch/hg/gs.18/build35/clones/unPlaced/${CLONE}.fa.gz
 OOC=/iscratch/i/gs.18/build35/contigOoc10/${CONTIG}.10.ooc
 mkdir -p psl/${CONTIG}
 if [ ! -s ${FAZ} ]; then
         echo "Can not find: ${FAZ}"
         exit 255
 fi
 if [ ! -s ${TARGET} ]; then
         echo "Can not find: ${TARGET}"
         exit 255
 fi
 if [ ! -s ${OOC} ]; then
         echo "Can not find: ${OOC}"
         exit 255
 fi
 zcat ${FAZ} > /tmp/${CLONE}.fa
 $HOME/bin/i386/psLayout ${TARGET} \
         /tmp/${CLONE}.fa genomic ${OOC} psl/${CONTIG}/${CLONE}.psl
 RET=$?
 rm -f /tmp/${CLONE}.fa
 exit ${RET}
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod +x runPsLayout.sh
 
     cat << '_EOF_' > gsub
 #LOOP
 ./runPsLayout.sh $(path1) $(path2) {check out line+ psl/$(path2)/$(path1).psl}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     gensub2 unPlaced.clone.list contig.list gsub jobList
 
    # XXXX - some time later ... 2004-07-12
     # Bringing this sequence to a close.  Difficulties encountered:
     #	Placed clones that did not survive the psLayout filter:
     #	AC006040.3 AC006328.5 AC007039.6 AC007241.3 AC007965.3
     #	AC009947.2 AC010682.2 AC012005.4 AC016707.2 AC016728.4
     #	AC016752.2 AC017005.7 AC025226.4 AC025246.6 AC055713.29
     #	AC068541.7 AC068601.8 AC068704.4 AC073649.3 AC073962.5
     #	AC091175.11 AC095381.1 AC104597.3 AC130223.2 AC130814.3
     #	AC133883.6 AC139103.3 AF003627.3 AF135405.3 AL021878.2
     #	AL137064.6 AL356803.2 AL390801.4 AL591480.8 AL901608.1
     #	AP005814.2 BX322790.2 Z84489.1 Z84814.1
     #	And placed clones that broken into two pieces during their
     #	psLayout alignment:
     #	AC006982.3 AC007742.4 AC023342.3 AC024183.4 AC025735.4
     #	AC095380.1 AL646104.4 BX293536.4
     #	For the above clones, their assignments in ref_placed.agp were
     #	used instead of trying to adjust the psLayout process.
 
     #	The PAR clones are a problem.  They were placed properly, but
     #	during their load with hgClonePos there was a warning issued
     #	about their dual existance.  hgClonePos said they were only
     #	going to be placed on chrX and not on chrY.  However in the
     #	browser when chrY is viewed it issues errors about these not
     #	having proper coordinates in the clonePos table.  These were
     #	removed from the coverage track to eliminate that error.
     #	AL954722.18 BX537334.4 BX000483.7 BX908402.3 BX649635.3 BX119919.5
     #	AC079176.15 AC097314.27 AC006209.25 AJ271735.1 AJ271736.1
     #
     #	And finally, after many different types of alignment attempts,
     #	there remain 1489 un-placed clones that could not be located.
 
     #	While trying to figure out which contigs many clones belonged
     #	to, the following cluster run script was used to take a survey
     #	using blat:
 #!/bin/sh
 #       runBlat.sh <clone> <contig>
 #       <clone> is one of the .fa.gz files in
 #               /scratch/hg/gs.18/build35/clones/
 #               without the .fa.gz extension
 #       <contig> is one of the contigs found in:
 #               /iscratch/i/gs.18/build35/maskedContigs
 #   
 # ./runBlat.sh unPlaced/AB000876.1.fa.gz NT_005612.fa {check out line+
 # psl/NT_005612.fa/unPlaced/AB000876.1.fa.gz.psl}
 #
 HERE=`pwd`
 CLONE=$1 
 CLONEDIR=`dirname ${CLONE}`
 CLONENAME=`basename ${CLONE}`
 CLONESRC=/iscratch/i/gs.18/build35/clones/${CLONE}.fa.gz
 CONTIG=$2
 CONTIGBASE=${CONTIG/.fa/}
 TARGET=/iscratch/i/gs.18/build35/maskedContigs/${CONTIG}
 if [ ! -s ${CLONESRC} ]; then
         echo "Can not find: ${CLONESRC}" 1>/dev/stderr
         exit 255
 fi
 if [ ! -s ${TARGET} ]; then
         echo "Can not find: ${TARGET}" 1>/dev/stderr
         exit 255
 fi
 mkdir -p /tmp/${CLONEDIR}/${CLONENAME}
 zcat ${CLONESRC} > /tmp/${CLONEDIR}/${CLONENAME}/${CLONENAME}.fa
 cd /tmp/${CLONEDIR}
 /cluster/data/hg17/bed/contig_overlaps/FfaSplit/faToFfa ${CLONENAME}
 ECOUNT=`cat error.convert | wc -l`
 if [ "${ECOUNT}" -ne 0 ]; then
         echo "Error during faToFfa, error.convert not empty" 1>/dev/stderr
         exit 255
 fi
 rm -f error.convert
 B=${CLONENAME/\.*/}
 cd /tmp/${CLONEDIR}/${CLONENAME}
 faSplit byname ${CLONENAME}.fa .
 RET=0
 export RET
 for F in ${CLONENAME}_*.fa
 do
     FA=${F/_*.fa/}
     A=${FA/.[0-9]*/}
     P=${F/.fa/}
     N=${P##*_}
     rm -f t.fa
     mv ${F} t.fa
     cat t.fa | faSplit -oneFile size stdin 3000 ${A}_${N}
     rm -f t.fa
     blat ${TARGET} ${A}_${N}.fa -ooc=/scratch/hg/h/11.ooc ${A}_${N}.psl \
         -t=dna -q=dna -fastMap -noHead
     RET=$?
     if [ "$RET" -ne 0 ]; then
         echo "Error during blat ${TARGET} ${A}_${N}.fa" 1>/dev/stderr
         break
     fi
 done
 rm -f ${CLONENAME}.fa
 rm -f ${B}_*.fa
 cd ${HERE}
 mkdir -p psl/${CONTIGBASE}
 sed -e "s/${A}/${CLONENAME}/" /tmp/${CLONEDIR}/${CLONENAME}/*.psl > \
         psl/${CONTIGBASE}/${CLONENAME}.psl
 rm -f /tmp/${CLONEDIR}/${CLONENAME}/*.psl
 rmdir --ignore-fail-on-non-empty /tmp/${CLONEDIR}/${CLONENAME}
 rmdir --ignore-fail-on-non-empty /tmp/${CLONEDIR}
 exit ${RET}
 
     #	The alignment with psLayout were done with the following cluster
     #	run script:
 
 #!/bin/sh
 #       kkiPsLayout.sh <clone> <contig>
 #       <clone> is one of the .fa.gz files in
 #               /scratch/hg/gs.18/build35/clones/unPlaced
 #               without the .fa.gz extension
 #       <contig> is one of the contigs found in:
 #               /iscratch/i/gs.18/build35/maskedContigs
 #
 # ./runPsLayout.sh unPlaced/AP001966.2 NT_016354 {check out exists
 # psl/NT_016354/AP001966.2.psl}
 #
 HERE=`pwd`
 CLONE=$1
 CONTIG=$2
 CLONEDIR=`dirname ${CLONE}`
 CLONENAME=`basename ${CLONE}`
 RESULT=psl/${CONTIG}/${CLONENAME}.psl
 CLONESRC=/iscratch/i/gs.18/build35/clones/${CLONE}.fa.gz
 TARGET=/iscratch/i/gs.18/build35/maskedContigs/${CONTIG}.fa
 OOC=/iscratch/i/gs.18/build35/contigOoc10/${CONTIG}.10.ooc
 if [ ! -s ${CLONESRC} ]; then
         echo "Can not find: ${CLONESRC}" 1>/dev/stderr
         exit 255
 fi
 if [ ! -s ${TARGET} ]; then
         echo "Can not find: ${TARGET}" 1>/dev/stderr
         exit 255
 fi
 if [ ! -s ${OOC} ]; then
         echo "Can not find: ${OOC}" 1>/dev/stderr
         exit 255
 fi
 mkdir -p /tmp/${CONTIG}/${CLONEDIR}/${CLONENAME}
 zcat ${CLONESRC} > /tmp/${CONTIG}/${CLONEDIR}/${CLONENAME}.fa
 cd /tmp/${CONTIG}
 /cluster/data/hg17/bed/contig_overlaps/FfaSplit/faToFfa ${CLONEDIR}
 cd ${HERE}
 mkdir -p psl/${CONTIG}
 $HOME/bin/i386/psLayout ${TARGET} /tmp/${CONTIG}/${CLONEDIR}/${CLONENAME}.fa genomic ${OOC} ${RESULT}
 RET=$?
 rm -f /tmp/${CONTIG}/${CLONEDIR}/${CLONENAME}.fa /tmp/${CONTIG}/error.convert
 rmdir --ignore-fail-on-non-empty /tmp/${CONTIG}/${CLONEDIR}/${CLONENAME}
 rmdir --ignore-fail-on-non-empty /tmp/${CONTIG}/${CLONEDIR}/
 rmdir --ignore-fail-on-non-empty /tmp/${CONTIG}
 exit ${RET}
 
 # BUILD KNOWN GENES TABLES (DONE 6/8/04 Fan)
 
   Build sp040515 and proteins040515 DBs first.
   
   hgsql hg17 -e "create database kgHg17"
   
   cd /cluster/store6/kgDB/bed
   mkdir kgHg17
   cd /cluster/store6/kgDB/bed/kgHg17
 
   ~/src/hg/protein/KGprocess.sh kgHg17 hg17 040515
   
   The script was run successfully with the last message:
 
   	Tue Jun  8 15:36:52 PDT 2004 DONE 
 
   After initial inspection of tables in kgHg17, do the following
   from mySql prompt:
 
   alter table kgHg17.cgapAlias rename as hg17.cgapAlias;
   alter table kgHg17.cgapBiocDesc rename as hg17.cgapBiocDesc;
   alter table kgHg17.cgapBiocPathway rename as hg17.cgapBiocPathway;
   alter table kgHg17.dupSpMrna rename as hg17.dupSpMrna;
   alter table kgHg17.keggMapDesc rename as hg17.keggMapDesc;
   alter table kgHg17.keggPathway rename as hg17.keggPathway;
   alter table kgHg17.kgAlias rename as hg17.kgAlias;
   alter table kgHg17.kgProtAlias rename as hg17.kgProtAlias;
   alter table kgHg17.kgXref rename as hg17.kgXref;
   alter table kgHg17.knownGene rename as hg17.knownGene;
   alter table kgHg17.knownGeneLink rename as hg17.knownGeneLink;
   alter table kgHg17.knownGeneMrna rename as hg17.knownGeneMrna;
   alter table kgHg17.knownGenePep rename as hg17.knownGenePep;
   alter table kgHg17.mrnaRefseq rename as hg17.mrnaRefseq;
   alter table kgHg17.spMrna rename as hg17.spMrna;
 
   hg17.knownGene has 43,401 entries and hg16.knownGene has 43,232 entries.
   and running featireBits shows:
   
    	featureBits hg17 knownGene
    	63983072 bases of 2866216770 (2.232%) in intersection
    
    	featureBits hg16 knownGene
    	63781799 bases of 2865248791 (2.226%) in intersection
   
   Connect to genome-testdb and use hgcentraltest DB.
   Add a new entry in gdbPdb table:
  
         insert into gdbPdb values('hg17', 'proteins040515');
 
 
 # CREATE LINEAGE-SPECIFIC REPEATS FOR BLASTZ WITH ZEBRAFISH
 # (DONE, 2004-06-08, hartera)
     # Treat all repeats as lineage-specific
     mkdir /iscratch/i/gs.18/build35/linSpecRep.notInZebrafish
     foreach f (/iscratch/i/gs.18/build35/rmsk/chr*.fa.out)
  cp -p $f /iscratch/i/gs.18/build35/linSpecRep.notInZebrafish/$f:t:r:r.out.spec
     end
     iSync
   
 
 # PREP FOR LIFTOVER CHAINS TO THIS ASSEMBLY (2004-06-10 kate)
 
     # split into 3K chunks
     ssh eieio
     set liftDir = /cluster/data/hg17/bed/liftOver/liftSplit
     mkdir -p $liftDir
     cd $liftDir
 cat > split.csh << 'EOF'
     set splitDir = /iscratch/i/hg17/liftOver/split
     mkdir -p $splitDir
     set liftDir = /cluster/data/hg17/bed/liftOver/liftSplit
     foreach n (`ls /cluster/data/hg17/nib`)
         set c = $n:r
         echo $c
         faSplit -lift=$liftDir/$c.lft size \
             /cluster/data/hg17/$d/$c.fa -oneFile 3000 $splitDir/$c
     end
 'EOF'
 # << for emacs
     csh split.csh >&! split.log &
     tail -100f split.log
 
     ssh kkr1u00
     iSync
 
 
 # STS MARKERS (DONE 2004-07-21 kate)
 # MANUAL UPDATE OF D21S168 and D21S167 (DONE, 2005-02-11, hartera)
 # FILTERED OUT noOoc ALIGNMENTS WITH tBaseInsert >=1000 
 # (DONE, 2005-02-17, hartera) AND RELOADED stsMap, stsInfo2 and all_sts_seq
 # DATABASE TABLES AFTER ADDING FILTERED ALIGNMENTS TO all_sts_seq AND 
 # REMOVING DATA FROM stsMap and stsInfo2 FOR THE MARKERS REMOVED FROM THE 
 # FILTERED SET (DONE, 2005-02-18, hartera)
 # UPDATE PSL ALIGNMENTS FOR D21S167 and D21S168 AND RELOAD INTO all_sts_seq
 # (DONE, 2005-02-23, hartera)
 # UPDATED stsAlias TABLE REMOVING OF FILTERED ALIGNMENTS (2005-02-24, hartera)
 
     # Terry's sts.9 dir is in /cluster/store5/sts.2004-07.old
     # remove this after verifying the newer version
 
    # update from NCBI (booch)
     ssh eieio
     # use store5 for space
     mkdir -p /cluster/store5/sts.2004-07
     ln -s /cluster/store5/sts.2004-07 /cluster/data/ncbi
     ln -s /cluster/data/ncbi/sts.2004-07 sts.9
     cd /cluster/data/ncbi/sts.2004-07
     wget ftp://ftp.ncbi.nih.gov/repository/dbSTS/dbSTS.sts
     wget ftp://ftp.ncbi.nih.gov/repository/dbSTS/dbSTS.aliases
     wget ftp://ftp.ncbi.nih.gov/blast/db/FASTA/sts.gz
     gunzip sts.gz
     mv sts dbSTS.fa
 
     # incremental update from previous build
     # NOTE: could mysql dump this, unless hand-updated (like hg16)
     # First - copy from Terry's dir
     ssh eieio
     ln -s /cluster/store1/sts.8 /cluster/data/ncbi
     cd /cluster/data/ncbi/sts.9
 
     # this time, snag from Terry's dir
     cd /cluster/data/ncbi/sts.9
     cp -p ~booch/tracks/update/all.STS.fa.prev .
     cp -p ~booch/tracks/update/stsInfo2.bed stsInfo2.bed.prev
 
     # Convert dbSTS.fa file to easier reading format, and get accessions
     /cluster/bin/scripts/convertGbFaFile dbSTS.fa > dbSTS.convert.fa
     grep ">" dbSTS.convert.fa | cut -f 2 -d ">" > dbSTS.acc
 
     # NOTE: updateStsInfo creates new stsInfo2.bed, all.primers, 
     #   all.STS.fa, stsAlias.bed files 
     updateStsInfo -verbose=1 -gb=dbSTS.acc stsInfo2.bed.prev all.STS.fa.prev \
                     dbSTS.sts dbSTS.aliases dbSTS.convert.fa new
     # 129991  SWXD2599        99622   (0) not in dbSTS anymore
     # 166473  D3S3812 154523  (0) not in dbSTS anymore
     # 185776  RH83562 209614  (0) not in dbSTS anymore
 
     mv new.info stsInfo2.bed
     mv new.primers all.primers
     mv new.alias stsAlias.bed
     mv new.fa all.STS.fa
 
     # get list of all STS id's in the fasta file
     sed -n 's/^>\([0-9][0-9]*\) .*/\1/p' all.STS.fa | sort -n >  all.STS.id
     wc -l all.STS.id
         # 92674 total sequences
     /cluster/bin/scripts/convertPrimerToFA all.primers > all.primers.fa
 
     # Copy stsInfo2.bed and stsAlias.bed to data directory becuase
     # these will be loaded into the database later
     mkdir -p /cluster/data/hg17/bed/sts
     cp stsInfo2.bed /cluster/data/hg17/bed/sts/
     cp stsAlias.bed /cluster/data/hg17/bed/sts/
 
     # Create sts sequence alignments
     mkdir -p /cluster/bluearc/sts.9/sts.split
     faSplit sequence all.STS.fa 50 /cluster/bluearc/sts.9/sts.split/sts
     cp /cluster/data/ncbi/sts.9/all.STS.fa /cluster/bluearc/sts.9
 
     # create small ooc file to use with alignments (if not existing)
     # NOTE: these were just used for experimenting; weren't used in
     # final runs
     ssh kolossus
     cd /cluster/data/hg17/bed/sts
     ls /cluster/bluearc/hg17/bothMaskedNibs/chr*.nib > nib.lst
     blat nib.lst /dev/null /dev/null \
         -tileSize=11 -makeOoc=/cluster/bluearc/hg/h/11.4096.ooc -repMatch=4096
     blat nib.lst /dev/null /dev/null \
         -tileSize=11 -makeOoc=/cluster/bluearc/hg/h/11.16384.ooc -repMatch=16384
 
     ssh kk
     cd /cluster/data/hg17/bed/sts
     mkdir run
     cd run
     ls -1S /scratch/hg/hg17/maskedContigs/*.fa > contigs.lst
     ls -1S /cluster/bluearc/sts.9/sts.split/sts*.fa > sts.lst
     mkdir -p /cluster/bluearc/hg17/sts/sts/out
     foreach f (`cat sts.lst`)
         set d = $f:t:r
         mkdir /cluster/bluearc/hg17/sts/sts/out/$d
     end
 
     # create alignments
 cat > template << 'EOF'
 #LOOP
 /cluster/bin/i386/blat $(path1) $(path2) -ooc=/cluster/bluearc/hg/h/11.ooc -stepSize=5 {check out line+ /cluster/bluearc/hg17/sts/sts/out/$(root2)/$(root1).$(root2).psl}
 #ENDLOOP
 'EOF'
 # << for emacs
 
     gensub2 contigs.lst sts.lst template jobList
     para create jobList
         # 17860 jobs
     para try
     para check
     para push
 # CPU time in finished jobs:     216985s    3616.41m    60.27h    2.51d  0.007 y
 # IO & Wait Time:                 48790s     813.17m    13.55h    0.56d  0.002 y
 # Average job time:                  15s       0.25m     0.00h    0.00d
 # Longest job:                      267s       4.45m     0.07h    0.00d
 # Submission to last job:          2228s      37.13m     0.62h    0.03d
 
     # Compile sts sequence results
     ssh kolossus
     cd /cluster/bluearc/hg17/sts/sts
     pslSort dirs raw.psl temp out/*
     rm -rf temp
     pslReps -nearTop=0.0001 -minCover=0.6 -minAli=0.8 -noIntrons raw.psl \
 	stsMarkers.psl /dev/null
             # Processed 7121016 alignments
     #cp stsMarkers.psl /cluster/data/hg17/bed/sts/run
 
     # Lift them and get them ready to combine with primer alignments
     #cd  /cluster/data/hg17/bed/sts/run
     #liftUp -nohead /cluster/data/hg17/bed/sts/run/stsMarkers.lifted.psl \
     liftUp -nohead stsMarkers.lifted.psl \
         /cluster/data/hg17/jkStuff/liftAll.lft warn stsMarkers.psl
 
     # missing some utilities for kolossus, so switch to fileserver
     # NOTE: probably no longer true -- try on kolossus next time
     ssh kksilo
     cd /cluster/bluearc/hg17/sts/sts
     /cluster/bin/scripts/extractPslInfo stsMarkers.lifted.psl
         # creates <file>.initial
     /cluster/bin/scripts/findAccession -agp stsMarkers.lifted.psl.initial \
 	/cluster/data/hg17
             # "Could not open /cluster/data/hg17/Y/chrY_random.agp" etc.
             # Looks like it trys all _randoms (even one's that don't
             # exist/aren't needed
             # creates <file>.acc
     #rm stsMarkers.lifted.psl.initial
     sort -k 4n stsMarkers.lifted.psl.initial.acc > stsMarkers.final
     #rm stsMarkers.lifted.psl.initial.acc
     #cp stsMarkers.final stsMarkers.lifted.psl.initial /cluster/data/hg17/bed/sts
     # determine found markers (4th field in file)
     cut -f 4 stsMarkers.final | sort -n -u > stsMarkers.found
     wc -l stsMarkers.found
         #   89532 stsMarkers.found
         # out of 92674 total sequences
 
     # extract sequences for markers not yet found, and
     # blat w/o ooc to try to place more
     comm -1 -3  stsMarkers.found /cluster/data/ncbi/sts.9/all.STS.id \
                 > stsMarkers.notFound
     wc -l stsMarkers.notFound
         # 3142 stsMarkers.notFound
     faSomeRecords /cluster/data/ncbi/sts.9/all.STS.fa stsMarkers.notFound \
                 notFound.STS.fa
     mkdir /cluster/bluearc/sts.9/sts.splitNotFound
     faSplit sequence notFound.STS.fa 20 \
                 /cluster/bluearc/sts.9/sts.splitNotFound/sts
 
     # blat with 11.ooc misses alignments, so reblat w/o the
     # sequences that aren't found
     # NOTE: filtering produces yield of only 149 markers placed (out of 3142).
     # not enough to justify this step next time
     ssh kk
     cd /cluster/data/hg17/bed/sts
     mkdir run.noOoc
     cd run.noOoc
     ls -1S /scratch/hg/hg17/maskedContigs/*.fa > contigs.lst
     ls -1S /cluster/bluearc/sts.9/sts.splitNotFound/sts*.fa > sts.lst
     mkdir -p /cluster/bluearc/hg17/sts/sts/out.noOoc
 
     foreach f (`cat sts.lst`)
         set d = $f:t:r
         mkdir /cluster/bluearc/hg17/sts/sts/out.noOoc/$d
     end
 
 cat > template << 'EOF'
 #LOOP
 /cluster/bin/i386/blat $(path1) $(path2) -stepSize=5 {check out line+ /cluster/bluearc/hg17/sts/sts/out.noOoc/$(root2)/$(root1).$(root2).psl}
 #ENDLOOP
 'EOF'
 # << for emacs
 
     gensub2 contigs.lst sts.lst template jobList
     para create jobList
         # 7220 jobs written to batch
     para try
     para check
 
     # process this set of alignments
     ssh kolossus
     cd /cluster/bluearc/hg17/sts/sts
     pslSort dirs raw.noOoc.psl temp out.noOoc/*
 
     rm -rf temp
     pslReps -nearTop=0.0001 -minCover=0.6 -minAli=0.8 -noIntrons \
         raw.noOoc.psl stsMarkers.noOoc.psl /dev/null
             # Processed 4254094 alignments
     #cp stsMarkers.psl /cluster/data/hg17/bed/sts/run
 
     # Lift them and get them ready to combine with primer alignments
     liftUp -nohead stsMarkers.noOoc.lifted.psl \
         /cluster/data/hg17/jkStuff/liftAll.lft warn stsMarkers.noOoc.psl
 
     /cluster/bin/scripts/extractPslInfo stsMarkers.noOoc.lifted.psl
         # creates <file>.initial
     /cluster/bin/scripts/findAccession -agp \
         stsMarkers.noOoc.lifted.psl.initial /cluster/data/hg17
             # "Could not open /cluster/data/hg17/Y/chrY_random.agp" etc.
             # Looks like it trys all _randoms (even one's that don't
             # exist/aren't needed
             # creates <file>.acc
     #rm stsMarkers.lifted.psl.initial
     mv stsMarkers.final stsMarkers.ooc.final
     sort -k 4n stsMarkers.noOoc.lifted.psl.initial.acc > stsMarkers.extra
     sort -k 4n stsMarkers.lifted.psl.initial.acc \
                 stsMarkers.noOoc.lifted.psl.initial.acc > stsMarkers.final
 
     # determine found markers (4th field in file)
     cut -f 4 stsMarkers.final | sort -n -u > stsMarkers.found
     wc -l stsMarkers.found
         #  89681 stsMarkers.found
     cut -f 4 stsMarkers.extra | sort -n -u > stsMarkers.extra.found
     wc -l stsMarkers.extra.found
         #   149 out of 3142 attempted
         # out of 92674 total sequences
     cp stsMarkers.final stsMarkers.lifted.psl stsMarkers.*lifted.psl.initial* stsMarkers.found \
                 /cluster/data/hg17/bed/sts
     # Alignments from noOoc set were not added to all_sts_seq but info for the markers
     # is in stsMap and stsInfo2. Some of the alignments are bad so filter by removing
     # all alignments from noOoc psl file where tBaseInsert >=1000. Add the remaining 
     # alignments to the set of final alignments for stsMarkers. The information for the  
     # removed markers from the filtered set was also removed from stsMap and stsInfo2.
     # (DONE, 2005-02-17, hartera)
     ssh eieio
     cd /cluster/data/hg17/bed/sts/fix
     cp /cluster/bluearc/hg17/sts/sts/stsMarkers.noOoc.lifted.psl .
     awk '{if ($8 < 1000) print;}' stsMarkers.noOoc.lifted.psl > stsMarkers.noOoc.lifted.filt1000.psl
     wc -l *.filt*.psl
     # 254    5334   26384 stsMarkers.noOoc.lifted.filt1000.psl
     sort -k 4n /cluster/bluearc/hg17/sts/sts/stsMarkers.noOoc.lifted.psl.initial.acc > stsMarkers.extra
     awk '{print $4;}' stsMarkers.extra | sort -n | uniq >  extra.ids
     # in psl file, the ids are the 10th field
     awk '{print $10;}' stsMarkers.noOoc.lifted.psl | sort -n | uniq \
         > noOoc.ids
     diff extra.ids noOoc.ids
     # there is no difference as expected
     # get list of IDs from filtered file, filter < 1000
     awk '{print $10;}' stsMarkers.noOoc.lifted.filt1000.psl \
         | sort -n | uniq > filt1000.ids
     foreach i (`cat filt1000.ids`)
      awk 'BEGIN {OFS="\t"} \
          {if ($4 == "'$i'") print $1, $2, $3, $4, $5, $6, $7}' \
          stsMarkers.extra >> stsMarkers.extra.filt1000
     end
     cp ../stsMarkers.final stsMarkers.final
   #  cat stsMarkers.extra.filt1000 >> stsMarkers.final2 
      # need to filter stsMarkers.final not just cat this on the end
     # get list of alignments with tBaseInsert >= 1000 and remove these
     cd /cluster/data/hg17/bed/sts/fix
     awk '{if ($8 >= 1000) print;}' stsMarkers.noOoc.lifted.psl > stsMarkers.noOoc.lifted.filtToRemove.psl
     wc -l *.filt*.psl
     # 254 stsMarkers.noOoc.lifted.filt1000.psl
     # 249 stsMarkers.noOoc.lifted.filt500.psl
     # 448 stsMarkers.noOoc.lifted.filtToRemove.psl
     # get list of IDs that need to be removed
     awk '{print $10;}' stsMarkers.noOoc.lifted.filtToRemove.psl | sort -n \
         | uniq  > noOoc.IdsToRemove.txt
     # get chrom and co-ordinates for IDs to be removed 
     awk 'BEGIN {OFS = "\t"} {print $14,$16,$17,$10}' \
         stsMarkers.noOoc.lifted.filtToRemove.psl | sort | uniq \
         > sts.noOoc.filtToRemove.coords
     # checked that the stsMarkers.final contain the noOoc alignments
     # wrote perl script to remove lines with these IDs from stsMarkers.final
 cat << '_EOF_' > removeIds.pl
 #!/usr/bin/perl -w
 use strict;
 
 my $ids = $ARGV[0];
 my $file = $ARGV[1];
 # list of IDs with chrom and coords to remove
 open(IDS, $ids) || die "Can not open $ids: $!\n";
 # file for removal of IDs
 open(FILE, $file) || die "Can not open $file: $!\n";
 open(OUT, ">removed.txt") || die "Can not create removed.txt: $!\n";
 
 my %idsHash;
 
 while (<IDS>) {
    chomp;
    my @a = split(/\t/);
 
    my $chr = $a[0];
    my $st = $a[1];
    my $end = $a[2];
    my $id = $a[3];
    my $key = $id."_".$chr . "_" . $st . "_" . $end;
    $idsHash{$key}->{chrom} = $chr;
    $idsHash{$key}->{start} = $st;
    $idsHash{$key}->{end} = $end;
 }
 close IDS;
 
 while (<FILE>) {
    chomp;
    my $l = $_;
    my $found = "FALSE";
    my @f = split(/\t/, $l);
    foreach my $k (keys(%idsHash)) {
       # if the id is contained in the key
       if ($k =~ /^$f[3]/) {
          my $c = $idsHash{$k}->{chrom};
          my $s = $idsHash{$k}->{start};
          my $e = $idsHash{$k}->{end};
          if ($f[0] eq $c && $f[1] == $s && $f[2] == $e) {
              print OUT "$c\t$s\t$e\t$f[3]\n";
              $found = "TRUE";
          }
       }
    }
    if ($found eq "FALSE") {
       print "$l\n";
    }
 }
 '_EOF_'
     chmod +x removeIds.pl
     perl removeIds.pl sts.noOoc.filtToRemove.coords stsMarkers.final \
          > stsMarkers.final.new
     wc -l stsMarkers.final*
     # 92338 stsMarkers.final
     # 91890 stsMarkers.final.new
     # There are 448 ids and sets of co-ordinates in list of Ids to remove
     # check that stsMarkers.final.new contains all the alignments that
     # are in filtered set: stsMarkers.noOoc.lifted.filt1000.psl
     awk 'BEGIN {OFS = "\t"} {print $14,$16,$17,$10}' \
         stsMarkers.noOoc.lifted.filt1000.psl | sort | uniq \
         > sts.noOoc.filt1000.coords
     awk 'BEGIN {OFS = "\t"} {print $1,$2,$3,$4}' \
         stsMarkers.final.new | sort | uniq \
         > sts.finalnew.coords
     diff sts.finalnew.coords sts.noOoc.filt1000.coords > finalnewvsfilt1000
     grep '>' finalnewvsfilt1000
     # there is nothing in sts.noOoc.filt1000.coords not found in the 
     # sts.finalnew.coords file therefore this contains all the alignments
     # from the filtered noOoc file.
     cp ../primers/primers.final .
     awk '{print $4}' primers.final | sort | uniq > primers.ids
     awk '{print $4}' stsMarkers.final.new | sort | uniq > stsfinal.new.ids
 
 
     # primers
     ssh eieio
     cd /cluster/data/ncbi/sts.9
     # strip out N's and wobbles (KS) from primers, as isPcr
     # can't currently handle them
     # strip out primers < 10 as isPcr can't handle them
     awk '$0 !~ /[^ACGT0-9\-\t]/ && (length($2) > 10) && (length($3) > 10) {printf "dbSTS_%s\t%s\t%s\n", $1,$2,$3}' \
                 all.primers > all.primers.ispcr
     mkdir -p /cluster/bluearc/sts.9/primers
     cd /cluster/bluearc/sts.9/primers
     split -l 2000 /cluster/data/ncbi/sts.9/all.primers.ispcr primers_
 
     ssh kk
     cd /cluster/data/hg17/bed/sts
     mkdir primers
     cd primers
     mkdir run
     cd run
     ls -1S /scratch/hg/gs.18/build35/maskedContigs/*.fa > contigs.lst
     ls -1S /cluster/bluearc/sts.9/primers/primers_* > primers.lst
     mkdir -p /cluster/bluearc/hg17/sts/primers/out
 
 cat > template << 'EOF'
 #LOOP
 /cluster/home/kate/bin/i386/isPcr -out=psl -minPerfect=2 -maxSize=5000 -tileSize=10 -ooc=/scratch/hg/h/10.ooc  -stepSize=5 $(path1) $(path2) {check out line /cluster/bluearc/hg17/sts/primers/out/$(root1)_$(root2).psl}
 #ENDLOOP
 'EOF'
 # << for emacs
 
     gensub2 contigs.lst primers.lst template jobList
     para create jobList
 	# 26980 jobs
     para try
     para check
     para push
 
     #Completed: 26953 of 26980 jobs
     #Crashed: 27 jobs
     #CPU time in finished jobs:    1130353s   18839.22m   313.99h   13.08d  0.036 y
     #IO & Wait Time:                 86067s    1434.44m    23.91h    1.00d  0.003 y
     #Average job time:                  45s       0.75m     0.01h    0.00d
     #Longest job:                     1255s      20.92m     0.35h    0.01d
     #Submission to last job:          2762s      46.03m     0.77h    0.03d
 
     # 27 jobs seg faulted due to -minPerfect=2.
     # Looks like a bug in isPcr -- till it's fixed,
     # we'll rerun with -minPerfect=5 (Terry determined they
     # all complete with this (he used 3, 4, or 5, tuned individually
     # for each job, but just using 5 should be adequate and 
     # less labor-intensive).
     # NOTE: isPcr bug is fixed -- this shouldn't be necessary for
     # next run
 
     para crashed | grep isPcr | sed 's/minPerfect=2/minPerfect=5/' \
         > jobList.minPerfect5
     para create jobList.minPerfect5
         # 28 jobs
     # repeat with increasing minPerfect, till all complete succesfully
 
     # Filter output file quickly based on simple parameters
     ssh kolossus
     cd /cluster/bluearc/hg17/sts/primers/
     mkdir -p filter
     pslQuickFilter -minMatch=26 -maxMismatch=5 -maxTinsert=5000 -verbose out/ filter/
 	# Note: there will be many messages saying files are empty - this is OK
     pslSort dirs primers.psl.unlifted temp filter
 
     # filter primer alignments and create not found primer file for ePCR run (booch)
     pslFilterPrimers /cluster/bluearc/hg17/sts/primers/primers.psl.unlifted  \
 	/cluster/data/ncbi/sts.9/all.primers primers.filter.unlifted.psl
         # creates $3.notfound.primers
     wc -l primers.filter.unlifted.psl.notfound.primers                   
     # 21919  primers.filter.unlifted.psl.notfound.primers
 
     # use Greg Schuler's ePCR to attempt alignment of primers missed
     # by isPcr
     mkdir -p /cluster/data/hg17/bed/sts/primers/run.epcr
     mkdir -p /cluster/bluearc/hg17/sts/primers/epcr
     cd /cluster/bluearc/hg17/sts/primers/epcr
     split -l 2500 /cluster/data/hg17/bed/sts/primers/primers.filter.unlifted.psl.notfound.primers  primers_
     cd /cluster/data/hg17/bed/sts/primers/run.epcr
     ls -1S /cluster/bluearc/hg17/sts/primers/epcr/primers_* > primers.lst
     # create contig.lst based on split in build dir
     # NOTE: should probably replace this with something more standard
     # and faster.  Also, this appears to cause load spikes on fileservers.
     # Should get contigs from bluearc, iservers, or cluster local disk
     # At least it's over pretty quick!
     ssh eieio
     cd /cluster/data/hg17/bed/sts/primers/run.epcr
     /cluster/bin/scripts/splitContigList -ncbi /cluster/data/hg17 1
     # next time... ls -1S /cluster/bluearc/hg17/contigs/* > contig.lst (?)
     mkdir -p /cluster/bluearc/hg17/sts/primers/epcr/out
 
     ssh kk
     cd /cluster/data/hg17/bed/sts/primers/run.epcr
 
 cat > template << 'EOF'
 #LOOP
 /cluster/bin/scripts/runEpcr $(path1) $(path2) {check out line /cluster/bluearc/hg17/sts/primers/epcr/out/$(root1).$(root2).epcr}
 #ENDLOOP
 'EOF'
 # << for emacs
     gensub2 primers.lst contig.lst template jobList
     para create jobList
 	# 3420 jobs
     para try
     para check
     para push
 
 # CPU time in finished jobs:      78897s    1314.95m    21.92h    0.91d  0.003 y
 # IO & Wait Time:                254582s    4243.03m    70.72h    2.95d  0.008 y
 # Average job time:                  98s       1.63m     0.03h    0.00d
 # Longest job:                      647s      10.78m     0.18h    0.01d
 # Submission to last job:          1112s      18.53m     0.31h    0.01d
 
     # merge output
     ssh eieio
     cd /cluster/bluearc/hg17/sts/primers/epcr
     cat out/*.epcr > all.epcr
     wc -l all.epcr
     # 3573 
 
     # use all.epcr file to re-filter alignemnts and determine which
     # ePCR records to keep
     cp all.epcr /cluster/data/hg17/bed/sts/primers
     cd /cluster/data/hg17/bed/sts/primers
     pslFilterPrimers -epcr=all.epcr -verbose=1 \
         /cluster/bluearc/hg17/sts/primers/primers.psl.unlifted \
 	/cluster/data/ncbi/sts.9/all.primers primers.unlifted.epcr.psl
 
     # convert to PSL and combine with other psl file (this takes a couple hours)
     /cluster/bin/scripts/epcrToHgPsl epcr.not.found \
         /cluster/data/ncbi/sts.9/all.primers /cluster/data/hg17
     cat primers.unlifted.epcr.psl epcr.not.found.psl \
                 | sort -k 10n > primers.final.unlifted.psl
 
     # Fix the query gap lengths so that they match the all.primers.fa 
     #   file lengths
     /cluster/bin/scripts/fixPrimersQueryGaps \
         /cluster/data/ncbi/sts.9/all.primers primers.final.unlifted.psl \
                 > primers.final.unlifted.fix.psl
 
     # lift results from contigs to chrom coordinates, and create final file
     liftUp -nohead /cluster/data/hg17/bed/sts/primers/primers.psl \
             /cluster/data/hg17/jkStuff/liftAll.lft warn \
             primers.final.unlifted.fix.psl
     # Extract relevant info, make alignments unique, and create final file to be merged
     # with full sequence alignments
     /cluster/bin/scripts/extractPslInfo primers.psl
     /cluster/bin/scripts/findAccession -agp primers.psl.initial \
                 /cluster/data/hg17
     #rm primers.psl.initial
 
     /cluster/bin/scripts/getStsId /cluster/data/ncbi/sts.9/stsInfo2.bed \
 	primers.psl.initial.acc \
         | sort -k 4n > primers.final
     #rm primers.psl.initial.acc
     wc -l primers.final
     # 314713 primers.final
 
     # Merge primer and sequence files to create final bed file
     # Merge (combineSeqPrimerPos) takes about an hour to run
     ssh kolossus
     cd /cluster/data/hg17/bed/sts
     
     /cluster/bin/scripts/combineSeqPrimerPos stsMarkers.final primers/primers.final
         # creates *_pos.rdb
     /cluster/bin/scripts/createSTSbed /cluster/data/ncbi/sts.9/stsInfo2.bed \
                 stsMarkers_pos.rdb > stsMap.bed
 
     # Set up sequence files
     ssh hgwdev
     mkdir -p /gbdb/hg17/sts.9/
     ln -s /cluster/data/ncbi/sts.9/all.STS.fa /gbdb/hg17/sts.9/all.STS.fa
     ln -s /cluster/data/ncbi/sts.9/all.primers.fa \
         /gbdb/hg17/sts.9/all.primers.fa
 
     # Load all files
     cd /cluster/data/hg17/bed/sts
     hgLoadSeq hg17 /gbdb/hg17/sts.9/all.STS.fa /gbdb/hg17/sts.9/all.primers.fa
     hgsql hg17 < ~kent/src/hg/lib/stsInfo2.sql
     hgsql hg17 < ~kent/src/hg/lib/stsAlias.sql
     cp /cluster/data/ncbi/sts.9/{stsInfo2.bed,stsAlias.bed} .
     hgsql hg17 -e 'load data local infile "stsInfo2.bed" into table stsInfo2'
     hgsql hg17 -e 'load data local infile "stsAlias.bed" into table stsAlias'
     hgLoadBed -noBin -tab -sqlTable=/cluster/home/kent/src/hg/lib/stsMap.sql \
 	hg17 stsMap stsMap.bed
     hgLoadPsl -nobin -table=all_sts_primer hg17 primers/primers.psl
     hgLoadPsl -nobin -table=all_sts_seq hg17 stsMarkers.lifted.psl
 
     # update of information for D21S167 and D21S168 (2005-02-11, hartera)
     # currently X52289 associated with D21S168 
     # and X53367 associated with D21S167 - these need to be switched as they
     # are causing incorrect positioning
     # On Terry's advice, 
     # first manually update the accession field stsInfo2.bed so that the 
     # corrected version is carried through to the next version
     cd /cluster/data/hg17/bed/sts
     # manually change accessions in this file so now X52289 is associated
     # with D21S167 and X53367 is now associated with D21S168 
     # manually update the chromStart and chromEnd fields for these
     # records in stsMap.bed
     # this change was not carried through after filtering to change stsMap.bed
     # again and reload this table (DONE, 2005-02-18, hartera)
     chr21   39867340        39867513        D21S167 1000    7888    AF064860
     # becomes
     chr21   37117635        37117858        D21S167 1000    7888    AF064860
 
     chr21   37117635        37117858        D21S168 1000    103256  AP000699
     # becomes
     chr21   39867340        39867513        D21S168 1000    103256  AP000699
     
     # then reload the stsMap.bed and stsInfo2.bed files
     # copy this updated bed file back to ncbi directory
     cp stsInfo2.bed /cluster/data/ncbi/sts.9/
     # delete previous data before reloading tables
     hgsql hg17 -e 'delete from stsInfo2'
     hgsql hg17 -e 'drop table stsMap'
     hgsql hg17 -e 'load data local infile "stsInfo2.bed" into table stsInfo2'
     hgLoadBed -noBin -tab -sqlTable=/cluster/home/kent/src/hg/lib/stsMap.sql \
 	hg17 stsMap stsMap.bed
     # (2005-02-19, hartera)
     # also need to update the psl alignment file and reload into all_sts_seq
     # for D21S168, the id is 103256, this is qName in the psl file
     # for D21S167, the id is 7888
     cd /cluster/data/hg17/bed/sts
     # manually update the stsMarkers.lifted.psl file with the new
     # co-ordinates as above.
     # (2005-02-23) Correct alignments.
     # need to swap the names for the alignments not just the start and end
     # coords as before as now the rest of the alignment data fields in the 
     # table are incorrect. Change the start and end co-ordinates and just swap 
     # the names for D21S167 and D21S168 in the psl file then reload the table. 
     # sort on the ID field (qName)
     sort -k 10n stsMarkers.lifted.psl > sts.lifted.sort
     mv sts.lifted.sort stsMarkers.lifted.psl 
     hgsql hg17 -e 'drop table all_sts_seq'
     hgLoadPsl -nobin -table=all_sts_seq hg17 stsMarkers.lifted.psl
     
     # Add new information after filtering the noOoc files 
     # (DONE, 2005-02-17, hartera)
     # latest psl file: stsMarker.lifted.new.psl is in fix dir    
     # Merge primer and sequence files to create final bed file
     ssh kolossus
     cd /cluster/data/hg17/bed/sts/fix
     nice /cluster/bin/scripts/combineSeqPrimerPos stsMarkers.final.new \
                                              ../primers/primers.final
     # creates *_pos.rdb
     /cluster/bin/scripts/createSTSbed /cluster/data/ncbi/sts.9/stsInfo2.bed \
                 stsMarkers_pos.rdb > stsMap.bed
     awk '{print $6;}' stsMap.bed | sort -n | uniq > stsMap.ids
     diff stsMap.ids filt1000.ids 
     # There is only 1 id that does not make it into this set (109375)
     # There are 38 of the IDs to remove that do not appear in stsMap.ids
     # there are 65 therefore that appear in stsMap.bed: noOoctoremoveinStsMap
     foreach i (`cat noOoctoremoveinStsMap`) 
        awk 'BEGIN {OFS = "\t"} {if ($10 == "'$i'" && $8 >= 1000) \
         print $14, $16, $17, $10;}' stsMarkers.noOoc.lifted.filtToRemove.psl \
         >> stsMap.noOoc.toRemove.coords
     end 
     sort stsMap.noOoc.toRemove.coords > stsMap.noOoc.toRemove.coords.sort
     wc -l stsMap.noOoc.toRemove.coords.sort
     # 122
     # get the equivalent co-ordinates from stsMap.bed
     foreach i (`cat noOoctoremoveinStsMap`)
        awk 'BEGIN {OFS = "\t"} {if ($6 == "'$i'") print $1,$2,$3,$6;}' \
            stsMap.bed >> stsMap.toRemove.coords
     end
     sort stsMap.toRemove.coords > stsMap.toRemove.coords.sort
     wc -l stsMap.toRemove.coords.sort
     # 68
     diff stsMap.noOoc.toRemove.coords stsMap.toRemove.coords.sort
     # They are different co-ordinates in each set although the same ID
     # is represented.
     # none of the noOoc alignments are in stsMarkers.lifted.psl so add
     cp ../stsMarkers.lifted.psl stsMarkers.lifted.psl
     awk '{print $10}' stsMarkers.lifted.psl | sort -n | uniq > sts.liftedpsl.ids
     # none of the noOoc alignments are in stsMarkers.lifted.psl so add
     # the filtered version
     cp stsMarkers.lifted.psl stsMarkers.lifted.new.psl
     cat stsMarkers.noOoc.lifted.filt1000.psl >> stsMarkers.lifted.new.psl
     wc -l stsMarkers.lifted.new.psl
     # 91890 
     awk '{print $1;}' ../stsInfo2.bed | sort -n | uniq > stsInfo2.ids
     # diff with filt1000.ids and noOoc.IdsToRemove.txt
     # all of these are in stsInfo2.bed
     # need to remove info for the filtered out set but only for the 38 that
     # were removed from stsMap.bed - noOocnotinstsMap 
     perl removeById.pl noOocnotinstsMap stsInfo2.bed > stsInfo2.new.bed
 cat << '_EOF_' > removeById.pl
 #!/usr/bin/perl -w
 use strict;
 
 my $ids = $ARGV[0];
 my $file = $ARGV[1];
 # list of to remove
 open(IDS, $ids) || die "Can not open $ids: $!\n";
 # file of stsMarkers.final
 open(FILE, $file) || die "Can not open $file: $!\n";
 open(OUT, ">removedIds.txt") || die "Can not create removedIds.txt: $!\n";
 
 my %idsHash;
 while (<IDS>) {
    chomp;
    my @a = split(/\t/);
    my $id = $a[0];
    $idsHash{$id} = 1;
 }
 close IDS;
 
 while (<FILE>) {
    my $l = $_;
    my $found = "FALSE";
    my @f = split(/\t/, $l);
    foreach my $k (keys(%idsHash)) {
       # if the id is contained in the key
       if ($k eq $f[0]) {
          $found = "TRUE";
          print OUT "$f[0]\n";
       }
    }
    if ($found eq "FALSE") {
       print $l;
    }
 }
 '_EOF_'
     # << emacs
     chmod +x removeById.pl
     # this removed data for all 38 of these Ids from stsInfo2.bed
     
     # need to reload database tables (2005-02-18, hartera)
     ssh hgwdev
     cd /cluster/data/hg17/bed/sts/fix
     hgsql hg17 -e 'drop table stsMap'
     hgsql hg17 -e 'drop table all_sts_seq'
     hgsql hg17 -e 'drop table stsInfo2'
     mv stsInfo2.new.bed stsInfo2.bed
     cp stsInfo2.bed /cluster/data/ncbi/sts.9/stsInfo2.bed
     mv stsMap.new.bed stsMap.bed
     mv stsMarkers.lifted.new.psl stsMarkers.lifted.psl
     hgLoadBed -noBin -tab -sqlTable=/cluster/home/kent/src/hg/lib/stsMap.sql \
 	hg17 stsMap stsMap.bed
     hgLoadPsl -nobin -table=all_sts_seq hg17 stsMarkers.lifted.psl
     hgsql hg17 < ~kent/src/hg/lib/stsInfo2.sql
     hgsql hg17 -e 'load data local infile "stsInfo2.bed" into table stsInfo2'
     cd ..
     mkdir old
     mv stsMap.bed stsInfo2.bed stsMarkers.lifted.psl ./old
     mv ./fix/stsMap.bed ./fix/stsInfo2.bed ./fix/stsMarkers.lifted.psl .
    
     # Update of stsAlias table (DONE, 2005-02-24, hartera)
     # stsAlias filtered IDs removed
     # should have same IDs as in stsInfo2
     ssh eieio
     cd /cluster/data/hg17/bed/sts/fix
     awk '{print $2;}' ../stsAlias.bed | sort -n | uniq > alias.ids
     # 145985 alias.ids
     awk '{print $6;}' ../stsMap.bed | sort -n | uniq > stsMap.new.ids.sort
     awk '{print $1;}' ../stsInfo2.bed | sort -n | uniq > stsInfo.new.ids.sort
     # 16678 ids in stsInfo2 that are not in stsMap
     # 16717 ids in stsAlias that are not in stsMap
     # 38 ids in stsAlias that are not in stsInfo2
     cat stsMap.new.ids.sort stsInfo.new.ids.sort | sort -n | uniq \
         > stsMapandInfo.ids.sort
     diff stsMapandInfo.ids.sort alias.ids | grep '>' > idstoremoveAlias
     # there are 38 of these IDs to remove
     perl -pi.bak -e 's/> //' idstoremoveAlias
     cp ../stsAlias.bed .
     foreach i (`cat idstoremoveAlias`)
       awk '{if ($2 != "'$i'") print;}' stsAlias.bed > stsAlias.tmp
       mv stsAlias.tmp stsAlias.bed
     end
     # check that ids are removed from file and that they are the correct ones
     # all looks good
     cd /cluster/data/hg17/bed/sts
     # save old stsAlias file and copy new one to sts dir and to ncbi sts dir
     mv stsAlias.bed ./old
     cp ./fix/stsAlias.bed .
     cp stsAlias.bed /cluster/data/ncbi/sts.9/stsAlias.bed
     ssh hgwdev
     # remove old table data and reload
     hgsql hg17 -e 'delete from stsAlias'
     hgsql hg17 -e 'load data local infile "stsAlias.bed" into table stsAlias'
 
 # PRUNE stsMap RECORDS (DONE 3/3/06)
 
   hgsql hg17 -e 'delete from stsMap where chromEnd-chromStart > 5000'
   
 # RECOMBINATION RATES (2004-07-13 Terry)
 #                       (2004-07-21 kate)
 
 # The STS MArkers track must be completed prior to creating this track
 
     ssh eieio
     cd /cluster/data/hg17/bed
     mv recombRate recombRate.terry
     mkdir -p recombRate
 	cd recombRate
 
 # Copy other necessary files here (in future, can take from previous version)
 # NOTE: these are stable, and could be saved in a permanent spot
     cp /projects/hg2/booch/psl/info/decode_all .
     cp /projects/hg2/booch/psl/info/marshfield_all .
     cp /projects/hg2/booch/psl/info/genethon_all .
 	
 # Determine maximum concordant set of markers for each of the maps
     /cluster/bin/scripts/assignGPsts -full -maxcon \
         /cluster/data/ncbi/sts.9/stsAlias.bed \
         /cluster/data/hg17/bed/sts/stsMarkers_pos.rdb \
         decode_all > decode.marker.rdb
     /cluster/bin/scripts/assignGPsts -full -maxcon \
         /cluster/data/ncbi/sts.9/stsAlias.bed \
         /cluster/data/hg17/bed/sts/stsMarkers_pos.rdb \
         marshfield_all > marshfield.marker.rdb
     /cluster/bin/scripts/assignGPsts -full -maxcon \
         /cluster/data/ncbi/sts.9/stsAlias.bed \
         /cluster/data/hg17/bed/sts/stsMarkers_pos.rdb \
         genethon_all > genethon.marker.rdb
 
 # Determine the rates for each of the maps
     /cluster/bin/scripts/markers_to_recomb_rate.terry.pl decode.marker.rdb \
             /cluster/data/hg17/chrom.sizes 1000000 1000000 \
                 > decode_1mb_slide_1mb
     /cluster/bin/scripts/markers_to_recomb_rate.terry.pl genethon.marker.rdb \
             /cluster/data/hg17/chrom.sizes 1000000 1000000 \
                 > genethon_1mb_slide_1mb
         # Marker number 2 at position 120005974 on chr9 is out of genetic distance order. DISCARDING
     /cluster/bin/scripts/markers_to_recomb_rate.terry.pl marshfield.marker.rdb \
             /cluster/data/hg17/chrom.sizes 1000000 1000000 \
                 > marshfield_1mb_slide_1mb
         # Marker number 1 at position 124276104 on chr9 is out of genetic distance order. DISCARDING
 
 # Convert files to proper format
     /cluster/bin/scripts/convertRecombRate decode_1mb_slide_1mb \
         /cluster/data/hg17/inserts \
         /cluster/data/hg17 1000 > decode_1mb_slide_1mb_conv
     /cluster/bin/scripts/convertRecombRate marshfield_1mb_slide_1mb \
         /cluster/data/hg17/inserts \
          /cluster/data/hg17 1000 > marshfield_1mb_slide_1mb_conv
     /cluster/bin/scripts/convertRecombRate genethon_1mb_slide_1mb \
         /cluster/data/hg17/inserts \
 	    /cluster/data/hg17 1000 > genethon_1mb_slide_1mb_conv
 
 # Create bed file and load
     /cluster/bin/scripts/createRRbed decode_1mb_slide_1mb_conv \
         marshfield_1mb_slide_1mb_conv genethon_1mb_slide_1mb_conv \
                 > recombRate.bed
     hgLoadBed -noBin -tab \
         -sqlTable=/cluster/home/kent/src/hg/lib/recombRate.sql \
 	    hg17 recombRate recombRate.bed
 	   
 
 # FISH CLONES (DONE 2004-07-22 Kate)
 #  Reloaded 2004-09-36 after Terry Furey reworked fishClones.c
 #       to improve scoring
 
 # The STS Marker, Coverage, and BAC End Pairs tracks must be completed prior to 
 # creating this track
 
     ssh eieio
     mkdir -p /cluster/data/ncbi/fishClones/fishClones.2004-07/
     cd /cluster/data/ncbi/fishClones/fishClones.2004-07/
 
 # Download information from NCBI
         # point browser at http://www.ncbi.nlm.nih.gov/genome/cyto/cytobac.cgi?CHR=all&VERBOSE=ctg
         # change "Show details on sequence-tag" to "yes"
         # change "Download or Display" to "Download table for UNIX"
         # press Submit - save as /cluster/data/ncbi/fishClones/fishClones.2004-07/hbrc.txt
     chmod 664 /cluster/data/ncbi/fishClones/fishClones.2004-07/hbrc.txt
 
 # Get current clone/accession information
     wget http://www.ncbi.nlm.nih.gov/genome/clone/DATA/clac.out
 
 # Create initial Fish Clones bed file
     mkdir -p /cluster/data/hg17/bed/fishClones
     cd /cluster/data/hg17/bed/fishClones
 
 # Copy previous sts info from fhcrc (take from previous build in future)
     cp ~booch/tracks/fish/fhcrc.sts .
     fishClones -verbose=1 -fhcrc=fhcrc.sts -noBin hg17 \
          /cluster/data/ncbi/fishClones/fishClones.2004-07/hbrc.txt \
          /cluster/data/ncbi/fishClones/fishClones.2004-07/clac.out \
          /cluster/data/ncbi/bacends/human/bacends.4/cl_acc_gi_len \
          /cluster/data/hg17/bed/bacends/lifted/bacEnds.lifted.psl \
             fishClones_initial
 	     
 # Get sequences for accessions not in genome
     ssh eieio
     mkdir -p /cluster/bluearc/hg17/fishClones/
     cd /cluster/bluearc/hg17/fishClones/
 	# goto http://www.ncbi.nlm.nih.gov/entrez/batchentrez.cgi?db=Nucleotide
 	# select file "/cluster/data/hg17/bed/fishClones/fishClones_initial.acc"
 	# change output to FASTA format
 	# download results to "/cluster/bluearc/hg17/fishClones/notFound.fa"
 
 # Align these using blat
     cp ~booch/tracks/gs.17/build34/fish/convert.pl .
     cp ~booch/tracks/gs.17/build34/fish/blatAll.pl .
         # edited to use ooc file on bluearc, so can run on kolossus
     convert.pl  < notFound.fa > notFound.convert.fa
     mkdir out
     blatAll.pl /cluster/data/hg17 notFound.convert.fa out
         # creates raw.psl, not.found.psl
 
 # Make final fishClones file with this new clone placement info
     cd /cluster/data/hg17/bed/fishClones
     fishClones -verbose=1 -fhcrc=fhcrc.sts -noBin \
          -psl=/cluster/bluearc/hg17/fishClones/not.found.psl hg17 \
          /cluster/data/ncbi/fishClones/fishClones.2004-07/hbrc.txt \
          /cluster/data/ncbi/fishClones/fishClones.2004-07/clac.out \
          /cluster/data/ncbi/bacends/human/bacends.4/cl_acc_gi_len \
          /cluster/data/hg17/bed/bacends/lifted/bacEnds.lifted.psl fishClones
 
 # Load the track
     ssh hgwdev
     cd /cluster/data/hg17/bed/fishClones
     hgLoadBed -noBin -tab \
         -sqlTable=/cluster/home/kent/src/hg/lib/fishClones.sql \
 	hg17 fishClones fishClones.bed
             # Loaded 10601 elements of size 16
 
 # fixed bad table entry (2004-08-12 kate)
 # NOTE: this won't be necessary in the future, as the fishClones program
 # will now accomodate more bad input data.
     hgsql hg17 -e "update fishClones set bandEnds='1q43,Yp' where  name='RP11-188A4' and placeCount=2"
 
 
 # CHROMOSOME BANDS TRACK (2004-07-13 Terry)
 
 # This must wait until the Fish Clones tracks is done
     mkdir -p /cluster/data/hg17/bed/cytoband
     cd /cluster/data/hg17/bed/cytoband
 
 # Copy in some necessary files (usually from previous version)
     cp /projects/hg2/booch/psl/cytobands/pctSetBands.txt .
     cp /projects/hg2/booch/psl/cytobands/ISCN800.txt .
 
 # Create some preliminary information files
     /cluster/bin/scripts/createSetBands pctSetBands.txt \
 	/cluster/data/hg17/inserts /cluster/data/hg17  100 > setBands.txt
     /cluster/bin/scripts/makeBands ISCN800.txt /cluster/data/hg17 > cytobands.pct.bed
     /cluster/bin/scripts/makeBandRanges cytobands.pct.bed > cytobands.pct.ranges
 
 # Reformat fishClones file
     /cluster/bin/scripts/createBanderMarkers \
 	/cluster/data/hg17/bed/fishClones/fishClones.bed > fishClones.txt
 
 # Create bed file
     /cluster/bin/scripts/runBander fishClones.txt \
 	ISCN800.txt setBands.txt /cluster/data/hg17
 
     # Should be 862 bands
     wc cytobands.bed
     # 862    4310   30748 cytobands.bed
 
 # Load track
     hgLoadBed -noBin -tab -sqlTable=/cluster/home/kent/src/hg/lib/cytoBand.sql \
 	hg17 cytoBand cytobands.bed
 	    
 # Load ideogram table
     hgLoadBed -noBin -tab -sqlTable=/cluster/home/booch/src/hg/lib/cytoBandIdeo.sql \
 	hg17 cytoBandIdeo cytobands.bed
 
 
 # CHROMOSOME BANDS TRACK REDO (2004-07-22 Kate)
 #       Just to make sure we know the proper steps.
 #       The tables were not reloaded, as Terry has already
 #       sent the data to NCBI
 
 # This must wait until the Fish Clones tracks is done
     ssh kolossus
     mkdir -p /cluster/data/hg17/bed/cytoband.kate
     cd /cluster/data/hg17/bed/cytoband.kate
 
 # Copy in some necessary files (usually from previous version)
     cp /projects/hg2/booch/psl/cytobands/pctSetBands.txt .
     cp /projects/hg2/booch/psl/cytobands/ISCN800.txt .
 
 # Create some preliminary information files
     /cluster/bin/scripts/createSetBands pctSetBands.txt \
 	/cluster/data/hg17/inserts /cluster/data/hg17  100 > setBands.txt
     /cluster/bin/scripts/makeBands ISCN800.txt \
         /cluster/data/hg17 > cytobands.pct.bed
     /cluster/bin/scripts/makeBandRanges cytobands.pct.bed \
         > cytobands.pct.ranges
 
 # Reformat fishClones file
     /cluster/bin/scripts/createBanderMarkers \
 	/cluster/data/hg17/bed/fishClones/fishClones.bed > fishClones.txt
 
 # Create bed file
     ssh eieio
     cd /cluster/data/hg17/bed/cytoband.kate
     /cluster/bin/scripts/runBander fishClones.txt \
 	ISCN800.txt setBands.txt /cluster/data/hg17
     # NOTE: fails on kolossus (C++ compiler different ??)
 
     # Should be 862 bands
     wc  -l cytobands.bed
     # 862    cytobands.bed
 
 # NOTE - don't load tracks, as Terry has already sent his
 #       versions to NCBI
 
 # Load track
     #hgLoadBed -noBin -tab \
         # -sqlTable=/cluster/home/kent/src/hg/lib/cytoBand.sql \
 	# hg17 cytoBand cytobands.bed
 	    
 # Load ideogram table
     #hgLoadBed -noBin -tab \
         # -sqlTable=/cluster/home/booch/src/hg/lib/cytoBandIdeo.sql \
 	# hg17 cytoBandIdeo cytobands.bed
 
 
 # LOAD AFFYRATIO (DONE - 2004-07-14 - Hiram)
 #	Copied from Hg16 doc
     # Set up cluster job to align consenesus/exemplars to hg17
     ssh eieio
     mkdir /cluster/bluearc/hg17/affyGnf
     cp -p /projects/compbio/data/microarray/affyGnf/sequences/HG-U95/HG-U95Av2_all.fa /cluster/bluearc/hg17/affyGnf
 
     ssh kkr1u00
     mkdir -p /iscratch/i/affyGnf
     cp -p /cluster/bluearc/hg17/affyGnf/* /iscratch/i/affyGnf
     /cluster/bin/iSync
 
     ssh kki
     mkdir /cluster/data/hg17/bed/affyGnf.2004-06-09
     cd /cluster/data/hg17/bed/affyGnf.2004-06-09
     ls -1 /iscratch/i/affyGnf/* > affy.lst
     ls -1 /iscratch/i/gs.18/build35/maskedContigs/* > allctg.lst
     cat << '_EOF_' > template.sub
 #LOOP
 /cluster/bin/i386/blat -fine -mask=lower -minIdentity=95 -ooc=/iscratch/i/gs.18/build35/hg17.11.ooc  $(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     gensub2 allctg.lst affy.lst template.sub jobList
     mkdir psl
     para create jobList
 # Completed: 380 of 380 jobs
 # CPU time in finished jobs:       2922s      48.70m     0.81h    0.03d  0.000 y
 # IO & Wait Time:                  1146s      19.10m     0.32h    0.01d  0.000 y
 # Average job time:                  11s       0.18m     0.00h    0.00d
 # Longest job:                       80s       1.33m     0.02h    0.00d
 # Submission to last job:           333s       5.55m     0.09h    0.00d
 
     # Do sort, best in genome filter, and convert to chromosome coordinates
     # to create affyU95.psl
     ssh eieio
     cd /cluster/data/hg17/bed/affyGnf.2004-06-09
     pslSort dirs raw.psl tmp psl
 
     # change filter parameters for these sequences. only use alignments that
     # cover 30% of sequence and have at least 95% identity in aligned
     # region. 
     # minAli = 0.97 too high. low minCover as a lot of n's in these
     # sequences
     pslReps -minCover=0.3 -sizeMatters -minAli=0.95 -nearTop=0.005 \
 	raw.psl contig.psl /dev/null
     liftUp affyU95.psl ../../jkStuff/liftAll.lft warn contig.psl
     #   Eliminate the long names
     sed -e "s/U95Av2://" affyU95.psl | sed -e "s/;//" > affyU95shortQname.psl
 
     # Merge with spot data and load into database. added -chip flag to 
     # affyPslAndAtlasToBed to allow correct parsing
     ssh hgwdev
     cd /cluster/data/hg17/bed/affyGnf.2004-06-09
     
     /cluster/home/sugnet/bin/i386/affyPslAndAtlasToBed -chip=U95Av2 \
 	affyU95shortQname.psl \
 	/projects/compbiodata/microarray/affyGnf/human_atlas_U95_gnf.noquotes.txt \
 	affyRatio.bed affyRatio.exr > affyPslAndAtlasToBed.log 2>&1
 
     hgLoadBed -sqlTable=$HOME/src/hg/lib/affyRatio.sql hg17 \
 	affyRatio affyRatio.bed
     #	Loaded 12740 elements of size 15
 
     mkdir affyU95
     hgLoadPsl hg17 -table=affyU95 affyU95shortQname.psl
     # sequences loaded 2004-08-06
     hgLoadSeq -abbr=U95Av2: hg17 /gbdb/hgFixed/affyProbes/HG-U95Av2_all.fa
     #	Advisory lock created
     #	Creating .tab file
     #	Adding /gbdb/hgFixed/affyProbes/HG-U95Av2_all.fa
     #	12386 sequences
     #	Updating seq table
     #	Advisory lock has been released
     #	All done
 
 # Load AFFYUCLANORM, extended version of affyUcla track. Hopefully
 # final freeze of data set.		(DONE - 2004-07-14 - Hiram)
     ssh kk
     mkdir /cluster/data/hg17/bed/affyUclaNorm
     cd /cluster/data/hg17/bed/affyUclaNorm
 
     cp /projects/compbio/data/microarray/affyUcla/sequences/HG-U133AB_all.fa .
     ls -1 /scratch/hg/gs.18/build35/maskedContigs/* > contig.lst
 
     cat << '_EOF_' > gsub
 #LOOP
 /cluster/bin/i386/blat -fine -mask=lower -minIdentity=95 -ooc=/scratch/hg/h/11.ooc  $(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
     mkdir psl
     ls HG-U133AB_all.fa > affy.lst
     gensub2 contig.lst affy.lst gsub jobList
     para create jobList
     para try
     para check
     para push ... etc
 # Completed: 380 of 380 jobs
 # CPU time in finished jobs:      20070s     334.51m     5.58h    0.23d  0.001 y
 # IO & Wait Time:                162784s    2713.06m    45.22h    1.88d  0.005 y
 # Average job time:                 481s       8.02m     0.13h    0.01d
 # Longest job:                      735s      12.25m     0.20h    0.01d
 # Submission to last job:           771s      12.85m     0.21h    0.01d
 
     ssh eieio
     cd /cluster/data/hg17/bed/affyUclaNorm
     pslSort dirs hg17.affyU133AB_all.psl tmp psl
     wc hg17.affyU133AB_all.psl
     #	61022 1281401 12934919 hg17.affyU133AB_all.psl
     liftUp hg17.affyU133AB_all.lifted.psl \
 	/cluster/data/hg17/jkStuff/liftAll.lft warn hg17.affyU133AB_all.psl 
     pslReps -minCover=0.5 -sizeMatters -minAli=0.97 \
 	-nearTop=0.005  hg17.affyU133AB_all.lifted.psl \
 	hg17.affyU133AB_all.lifted.pslReps.psl out.psr
     #	Processed 61017 alignments
     affyUclaMergePslData -pslFile=hg17.affyU133AB_all.lifted.pslReps.psl \
 	-affyFile=/projects/compbio/data/microarray/affyUcla/data/feature_biomaterial_chip_logratios_formatForTrack.txt \
 	-bedOut=hg17.affyUcla.bed \
 	-expRecordOut=hg17.affyUcla.expRecords \
 	-expFile=/projects/compbio/data/microarray/affyUcla/data/expNames.sorted.txt
 
     ~/kent/src/hg/affyGnf/addUclaAnnotations.pl hg17.affyUcla.expRecords \
 	/projects/compbio/data/microarray/affyUcla/data/normal_tissue_database_annotations2.txt > hg17.affyUcla.annotations.expRecords
 
     # Load the databases
     ssh hgwdev
     cd /cluster/data/hg17/bed/affyUclaNorm
     sed -e 's/affyRatio/affyUclaNorm/' ~/kent/src/hg/lib/affyRatio.sql \
 	> affyUclaNorm.sql
     hgLoadBed hg17 affyUclaNorm hg17.affyUcla.bed -sqlTable=affyUclaNorm.sql
 
 # MAKE AFFY U133 - made after above affyUclaNorm (DONE - 2004-07-15 - Hiram)
     #	Someday the names can be fixed.
     ssh hgwdev
     mkdir /cluster/data/hg17/bed/affyU133
     cd /cluster/data/hg17/bed/affyU133
     ln -s ../affyUclaNorm/hg17.affyU133AB_all.lifted.pslReps.psl affyU133.psl
     hgLoadPsl hg17 affyU133.psl
     #	hgsql -e "select count(*) from affyU133;" hg17
     #	row count in hg16: 45693, in hg17: 44620
     hgLoadSeq hg17 /gbdb/hgFixed/affyProbes/HG-U133AB_all.fa
     #	44792 sequences
 
 # MAKE LINEAGE-SPECIFIC REPEATS FOR CHICKEN & FUGU (DONE 2004-06-10 kate)
     # In an email 2/13/04 to Angie, Arian said we could treat all 
     # human repeats as 
     # lineage-specific for human-chicken blastz.  
     # and Angie did the same for fugu.
     # Lacking input from Arian, and using blastzSelf as a model,
     # I'm also using all human repeats for the human/chimp blastz.
     # Scripts expect *.out.spec filenames.
     ssh kkr1u00
     cd /cluster/data/hg17
     mkdir /iscratch/i/hg17/linSpecRep.chicken
     foreach f (/iscratch/i/hg17/rmsk/chr*.fa.out)
       cp -p $f /iscratch/i/hg17/linSpecRep.chicken/$f:t:r:r.out.spec
     end
     ln -s /iscratch/i/hg17/linSpecRep.chicken \
           /iscratch/i/hg17/linSpecRep.fugu
     ln -s /iscratch/i/hg17/linSpecRep.chicken \
           /iscratch/i/hg17/linSpecRep.chimp
     iSync
 
 
 # BLASTZ FUGU (FR1) (DONE 2004-06-24 kate)
     ssh kk
     mkdir -p /cluster/data/hg17/bed/blastz.fr1.2004-06-10
     ln -s /cluster/data/hg17/bed/blastz.fr1.2004-06-10 \
             /cluster/data/hg17/bed/blastz.fr1
     cd /cluster/data/hg17/bed/blastz.fr1
     # Set L=6000 (more relaxed than chicken) and abridge repeats.
     # Treat all repeats as lineage-specific (reuse linSpecRep.Chicken).
     cat << '_EOF_' > DEF
 # human vs. fugu
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin
 
 ALIGN=blastz-run
 BLASTZ=blastz
 
 # Reuse parameters from human-chicken.
 BLASTZ_H=2000
 BLASTZ_Y=3400
 BLASTZ_L=6000
 BLASTZ_K=2200
 BLASTZ_Q=/cluster/data/blastz/HoxD55.q
 BLASTZ_ABRIDGE_REPEATS=1
 
 # TARGET: Human
 SEQ1_DIR=/iscratch/i/hg17/bothMaskedNibs
 SEQ1_RMSK=
 SEQ1_FLAG=
 SEQ1_SMSK=/iscratch/i/hg17/linSpecRep.fugu
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY: Fugu
 SEQ2_DIR=/iscratch/i/fr1/nib
 SEQ2_RMSK=
 SEQ2_FLAG=
 SEQ2_SMSK=/iscratch/i/fr1/linSpecRep
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=10000000
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/blastz.fr1
 
 DEF=$BASE/DEF
 RAW=$BASE/raw
 CDBDIR=$BASE
 SEQ1_LEN=$BASE/S1.len
 SEQ2_LEN=$BASE/S2.len
 '_EOF_'
     # << this line keeps emacs coloring happy
 
     bash # if a csh/tcsh user
     source DEF
     mkdir $RAW run.0
     /cluster/home/angie/hummus/make-joblist $DEF > $BASE/run.0/j
     # GOT HERE
     sh ./xdir.sh
     cd run.0
     sed -e 's@^blastz-run@/cluster/bin/penn/blastz-run@' j > jobList
     para create jobList
         # 11935 jobs
     para try
     para check
     para push
 # Completed: 11935 of 11935 jobs
 # CPU time in finished jobs:    4673316s   77888.60m  1298.14h   54.09d  0.148 y
 # IO & Wait Time:                329249s    5487.48m    91.46h    3.81d  0.010 y
 # Average job time:                 419s       6.99m     0.12h    0.00d
 # Longest job:                      714s      11.90m     0.20h    0.01d
 # Submission to last job:          5575s      92.92m     1.55h    0.06d
 
     # second cluster run: lift raw alignments -> lav dir
     ssh kki
     cd /cluster/data/hg17/bed/blastz.fr1
     bash # if a csh/tcsh user
     source DEF
     mkdir run.1 lav
     /cluster/bin/scripts/blastz-make-out2lav $DEF $BASE > $BASE/run.1/jobList
     cd run.1
     wc -l jobList
     para create jobList
         # 341 jobs
     para try
     para check 
     para push
 # CPU time in finished jobs:        315s       5.26m     0.09h    0.00d  0.000 y
 # IO & Wait Time:                  4451s      74.18m     1.24h    0.05d  0.000 y
 # Average job time:                  14s       0.23m     0.00h    0.00d
 # Longest job:                      107s       1.78m     0.03h    0.00d
 # Submission to last job:           368s       6.13m     0.10h    0.00d
 
     # third run: lav -> axt
     ssh kki
     cd /cluster/data/hg17/bed/blastz.fr1
     mkdir axtChrom pslChrom run.2
     cd run.2
     cat << 'EOF' > do.csh
 #!/bin/csh -ef
 cd $1
 set chr = $1:t
 cat `ls -1 *.lav | sort -g` \
 | lavToAxt stdin \
         /iscratch/i/hg17/bothMaskedNibs /iscratch/i/fr1/nib stdout \
 | axtSort stdin ../../axtChrom/$chr.axt 
 axtToPsl ../../axtChrom/$chr.axt ../../S1.len ../../S2.len \
         ../../pslChrom/$chr.psl
 'EOF'
     # << this line keeps emacs coloring happy
     chmod a+x do.csh
     cp /dev/null jobList
     foreach d (../lav/chr*)
       echo "do.csh $d" >> jobList
     end
     para create jobList
         # 41 jobs
     para try
     para check
     para push
 
 
 # CHAIN FUGU BLASTZ (2004-06-11 kate)
     # Run axtChain on little cluster
     ssh kki
     cd /cluster/data/hg17/bed/blastz.fr1
     mkdir -p axtChain/run1
     cd axtChain/run1
     mkdir out chain
     ls -1S /cluster/data/hg17/bed/blastz.fr1/axtChrom/*.axt \
       > input.lst
     cat << '_EOF_' > gsub
 #LOOP
 doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     # Reuse gap penalties from chicken run.
     cat << '_EOF_' > temp.gap
 tablesize	11
 smallSize	111
 position	1	2	3	11	111	2111	12111	32111	72111	152111	252111
 qGap	325	360	400	450	600	1100	3600	7600	15600	31600	56600
 tGap	325	360	400	450	600	1100	3600	7600	15600	31600	56600
 bothGap	625	660	700	750	900	1400	4000	8000	16000	32000	57000
 '_EOF_'
     # << this line makes emacs coloring happy
     sed 's/  */\t/g' temp.gap > ../../fuguHumanTuned.gap
     rm -f temp.gap
 
     cat << '_EOF_' > doChain
 #!/bin/csh
 axtChain -scoreScheme=/cluster/data/blastz/HoxD55.q \
                       -linearGap=../../fuguHumanTuned.gap \
                       -minScore=5000 $1 \
     /iscratch/i/hg17/bothMaskedNibs \
     /iscratch/i/fr1/nib $2 > $3
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod a+x doChain
     gensub2 input.lst single gsub jobList
     para create jobList
         # 46 jobs
     para try
     para check
     para push
         # 1 crashed job -- chr6_hla_hap1.chain is empty
 # CPU time in finished jobs:        610s      10.16m     0.17h    0.01d  0.000 y
 # IO & Wait Time:                  1644s      27.40m     0.46h    0.02d  0.000 y
 # Average job time:                  50s       0.83m     0.01h    0.00d
 # Longest job:                      233s       3.88m     0.06h    0.00d
 # Submission to last job:           339s       5.65m     0.09h    0.00d
 
     # now on the cluster server, sort chains
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.fr1/axtChain
     chainMergeSort run1/chain/*.chain > all.chain
     chainSplit chain all.chain
     rm run1/chain/*.chain
 
     # Load chains into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.fr1/axtChain/chain
     foreach i (*.chain)
         set c = $i:r
         echo loading $c
         hgLoadChain hg17 ${c}_chainFr1 $i
     end
     featureBits hg16 chainFr1Link
         # 50709290 bases of 2865248791 (1.770%) in intersection
 
 
 # ANCIENT REPEAT TABLE (2004-06-11 kate)
 
     # The netClass operations requires an "ancientRepeat" table in one 
     # of the databases.
     # This is a hand curated table obtained from Arian.
 
     ssh hgwdev
     mkdir -p /cluster/data/hg17/bed/ancientRepeat
     cd /cluster/data/hg17/bed/ancientRepeat
     # mysqldump needs write permission to this directory
     chmod 777 .
     hgsqldump --all --tab=. hg15 ancientRepeat
     chmod 775 .
     hgsql hg17 < ancientRepeat.sql
     echo "LOAD DATA LOCAL INFILE 'ancientRepeat.txt' into table ancientRepeat"\
                 | hgsql hg17
 
 
 # NET FUGU BLASTZ (2004-06-11 kate)
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.fr1/axtChain
     chainPreNet all.chain ../S1.len ../S2.len stdout \
     | chainNet stdin -minSpace=1 ../S1.len ../S2.len stdout /dev/null \
     | netSyntenic stdin noClass.net
 
     # Add classification info using db tables:
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.fr1/axtChain
     netClass noClass.net hg17 fr1 human.net
 
     # Make a 'syntenic' subset:
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.fr1/axtChain
     rm noClass.net
     # Make a 'syntenic' subset of these with
     netFilter -syn human.net > humanSyn.net
 
     # Load the nets into database 
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.fr1/axtChain
     netFilter -minGap=10 human.net |  hgLoadNet hg17 netFr1 stdin
     #netFilter -minGap=10 humanSyn.net | hgLoadNet hg17 netSyntenyFr1 stdin
 
 
 # EXTRACT AXT'S AND MAF'S FROM THE NET (kate)
     # NOTE: Redo 2005-08-16 to fix overlap problem (use 8/05 netToAxt)
 # Replace bad chr5 axtNet and mafNet (2006-01-05 kate) 
     ssh kkstore2
     cd /cluster/data/hg17/bed/blastz.fr1/axtChain
     netSplit human.net humanNet
     mkdir -p ../axtNet ../mafNet
 cat > makeMaf.csh << 'EOF'
     foreach f (humanNet/chr*.net)
         set c = $f:t:r
         echo "axtNet on $c"
         netToAxt humanNet/$c.net chain/$c.chain /cluster/data/hg17/nib /cluster/data/fr1/nib stdout | axtSort stdin ../axtNet/$c.axt
         axtToMaf ../axtNet/$c.axt \
             /cluster/data/hg17/chrom.sizes /cluster/data/fr1/chrom.sizes \
             ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=fr1.
     end
 'EOF'
     csh makeMaf.csh >&! makeMaf.log &
     tail -100f makeMaf.log
     mkdir -p /cluster/bluearc/hg17/mafNet
     cp -rp ../mafNet /cluster/bluearc/hg17/mafNet/fr1
 
 
 # FUGU FR1 DOWNLOADS (DONE 2004-09-17 kate)
 #    REDO axtNet downloads for fix, above (2005-09-12 kate)
 # Replace bad chr5 axtNet and mafNet (2006-01-05 kate) 
 
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.fr1/axtChain
     ln -s all.chain fugu.chain
     mkdir gz
     gzip -c fugu.chain > gz/fugu.chain.gz
     gzip -c human.net > gz/fugu.net.gz
     cd ../axtNet
     nice gzip *.axt
 
     ssh kkstore02
     cd /cluster/data/hg17/bed/blastz.fr1/axtNet
     gzip *.axt
     md5sum *.gz > md5sum.txt
 
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17
     mkdir -p vsFr1
     cd vsFr1
     # Copy and edit README
     cp /cluster/data/hg17/bed/blastz.fr1/axtChain/gz/*.gz .
     md5sum *.gz > md5sum.txt
 
     mv axtNet axtNet.old
     ln -s /cluster/data/hg17/bed/blastz.fr1/axtNet .
 
 
 # PRODUCE FUGU BLAT ALIGNMENT (DONE - 2004-07-07 - Hiram)
 
     # Use masked scaffolds from fr1 assembly (same sequence as
     # previous BlatFugu, however it's repeat and TRF-masked).
 
     ssh kk
     mkdir /cluster/data/hg17/bed/blatFr1
     cd /cluster/data/hg17/bed/blatFr1
     mkdir psl 
     # next time, use N?_?????? (to pick up NG_ contigs)
     foreach f ( `cat /cluster/data/hg17/contig.lst` )
       set c=$f:t:r
       echo $c
       mkdir psl/$c
     end
 
     # create cluster job
     mkdir run
     cd run
     ls -1S /iscratch/i/fugu/trfFa/*.fa > fugu.lst
     ls -1S /scratch/hg/gs.18/build35/maskedContigs/*.fa > human.lst
 cat << 'EOF' > gsub
 #LOOP
 /cluster/bin/i386/blat -mask=lower -qMask=lower -q=dnax -t=dnax {check in line+ $(path1)} {check in line+ $(path2)} {check out line+ /cluster/data/hg17/bed/blatFr1/psl/$(root1)/$(root1)_$(root2).psl}
 #ENDLOOP
 'EOF'
     # << keep emacs happy
     gensub2 human.lst fugu.lst gsub jobList
     para create spec
        # 219640 jobs  
     para try
     para check
     para push -maxQueue=300000 -maxPush=220000
     para check
 # Completed: 219640 of 219640 jobs
 # CPU time in finished jobs:    5206945s   86782.41m  1446.37h   60.27d  0.165 y
 # IO & Wait Time:                797791s   13296.52m   221.61h    9.23d  0.025 y
 # Average job time:                  27s       0.46m     0.01h    0.00d
 # Longest job:                      951s      15.85m     0.26h    0.01d
 # Submission to last job:          7553s     125.88m     2.10h    0.09d
         # cd psl
         # count files with aligments
         # find . -not -size 427c | wc -l
         # 44558
         # count files with no aligments
         # find . -size 427c | wc -l
         # 175463
 
    # When cluster run is done, sort alignments
    # into chrom directory
     ssh eieio
     cd /cluster/data/hg17/bed/blatFr1
     pslCat -dir psl/N?_?????? | \
       liftUp -type=.psl stdout \
         /cluster/data/hg17/jkStuff/liftAll.lft warn stdin | \
       pslSortAcc nohead chrom temp stdin
         # 65 minutes ?
         # Processed 216595 lines into 1 temp files
 
     # Rename to correspond with tables and load into database:
     ssh hgwdev
     cd /cluster/data/hg17/bed/blatFr1/chrom
     foreach i (chr*.psl)
         set r = $i:r
         echo mv $i ${r}_blatFr1.psl
         mv $i ${r}_blatFr1.psl
     end
 
     # lift fugu scaffolds to Fugu browser chrUn,
     # so you can link to other browser.  And don't need to load sequence
     cd /cluster/data/hg17/bed/blatFr1
     liftUp -pslQ all.psl /cluster/data/fr1/fugu_v3.masked.lft warn chrom/*.psl
 
     hgLoadPsl -table=blatFr1 hg17 all.psl
     #	load of blatFr1 did not go as planned: 216595 record(s),
     #	0 row(s) skipped, 3 warning(s) loading psl.tab
     #	featureBits hg17 blatFr1 refGene:CDS
     #	13563544 bases of 2866216770 (0.473%) in intersection
     #	featureBits hg16 blatFr1 refGene:CDS
     #	13547219 bases of 2865248791 (0.473%) in intersection
     #	featureBits hg15 blatFugu refGene:CDS
     #	12427544 bases of 2866466359 (0.434%) in intersection
 
 #  BLASTZ RAT RN3 (DONE - 2004-06-14 - Hiram)
 
     ssh kk
     mkdir -p /cluster/data/hg17/bed/blastz.rn3.2004-06-11
     cd /cluster/data/hg17/bed
     ln -s  blastz.rn3.2004-06-11 blastz.rn3
     cd blastz.rn3
 
     cat << '_EOF_' > DEF
 # rat vs. human
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/home/angie/schwartzbin:/cluster/home/kent/bin/i386
 
 ALIGN=blastz-run
 BLASTZ=blastz
 BLASTZ_H=2000
 BLASTZ_ABRIDGE_REPEATS=1
 
 # TARGET
 # Human
 SEQ1_DIR=/iscratch/i/gs.18/build35/bothMaskedNibs
 # not used
 SEQ1_RMSK=
 # not used
 SEQ1_FLAG=
 SEQ1_SMSK=/iscratch/i/gs.18/build35/linSpecRep.notInRat
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY
 # Rat
 SEQ2_DIR=/iscratch/i/rn3/bothMaskedNibs
 # not currently used
 SEQ2_RMSK=
 # not currently used
 SEQ2_FLAG=
 SEQ2_SMSK=/cluster/bluearc/rat/rn3/linSpecRep.notInHuman
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=30000000
 SEQ2_LAP=0
 
 BASE=/cluster/store5/gs.18/build35/bed/blastz.rn3
 
 DEF=$BASE/DEF
 RAW=$BASE/raw
 CDBDIR=$BASE
 SEQ1_LEN=$BASE/S1.len
 SEQ2_LEN=$BASE/S2.len
 '_EOF_'
     # << this line makes emacs coloring happy
 
     # prepare first cluster run
     ssh kk
     cd /cluster/data/hg17/bed/blastz.rn3
     source DEF
     # script copied over from /cluster/data/mm4/jkStuff/BlastZ_run0.sh
     #	it is a generic script and works for any assembly
     /cluster/data/hg17/jkStuff/BlastZ_run0.sh
     cd run.0
     para try, check, push, check, ....
 Completed: 41943 of 41943 jobs
 CPU time in finished jobs:   15330421s  255507.02m  4258.45h  177.44d  0.486 y
 IO & Wait Time:                673809s   11230.15m   187.17h    7.80d  0.021 y
 Average job time:                 382s       6.36m     0.11h    0.00d
 Longest job:                     4651s      77.52m     1.29h    0.05d
 Submission to last job:        169197s    2819.95m    47.00h    1.96d
 
     #	Second cluster run to convert the .out's to .lav's
     #	You do NOT want to run this on the big cluster.  It brings
     #	the file server to its knees.  Run this on the small cluster.
     ssh kki
     cd /cluster/data/hg17/bed/blastz.rn3
     # script copied over from /cluster/data/mm4/jkStuff/BlastZ_run1.sh
     #	fixup machine check, should be kki, not kk
     /cluster/data/hg17/jkStuff/BlastZ_run1.sh
     cd run.1
     para try, check, push, etc ...
 # Completed: 341 of 341 jobs
 # CPU time in finished jobs:       1894s      31.56m     0.53h    0.02d  0.000 y
 # IO & Wait Time:                  6271s     104.52m     1.74h    0.07d  0.000 y
 # Average job time:                  24s       0.40m     0.01h    0.00d
 # Longest job:                      131s       2.18m     0.04h    0.00d
 # Submission to last job:           590s       9.83m     0.16h    0.01d
 
     #	Third cluster run to convert lav's to axt's
     cd /cluster/data/hg17/bed/blastz.rn3
     #	The copy of this in mm4 was broken, fixed here
     /cluster/data/hg17/jkStuff/BlastZ_run2.sh
     cd run.2
     para try, check, push, etc ...
 # Completed: 46 of 46 jobs
 # CPU time in finished jobs:        426s       7.09m     0.12h    0.00d  0.000 y
 # IO & Wait Time:                  7283s     121.39m     2.02h    0.08d  0.000 y
 # Average job time:                 168s       2.79m     0.05h    0.00d
 # Longest job:                      642s      10.70m     0.18h    0.01d
 # Submission to last job:           642s      10.70m     0.18h    0.01d
 
     # translate sorted axt files into psl
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.rn3
     mkdir pslChrom
     set tbl = "blastzRn3"
     foreach f (axtChrom/chr*.axt)
       set c=$f:t:r
       echo "Processing chr $c"
       /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
     end
     #	That takes about 30 minutes
 
     # Load database tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.rn3/pslChrom
     for I in *.psl
     do
 	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
 	echo "done: ${I}"
     done
     # this is a 55 minute job
     #	Check results
     #	featureBits hg16 blastzRn3
     #	1013603401 bases of 2865248791 (35.376%) in intersection
     #	featureBits hg17 blastzRn3
     #	1013003285 bases of 2866216770 (35.343%) in intersection
 
 # CHAIN RN3 BLASTZ (DONE - 2004-06-14 - Hiram)
 #  re-worked with no 'axtFilter -notQ_random' on the axtChain step - 2004-06-23
 #    used to be: axtFilter -notQ_random $1 | axtChain stdin
 
 # The axtChain is best run on the small kluster, or the kk9 kluster
     ssh kki
     mkdir -p /cluster/data/hg17/bed/blastz.rn3/axtChain/run1
     cd /cluster/data/hg17/bed/blastz.rn3/axtChain/run1
     mkdir out chain
 
     ls -1S /cluster/data/hg17/bed/blastz.rn3/axtChrom/*.axt > input.lst
     cat << '_EOF_' > gsub
 #LOOP
 doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} out/$(root1).out
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     cat << '_EOF_' > doChain
 #!/bin/csh
     axtChain $1 \
 	/iscratch/i/gs.18/build35/bothMaskedNibs \
 	/iscratch/i/rn3/bothMaskedNibs $2 > $3
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod a+x doChain
 
     # 46 jobs
     gensub2 input.lst single gsub jobList
     para create jobList
     para try
     para push # ... etc ...
 # Completed: 46 of 46 jobs
 # CPU time in finished jobs:       4645s      77.41m     1.29h    0.05d  0.000 y
 # IO & Wait Time:                  6840s     114.00m     1.90h    0.08d  0.000 y
 # Average job time:                 250s       4.16m     0.07h    0.00d
 # Longest job:                     1539s      25.65m     0.43h    0.02d
 # Submission to last job:          3761s      62.68m     1.04h    0.04d
 
 
     # now on the file server, sort chains
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.rn3/axtChain
     time chainMergeSort run1/chain/*.chain > all.chain
     #	real    36m42.170s
     #	user    4m55.970s
     #	sys     1m49.840s
 
     time chainSplit chain all.chain
     #	real    13m54.860s
     #	user    4m50.370s
     #	sys     1m3.260s
 
     # optionally: rm run1/chain/*.chain
 
     # Load chains into database
     # next machine
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.rn3/axtChain/chain
     foreach i (*.chain)
         set c = $i:r
         hgLoadChain hg17 ${c}_chainRn3 $i
         echo done $c
     end
     #	featureBits hg17 chainRn3
     #	2827052992 bases of 2866216770 (98.634%) in intersection
     #	(with filter:) 2826192649 bases of 2866216770 (98.604%) in intersection
     #	featureBits hg16 chainRn3
     #	2830563493 bases of 2865248791 (98.789%) in intersection
 
 # NET RN3 (DONE - 2004-06-15 - Hiram)
 #	Re-done due to Chain being re-done 2004-06-23
 #       NOTE: Redo net axt's and net maf's to fix overlaps,
 #       (using 8/05 netToAxt). (2005-08-16 kate)
 
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.rn3/axtChain
     mkdir preNet
     cd chain
     foreach i (*.chain)
       echo preNetting $i
       /cluster/bin/i386/chainPreNet $i /cluster/data/hg17/chrom.sizes \
                         /cluster/data/rn3/chrom.sizes ../preNet/$i
     end
 
     cd ..
     mkdir n1
     cd preNet
     foreach i (*.chain)
       set n = $i:r.net
       echo primary netting $i
       /cluster/bin/i386/chainNet $i -minSpace=1 /cluster/data/hg17/chrom.sizes \
                             /cluster/data/rn3/chrom.sizes ../n1/$n /dev/null
     end
 
     cd ..
     cat n1/*.net | /cluster/bin/i386/netSyntenic stdin hNoClass.net
     #	memory usage 2510467072, utime 19307 s/100, stime 3181
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.rn3/axtChain
     time netClass hNoClass.net hg17 rn3 rat.net \
 	-tNewR=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInRat \
 	-qNewR=/cluster/bluearc/rat/rn3/linSpecRep.notInHuman
     #	real    34m29.829s
     #	user    11m30.440s
     #	sys     1m52.730s
 
     # If things look good do
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.rn3/axtChain
     rm -r n1 hNoClass.net
     # Make a 'syntenic' subset of these with
     time netFilter -syn rat.net > ratSyn.net
     #	real    16m25.640s
     #	user    7m41.330s
     #	sys     1m1.150s
 
     # Load the nets into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.rn3/axtChain
     netFilter -minGap=10 rat.net |  hgLoadNet hg17 netRn3 stdin
     netFilter -minGap=10 ratSyn.net | hgLoadNet hg17 syntenyNetRn3 stdin
     #	real    37m0.199s
     #	user    15m13.770s
     #	sys     1m41.540s
 
     # check results
     # featureBits hg17 netRn3
     # 2817656275 bases of 2866216770 (98.306%) in intersection
     # (with axtFilter) 2816623107 bases of 2866216770 (98.270%) in intersection
     # featureBits hg16 netRn3
     # 2820958389 bases of 2865248791 (98.454%) in intersection
 
     # featureBits hg17 syntenyNetRn3
     # 2781748096 bases of 2866216770 (97.053%) in intersection
     # (with axtFilter) 2780883450 bases of 2866216770 (97.023%) in intersection
     # featureBits hg16 syntenyNetRn3
     # 2784011730 bases of 2865248791 (97.165%) in intersection
 
     # Add entries for net and chain to rat/hg17 trackDb
 
     # make net
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.rn3/axtChain
     mkdir ratNet
     time netSplit rat.net ratNet
     #	real    12m1.478s
     #	user    8m35.050s
     #	sys     1m7.230s
 
     # extract axts from net 
     mkdir ../axtNet ../mafNet
 cat << 'EOF' > makeMaf.csh
     foreach n (ratNet/chr*.net)
 	set c=$n:t:r
         echo $c
 	netToAxt ratNet/$c.net chain/$c.chain \
 		/cluster/data/hg17/nib  /cluster/data/rn3/nib stdout | \
                     axtSort stdin ../axtNet/$c.axt
         axtToMaf ../axtNet/$c.axt \
             /cluster/data/hg17/chrom.sizes /cluster/data/rn3/chrom.sizes \
                 ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=rn3.
     end
 'EOF'
     csh makeMaf.csh >&! makeMaf.log &
     tail -100f makeMaf.log
     mkdir -p /cluster/bluearc/hg17/mafNet
     cp -rp ../mafNet /cluster/bluearc/hg17/mafNet/rn3
 
     ssh hgwdev
     mkdir -p /cluster/data/hg17/bed/blastz.rn3/axtBest
     cd /cluster/data/hg17/bed/blastz.rn3/axtBest
     ln -s ../axtNet/chr*.axt .
 
     # copy net axt's to download area
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.rn3/axtNet
     mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/axtNet
     cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/axtNet
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/axtNet
     gzip *.axt
     # add README.txt file to dir (use previous assembly's copy as template)
 
     #  Convert those axt files to psl
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.rn3
     mkdir pslBest
     foreach a (axtBest/chr*.axt)
 	set c=$a:t:r
 	echo "processing $c.axt -> ${c}_blastzBestRn3.psl"
     /cluster/bin/i386/axtToPsl axtBest/${c}.axt \
 	S1.len S2.len pslBest/${c}_blastzBestRn3.psl
 	echo "Done: ${c}_blastzBestRn3.psl"
     end
 
     # Load tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.rn3/pslBest
     for I in chr*BestRn3.psl
     do
 	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
 	echo "done ${I}"
     done
 
      # check results
     # featureBits hg17 blastzBestRn3
     #	975533772 bases of 2866216770 (34.036%) in intersection
     # (with axtFilter) 970005525 bases of 2866216770 (33.843%) in intersection
     # featureBits hg16 blastzBestRn3
     #	976121391 bases of 2865248791 (34.068%) in intersection
 
     # Make /gbdb links and add them to the axtInfo table:
      mkdir -p /gbdb/hg17/axtBest/Rn3
      cd /gbdb/hg17/axtBest/Rn3
      ln -s /cluster/data/hg17/bed/blastz.rn3/axtNet/chr*.axt .
      cd /cluster/data/hg17/bed/blastz.rn3/axtNet
      rm -f axtInfoInserts.sql
      foreach f (/gbdb/hg17/axtBest/Rn3/chr*.axt)
        set chr=$f:t:r
        echo "INSERT INTO axtInfo (species, alignment, chrom, fileName) \
                 VALUES ('rn3','Blastz Best in Genome','$chr','$f');" \
          >> axtInfoInserts.sql
      end
     hgsql hg17 < ~/kent/src/hg/lib/axtInfo.sql
     #	table axtInfo may already exist, ignore create error.
     hgsql hg17 < axtInfoInserts.sql
 
 
 # MAKING RAT SYNTENY (DONE - 2004-06-30 - Hiram)
 #	Re-Done after above done without the axtFilter
 
 ssh hgwdev
 mkdir /cluster/data/hg17/bed/syntenyRn3
 cd /cluster/data/hg17/bed/syntenyRn3
 
 # Copy all the needed scripts from /cluster/data/hg16/bed/syntenyMm3
 cp -p /cluster/data/hg16/bed/syntenyMm3/*.pl .
 cp -p /cluster/data/hg16/bed/syntenyMm3/*.sh .
 
 ./syntenicBest.pl -db=hg17 -table=blastzBestRn3
 ./smooth.pl
 ./joinsmallgaps.pl
 ./fillgap.pl -db=hg17 -table=blastzBestRn3
 ./synteny2bed.pl
 #	The five commands above
 #	real    196m2.565s
 #	user    0m21.170s
 #	sys     0m4.690s
 
 #	Used to load this in syntenyRn3, but that type is misleading to
 #	the table browser and fails the checkTableCoords check.
 #	Better to use this ensRatMusHom type:
 sed -e 's/ensPhusionBlast/ensRn3MusHom/g' \
       $HOME/kent/src/hg/lib/ensPhusionBlast.sql \
       > ensRn3MusHom.sql
 hgLoadBed hg17 ensRn3MusHom ucsc100k.bed -sqlTable=ensRn3MusHom.sql
 
     #	featureBits hg17 ensRn3MusHom
     #	2592164486 bases of 2866216770 (90.439%) in intersection
     #	featureBits hg16 syntenyRn3
     #	2595919851 bases of 2865248791 (90.600%) in intersection
 
 
 # MAKING RAT AXTTIGHT FROM AXTBEST (DONE - 2004-06-15 - Hiram)
     # After creating axtBest alignments above, use subsetAxt to get axtTight:
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.rn3/axtNet
     mkdir -p ../axtTight
     foreach i (*.axt)
       echo $i
       subsetAxt  $i ../axtTight/$i \
         ~kent/src/hg/mouseStuff/subsetAxt/coding.mat 3400
     end
 
     # translate to psl
     cd ../axtTight
     mkdir ../pslTight
     foreach i (*.axt)
       set c = $i:r
       axtToPsl $i ../S1.len ../S2.len ../pslTight/${c}_blastzTightRn3.psl
       echo "Done: $i"
     end
 
     # Load tables into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.rn3/pslTight
     for I in chr*TightRn3.psl
     do
 	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
 	echo "done ${I}"
     done
 
     #	Compare results with previous assembly
     #	featureBits hg17 blastzTightRn3
     #	153936720 bases of 2866216770 (5.371%) in intersection
     #	featureBits hg16 blastzTightRn3
     #	153151903 bases of 2865248791 (5.345%) in intersection
 
     # copy  axt's to download area
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.rn3/axtTight
     mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/axtTight
     cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/axtTight
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/axtTight
     gzip *.axt
     # add README.txt file to dir (use previous assembly's copy as template)
 
     # REDO downloads with fixed axtNet's (2005=09-13 kate)
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsRn3
     mv axtNet axtNet.old
     nice cp -rp /cluster/data/hg17/bed/blastz.rn3/axtNet .
     cd axtNet
     nice gzip *.axt
     md5sum *.axt.gz > md5sum.txt
 
 
 # BLASTZ RN3 CLEAN UP (DONE - 2004-07-02 - Hiram)
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.rn3
     nice rm -rf raw &
     nice rm axtChain/run1/chain/* &
     nice rm -fr axtChain/n1 axtChain/hNoClass.net &
     nice gzip axtChrom/* pslChrom/* lav/*/* axtChain/all.chain axtChain/*.net &
 
 # BLASTZ CHICKEN (GALGAL2) (DONE - 2004-06-14 - Fan)
 
     ssh kk
     mkdir /cluster/data/hg17/bed/blastz.galGal2.2004-06-14
     cd /cluster/data/hg17/bed
     ln -s /cluster/data/hg17/bed/blastz.galGal2.2004-06-14 blastz.galGal2
     cd blastz.galGal2
     # Set L=10000 (higher threshold on blastz's outer loop) and abridge 
     # repeats.
     cat << '_EOF_' > DEF
 # human vs. chicken
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin
 
 ALIGN=blastz-run
 BLASTZ=blastz
 
 # Specific settings for chicken (per Webb email to Brian Raney)
 BLASTZ_H=2000
 BLASTZ_Y=3400
 BLASTZ_L=10000
 BLASTZ_K=2200
 BLASTZ_Q=/cluster/data/blastz/HoxD55.q
 BLASTZ_ABRIDGE_REPEATS=1
 
 # TARGET: Human
 SEQ1_DIR=/iscratch/i/hg17/bothMaskedNibs
 SEQ1_RMSK=
 SEQ1_FLAG=
 SEQ1_SMSK=/iscratch/i/gs.18/build35/linSpecRep.chicken
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY: Chicken
 SEQ2_DIR=/iscratch/i/galGal2/nib
 SEQ2_RMSK=
 SEQ2_FLAG=
 SEQ2_SMSK=/iscratch/i/galGal2/linSpecRep
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=10000000
 SEQ2_LAP=0
 
 BASE=/cluster/store5/gs.18/build35/bed/blastz.galGal2
 
 DEF=$BASE/DEF
 RAW=$BASE/raw
 CDBDIR=$BASE
 SEQ1_LEN=$BASE/S1.len
 SEQ2_LEN=$BASE/S2.len
 '_EOF_'
     # << this line keeps emacs coloring happy
 
     # prepare first cluster run
     ssh kk
     cd /cluster/data/hg17/bed/blastz.galGal2
     bash
     # script copied over from /cluster/data/mm4/jkStuff/BlastZ_run0.sh
     #	it is a generic script and works for any assembly
     /cluster/data/hg17/jkStuff/BlastZ_run0.sh
     cd run.0
     para try, check, push, check, ....
 Completed: 41943 of 41943 jobs
 CPU time in finished jobs:   15330421s  255507.02m  4258.45h  177.44d  0.486 y
 IO & Wait Time:                673809s   11230.15m   187.17h    7.80d  0.021 y
 Average job time:                 382s       6.36m     0.11h    0.00d
 Longest job:                     4651s      77.52m     1.29h    0.05d
 Submission to last job:        169197s    2819.95m    47.00h    1.96d
 
     #	Second cluster run to convert the .out's to .lav's
     #	You do NOT want to run this on the big cluster.  It brings
     #	the file server to its knees.  Run this on the small cluster.
     ssh kki
     cd /cluster/data/hg17/bed/blastz.galGal2
     bash
     # script copied over from /cluster/data/mm4/jkStuff/BlastZ_run1.sh
     #	fixup machine check, should be kki, not kk
     /cluster/data/hg17/jkStuff/BlastZ_run1.sh
     cd run.1
     para try, check, push, etc ...
 # Completed: 341 of 341 jobs
 # CPU time in finished jobs:       1894s      31.56m     0.53h    0.02d  0.000 y
 # IO & Wait Time:                  6271s     104.52m     1.74h    0.07d  0.000 y
 # Average job time:                  24s       0.40m     0.01h    0.00d
 # Longest job:                      131s       2.18m     0.04h    0.00d
 # Submission to last job:           590s       9.83m     0.16h    0.01d
 
     #	Third cluster run to convert lav's to axt's
     cd /cluster/data/hg17/bed/blastz.galGal2
     #	The copy of this in mm4 was broken, fixed here
     /cluster/data/hg17/jkStuff/BlastZ_run2.sh
     cd run.2
     para try, check, push, etc ...
 # Completed: 46 of 46 jobs
 # CPU time in finished jobs:        426s       7.09m     0.12h    0.00d  0.000 y
 # IO & Wait Time:                  7283s     121.39m     2.02h    0.08d  0.000 y
 # Average job time:                 168s       2.79m     0.05h    0.00d
 # Longest job:                      642s      10.70m     0.18h    0.01d
 # Submission to last job:           642s      10.70m     0.18h    0.01d
 
     # translate sorted axt files into psl
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.galGal2
     mkdir pslChrom
     set tbl = "blastzGalGal2"
     foreach f (axtChrom/chr*.axt)
       set c=$f:t:r
       echo "Processing chr $c"
       /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
     end
     #	That takes about 30 minutes
 
     # Load database tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.galGal2/pslChrom
     bash
     for I in *.psl
     do
         /cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
         echo "done: ${I}"
     done   
 
 
 # GNF ATLAS 2 (DONE - 2004-07-14 - Hiram
     # Align probes from GNF1H chip.
     ssh kk
     cd /cluster/data/hg17/bed
     mkdir -p geneAtlas2/run/psl
     cd geneAtlas2/run
     #	This bluearc/geneAtlas2 directory already exists
     # mkdir -p /cluster/bluearc/geneAtlas2
     # cp /projects/compbio/data/microarray/geneAtlas2/human/gnf1h.fa /cluster/bluearc/geneAtlas2
     ls -1 /scratch/hg/gs.18/build35/maskedContigs > genome.lst
     ls -1 /cluster/bluearc/geneAtlas2/gnf1h.fa > mrna.lst
     cat << '_EOF_' > gsub
 #LOOP
 blat -fine -ooc=/scratch/hg/h/11.ooc  /scratch/hg/gs.18/build35/maskedContigs/$(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     gensub2 genome.lst mrna.lst gsub jobList
     para create jobList
     para try
     para check
     para push
     para time
 # Completed: 380 of 380 jobs
 # CPU time in finished jobs:      10599s     176.65m     2.94h    0.12d  0.000 y
 # IO & Wait Time:                  3893s      64.88m     1.08h    0.05d  0.000 y
 # Average job time:                  38s       0.64m     0.01h    0.00d
 # Longest job:                      649s      10.82m     0.18h    0.01d
 # Submission to last job:           663s      11.05m     0.18h    0.01d
 
     # Do sort, best in genome filter, and convert to chromosome coordinates
     # to create gnf1h.psl.
     pslSort dirs raw.psl tmp psl
     pslReps -minCover=0.3 -minAli=0.95 -nearTop=0.005 raw.psl \
 	contig.psl /dev/null
     #	Processed 80818 alignments
     liftUp ../affyGnf1h.psl ../../../jkStuff/liftAll.lft warn contig.psl
     rm -r contig.psl raw.psl psl
 
     # Load probes and alignments from GNF1H into database.
     ssh hgwdev
     cd /cluster/data/hg17/bed/geneAtlas2
     #	Already symlinked
     # ln -s /projects/compbio/data/microarray/geneAtlas2/human/gnf1h.fa \
     #	/gbdb/hgFixed/affyProbes
     hgLoadPsl hg17 affyGnf1h.psl
     hgLoadSeq hg17 /gbdb/hgFixed/affyProbes/gnf1h.fa
 
     grep -v U133B ../affyUclaNorm/hg17.affyU133AB_all.lifted.pslReps.psl \
 	| sed -e "s/exemplar://; s/consensus://; s/U133A://" \
 	| sed -e "s/;//" > affyU133A.psl
 
     hgMapMicroarray gnfAtlas2.bed hgFixed.gnfHumanAtlas2MedianRatio \
     	affyU133A.psl  /cluster/data/hg17/bed/geneAtlas2/affyGnf1h.psl
 
 # Loaded 44696 rows of expression data from hgFixed.gnfHumanAtlas2MedianRatio
 # Mapped 32857,  multiply-mapped 1462, missed 49, unmapped 11839
 
     hgLoadBed hg17 gnfAtlas2 gnfAtlas2.bed
     #	Loaded 34319 elements of size 15
 
 # LOAD SNPS ( Daryl Thomas; November 7, 2004 ; 
 	      snpExceptions added January 8, 2005 ; 
 	      updated to build 124 on January 13, 2005;
 	      added affy snps March 5, 2004 )
     set db    = hg17
     set org   = human
     set build = 124
     set dir   = /cluster/bluearc/snp/$db/build$build
 
     # ssh to some quiet machine with fast access to the bluearc
     # it takes ~4.5 hours to download the data 
     # (build 124 directly to /cluster/bluearc/... from eieio)
     # Check to make sure the chrMT file is included
     mkdir -p $dir $dir/ds_ch.xml $dir/det $dir/str $dir/loc $dir/seq
     cd $dir
     ln -s /cluster/data/$db/jkStuff/liftAll.lft .
 
     screen
     ftp ftp.ncbi.nih.gov
     cd snp/$org/XML
     prompt
     mget ds_ch*.xml.gz
     exit # screen
     exit # machine
 
     # TODO: check chromStart for each locType
 
     cp -f {$HOME}/kent/src/hg/snp/parseDbSnpXML /cluster/bin/scripts
     chmod 775 /cluster/bin/scripts/parseDbSnpXML
 
     #ssh kk
     touch jobList
     foreach file ( /cluster/bluearc/snp/$db/build$build/ds_ch*.xml.gz )
 	    set out = $file:t:r
 	    echo /cluster/bin/scripts/parseDbSnpXML $file /cluster/bluearc/snp/$db/build$build $out.contig >> jobList
     end
 
     # I removed ds_chMulti.xml.gz and ds_chNotOn.xml.gz from the job list
     # para create jobList; para push; para check ... 
 
 #Completed: 25 of 25 jobs
 #CPU time in finished jobs:      30120s     502.01m     8.37h    0.35d  0.001 y
 #IO & Wait Time:                  2533s      42.21m     0.70h    0.03d  0.000 y
 #Average job time:                1306s      21.77m     0.36h    0.02d
 #Longest job:                     2611s      43.52m     0.73h    0.03d
 #Submission to last job:          2611s      43.52m     0.73h    0.03d
     exit # kk
 
     mv -r $dir /cluster/data/$db/bed/snp/build$build
     set dir = /cluster/data/$db/bed/snp/build$build
     cd $dir
 
     # concatenate the details files to make it easier to lift (and load)
     time zcat det/ds_ch*.xml.contig.det.gz > $db.build$build.contig.bed
     # 33.380u 24.470s 1:54.79 50.3%   0+0k 0+0io 86pf+0w (hgwdev)
     time gzip $db.build$build.contig.bed
     # 251.160u 16.770s 12:40.77 35.2% 0+0k 0+0io 83pf+0w (hgwdev/bluearc - should have done it on eieio/store5)
 
     # some of the NT contigs are not in the liftSpec - this is expected as snps that map to
     # alternate assemblies (Celera) are in the original files, but we disregard their mappings.
     time liftUp $db.build$build.bed liftAll.lft warn $db.build$build.contig.bed.gz 
     # 232.260u 30.050s 5:09.04 84.8%  0+0k 0+0io 379pf+0w (hgwdev/store5)
 
     time gzip hg17.build124.bed                                                                      
     # 141.980u 8.180s 2:34.43 97.2%   0+0k 0+0io 83pf+0w
 
     # hgLoadBed is the important step - check to make sure there are no warnings
     time hgLoadBed $db snp $db.build$build.bed.gz -sqlTable=${HOME}/kent/src/hg/lib/snp.sql
     # Loaded 9131054 elements of size 16
     # 225.040u 37.030s 35:20.45 12.3% 0+0k 0+0io 308pf+0w
 
     # basic snp table is now loaded, but exception column needs to be updated
     # ~ 3 hours wall clock time from here to end
 
     # run queries from snpException.query against snp table
     mkdir -p /usr/local/apache/htdocs/qa/test-results/snpException/build$build
     cd       /usr/local/apache/htdocs/qa/test-results/snpException/build$build
     time snpException hg17 0 ${db}snpException > ${db}snpException.log
     chmod o+rx . 
     chmod o+r  * 
     # 10.610u 19.200s 53:59.98 0.9%   0+0k 0+0io 264pf+0w
 
     # check alignment of flanking sequences
     time snpValid $db /cluster/data/$db/bed/snp/build$build/seq > ${db}snpValid.log
     # 5205.860u 216.570s 1:55:10.27 78.4%     0+0k 0+0io 72408pf+0w (hgwdev)
 
     ### NOTE: the pseudoautosomal snps are reported in the chrX files
     ### only, which causes problems for snpValid when checking the
     ### chrY snp mappings.  I got around this by confirming that all
     ### of the 'missing flank' errors (#23) were in pseudoautosomal
     ### regions and ignoring them.  I manually truncated the
     ### hg17snpException.23.bed file before continuing with the next
     ### step.  This could/should be fixed in the next iteration.
 
     # create list of statements to update the snp table and run them
     time tail +3 ${db}snpException.* | awk '/rs/ {printf "%s\t%d\t%d\n",$4,$2,$5}' | sort -k1,2n > exceptionList.txt
     # ~10 seconds
     time updateExceptionList.pl < exceptionList.txt > updateExceptionList.sql
     # 7.250u 0.390s 0:07.87 97.0%     0+0k 0+0io 337pf+0w
     time hgsql hg17 < updateExceptionList.sql
     # 8.420u 10.370s 11:58.44 2.6%    0+0k 0+0io 413pf+0w build123 (this is mostly a mysql process)
     # 6.550u  9.370s 14:34.17 1.8%    0+0k 0+0io 413pf+0w build124
     # > wc -l build12*/updateExceptionList.sql
     #  387166 build123/updateExceptionList.sql
     #  383759 build124/updateExceptionList.sql
 
     # Add Affy SNPs from new submission 
     #!/bin/csh -fe
     # rm -f log ; date ; ./loadAffySnps.csh > & log ; date ; cat log
 
     set db = hg17
     cd /cluster/data/$db/bed/snp/affy/latest
     touch affy.txt affy.bed Affy.bed bed.tab
     rm -f affy*.txt affy*.bed Affy.bed* bed.tab
 
     # datafile was provided by Valmeekam, Venu [Venu_Valmeekam@affymetrix.com]
     tar xfz affyhg17maps_withstrand_alleles.tgz
     wc -l affy*txt
 
     awk '$1 !~ /^chrom/ {printf("chr%s\t%d\t%d\t%s\t0\t%s\t%s\tunknown\tsnp\tunknown\t0\t0\tunknown\texact\tAffy10K\t0\n",        $1,$2,$3,$4,$6,$7);}' < affy10K.txt         > affy10K.bed
     awk '$1 !~ /^chrom/ {printf("chr%s\t%d\t%d\t%s\t0\t%s\t%s\tunknown\tsnp\tunknown\t0\t0\tunknown\texact\tAffy10Kv2\t0\n",      $1,$2,$3,$4,$6,$7);}' < affy10Kv2.txt       > affy10Kv2.bed
     awk '$1 !~ /^chrom/ {printf("chr%s\t%d\t%d\t%s\t0\t%s\t%s\tunknown\tsnp\tunknown\t0\t0\tunknown\texact\tAffy50K_HindIII\t0\n",$1,$2,$3,$4,$6,$7);}' < affy50K_HindIII.txt > affy50K_HindIII.bed
     awk '$1 !~ /^chrom/ {printf("chr%s\t%d\t%d\t%s\t0\t%s\t%s\tunknown\tsnp\tunknown\t0\t0\tunknown\texact\tAffy50K_XbaI\t0\n",   $1,$2,$3,$4,$6,$7);}' < affy50K_XbaI.txt    > affy50K_XbaI.bed
     
     # this is a temporary kluge to fix some bad input data.
     cat affy*.bed | sed 's/_par//' > Affy.bed
     
     # the source enum for 'dbSnp' is 2; all of the affy* values are higher.
     hgsql $db -e "delete from snp where source > 2 "
     
     hgLoadBed $db snp Affy.bed -oldTable -tab
     
     rm -f affy*.txt affy*.bed bed.tab
     gzip Affy.bed
     
     #mysql> select source, count(*) from hg17.snp group by source;                                                                                                                   
     #+-----------------+----------+
     #| source          | count(*) |
     #+-----------------+----------+
     #| dbSnp           |  9131054 |
     #| Affy10K         |    11344 |
     #| Affy10Kv2       |    10032 |
     #| Affy50K_HindIII |    56859 |
     #| Affy50K_XbaI    |    58494 |
     #+-----------------+----------+
 
     #March 7, 2005: fix pseudoautosomal snps:
     #SNP_A-1606360
     #SNP_A-1606329
     #SNP_A-1666553
     #SNP_A-1715750
     #SNP_A-1726331
     #SNP_A-1685712
     #SNP_A-1735899
     #SNP_A-1726272
     #SNP_A-1660936
     #SNP_A-1662285
     #SNP_A-1680848
     #SNP_A-1671440
     #SNP_A-1719355
     #SNP_A-1716499
     #SNP_A-1643847
     #SNP_A-1646007
     #SNP_A-1715285
     #SNP_A-1657714
     #SNP_A-1725038
     #SNP_A-1713938
     #SNP_A-1708565
     #SNP_A-1510243
     #SNP_A-1510197
     #SNP_A-1606356
 
     delete from snp 
     where  chrom = 'chrY' 
     and    name in ('SNP_A-1606360','SNP_A-1606329','SNP_A-1666553','SNP_A-1715750','SNP_A-1726331','SNP_A-1685712','SNP_A-1735899','SNP_A-1726272','SNP_A-1660936','SNP_A-1662285','SNP_A-1680848','SNP_A-1671440','SNP_A-1719355','SNP_A-1716499','SNP_A-1643847','SNP_A-1646007','SNP_A-1715285','SNP_A-1657714','SNP_A-1725038','SNP_A-1713938','SNP_A-1708565','SNP_A-1510243','SNP_A-1510197','SNP_A-1606356');
 
     update snp 
     set chrom = 'chrX' 
     where name in ('SNP_A-1606360','SNP_A-1606329','SNP_A-1666553','SNP_A-1715750','SNP_A-1726331','SNP_A-1685712','SNP_A-1735899','SNP_A-1726272','SNP_A-1660936','SNP_A-1662285','SNP_A-1680848','SNP_A-1671440','SNP_A-1719355','SNP_A-1716499','SNP_A-1643847','SNP_A-1646007','SNP_A-1715285','SNP_A-1657714','SNP_A-1725038','SNP_A-1713938','SNP_A-1708565','SNP_A-1510243','SNP_A-1510197','SNP_A-1606356');
 
     insert into snp 
 	select  bin, 'chrY' as chrom, chromStart, chromEnd, name, score, strand, 
 		observed, molType, class, valid, avHet, avHetSE, func, locType, source, exception 
 	from    snp 
 	where   name in ('SNP_A-1606360','SNP_A-1606329','SNP_A-1666553','SNP_A-1715750','SNP_A-1726331','SNP_A-1685712','SNP_A-1735899','SNP_A-1726272','SNP_A-1660936','SNP_A-1662285','SNP_A-1680848','SNP_A-1671440','SNP_A-1719355','SNP_A-1716499','SNP_A-1643847','SNP_A-1646007','SNP_A-1715285','SNP_A-1657714','SNP_A-1725038','SNP_A-1713938','SNP_A-1708565','SNP_A-1510243','SNP_A-1510197','SNP_A-1606356');
 
     select chrom, count(*) 
     from snp 
     where name in ('SNP_A-1606360','SNP_A-1606329','SNP_A-1666553','SNP_A-1715750','SNP_A-1726331','SNP_A-1685712','SNP_A-1735899','SNP_A-1726272','SNP_A-1660936','SNP_A-1662285','SNP_A-1680848','SNP_A-1671440','SNP_A-1719355','SNP_A-1716499','SNP_A-1643847','SNP_A-1646007','SNP_A-1715285','SNP_A-1657714','SNP_A-1725038','SNP_A-1713938','SNP_A-1708565','SNP_A-1510243','SNP_A-1510197','SNP_A-1606356') 
     group by chrom;;
 
 
 ## LS-SNP links [load data only] (Daryl Thomas; November 3, 2005)
     # Data from Rachel Karchin in the Andrej Sali lab at UCSF
     # /cluster/data/hg17/bed/lssnp
     hgsql hg17 < ${HOME}/kent/src/hg/lib/lsSnpFunction.sql
     hgsql hg17 < ${HOME}/kent/src/hg/lib/lsSnpStructure.sql
     mysql> load data local infile "snp-human3-function-predictions.txt" into table lsSnpFunction;
     Query OK, 24337 rows affected (1.27 sec)
     mysql> load data local infile "snp-human3-structure-predictions.txt" into table lsSnpStructure;
     Query OK, 34764 rows affected (2.36 sec)
 
 # Tajima's D (DONE -- 2005-09-20 -- Daryl)
 # Data from Chris Carlson in Debbie Nickerson's lab
 # Chris Carlson [csc47<AT>u<DOT>washington<DOT>edu]
 
     # get data from ftp site, unpack in $dir:
     # tar tvfz *gz | more
     # -rw-r--r-- chris/admin 34405061 2005-06-03 13:22:15 AD.SNP.track
     # -rw-r--r-- chris/admin 29869512 2005-06-03 13:22:30 ED.SNP.track
     # -rw-r--r-- chris/admin 27154049 2005-06-03 13:22:41 XD.SNP.track
     # -rw-r--r-- chris/admin 10948753 2005-06-02 21:12:27 AD.tajd.track
     # -rw-r--r-- chris/admin 10928630 2005-06-02 21:12:39 ED.tajd.track
     # -rw-r--r-- chris/admin 10926122 2005-06-02 21:12:51 XD.tajd.track
 
     set db=hg17
     set dir=/cluster/data/$db/bed/tajdpoly/latest
     cd $dir
     
     tar xvfz TajDtracks.tar.gz
 
     mac2unix < AD.SNP.track  | grep -v track | sed 's/1\.02e+08/102000000/;s/8\.8e+07/88000000/;s/1\.5e+07/15000000/' > hg17.tajdSnpAd.bed
     mac2unix < ED.SNP.track  | grep -v track | sed 's/1\.02e+08/102000000/;s/8\.8e+07/88000000/;s/1\.5e+07/15000000/' > hg17.tajdSnpEd.bed
     mac2unix < XD.SNP.track  | grep -v track | sed 's/1\.02e+08/102000000/;s/8\.8e+07/88000000/;s/1\.5e+07/15000000/' > hg17.tajdSnpXd.bed
     mac2unix < AD.tajd.track | grep -v track | awk '{printf"%s\t%s\t%d\t%.3f\n",$1,$2,$3,$4}' > hg17.tajdAd.bedGraph
     mac2unix < ED.tajd.track | grep -v track | awk '{printf"%s\t%s\t%d\t%.3f\n",$1,$2,$3,$4}' > hg17.tajdEd.bedGraph
     mac2unix < XD.tajd.track | grep -v track | awk '{printf"%s\t%s\t%d\t%.3f\n",$1,$2,$3,$4}' > hg17.tajdXd.bedGraph
     
     set chain = /cluster/data/hg17/bed/bedOver/hg17ToHg16.over.chain
 
     foreach pop (Ad Ed Xd)
 	liftOver hg17.tajdSnp$pop.bed   $chain hg16.tajdSnp$pop.bed   hg17ToHg16.tajdSnp$pop.unmapped
 	liftOver hg17.tajd$pop.bedGraph $chain hg16.tajd$pop.bedGraph hg17ToHg16.tajd$pop.unmapped
 	foreach db (hg16 hg17)
 	    hgLoadBed -bedGraph=4 $db tajd$pop    $db.tajd$pop.bedGraph
 	    hgLoadBed             $db tajdSnp$pop $db.tajdSnp$pop.bed
 	end
     end
 
     set where1 = "where t.bin=g.bin and t.chrom=g.chrom and (t.chromStart between g.chromStart and g.chromEnd or t.chromEnd between g.chromStart and g.chromEnd)"
     set where2 = "t, chromInfo c where t.chromStart < 0 or (t.chrom=c.chrom and t.chromEnd > c.size)"
     set list = "as pop, t.chrom, t.chromStart from"
     foreach db (hg16 hg17)
 	rm -f $db.delete.sql
 	touch $db.delete.sql
 	foreach p (Ad Ed Xd SnpAd SnpEd SnpXd)
 	    foreach c (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y)
 		echo "select 'tajd$p' $list tajd${p} t,chr${c}_gap g $where1" | \
 		    hgsql $db | \
 		    grep -v pop | \
 		    awk '{printf "delete from %s where chrom=\"%s\" and chromStart=%d;\n",$1,$2,$3}' \
 		    >> $db.delete.sql
 	    end
 	    echo "select 'tajd$p' $list tajd${p} $where2" | \
 		hgsql $db | \
 		grep -v pop | \
 		awk '{printf "delete from %s where chrom=\"%s\" and chromStart=%d;\n",$1,$2,$3}'\
 		>> $db.delete.sql
 	end
 	hgsql $db < $db.delete.sql
     end
 
 
 # GENE SORTER (AKA: FAMILY BROWSER) (DONE - 2004-06-16 - Hiram)
 # Added knownToU133Plus2 track (2004-10-14)
 #	to be done after knownGene tables are complete from known gene
 #	process.
 #
 # Cluster together various alt-splicing isoforms.
 #	Creates the knownIsoforms and knownCanonical tables
 ssh hgwdev
 mkdir /cluster/data/hg17/bed/geneSorter.2004-06-15
 ln -s /cluster/data/hg17/bed/geneSorter.2004-06-15 \
 	/cluster/data/hg17/bed/geneSorter
 cd /cluster/data/hg17/bed/geneSorter
 hgClusterGenes hg17 knownGene knownIsoforms knownCanonical
 
 # Extract peptides from knownGenes into fasta file
 # and create a blast database out of them.
 mkdir /cluster/data/hg17/bed/geneSorter/blastp
 cd /cluster/data/hg17/bed/geneSorter/blastp
 pepPredToFa hg17 knownGenePep known.faa
 #	You may need to build this binary in src/hg/near/pepPredToFa
 /scratch/blast/formatdb -i known.faa -t known -n known
 #	This command is in /projects/compbio/bin/$MACH/formatdb
 
 # Copy over database to bluearc
 rm -fr /cluster/bluearc/hg17/blastp
 mkdir -p /cluster/bluearc/hg17/blastp
 cp -p /cluster/data/hg17/bed/geneSorter/blastp/known.* \
 	/cluster/bluearc/hg17/blastp
 
 #	Had to pick up a new blastall binary (2004-06-15)
 #	Our old one would no longer run on our systems that have
 #	updated Linux versions
 mkdir /cluster/bluearc/blast229
 cd /cluster/bluearc/blast229
 wget --timestamping \
     ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/2.2.9/blast-2.2.9-ia32-linux.tar.gz
 wget --timestamping \
     ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/2.2.9/ChangeLog.txt
 wget --timestamping \
     ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/2.2.9/ReleaseNotes.txt
 tar xvzf blast-2.2.9-ia32-linux.tar.gz
 
 
 # Split up fasta file into bite sized chunks for cluster
 cd /cluster/data/hg17/bed/geneSorter/blastp
 mkdir split
 faSplit sequence known.faa 8000 split/kg
 
 # Make parasol run directory
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/self
 cd /cluster/data/hg17/bed/geneSorter/blastp/self
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 	-p blastp -d /cluster/bluearc/hg17/blastp/known -i $1 -o $2 \
 	-e 0.01 -m 8 -b 1000
 '_EOF_'
     # << keep emacs happy
 chmod +x blastSome
 
 # Make gensub2 file
 cat  << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #	'ls ../../split/*.fa' is too much, hence the echo
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 # Wait a couple of minutes, and do a para check,  if all is good
 # then do a
 para push
 # This should finish in ~15 minutes if the cluster is free.
 Completed: 7749 of 7749 jobs
 CPU time in finished jobs:     182148s    3035.81m    50.60h    2.11d  0.006 y
 IO & Wait Time:                 22954s     382.56m     6.38h    0.27d  0.001 y
 Average job time:                  26s       0.44m     0.01h    0.00d
 Longest job:                      372s       6.20m     0.10h    0.00d
 Submission to last job:           871s      14.52m     0.24h    0.01d
 
 # Load into database.  This takes about 30 minutes
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/self/run/out
 time hgLoadBlastTab hg17 knownBlastTab *.tab
 # Scanning through 7749 files
 # Loading database with 11799667 rows
 #	Hg16 was:       11376875 rows
 # real    30m10.761s
 # user    5m25.490s
 # sys     1m0.630s
 
 cd /cluster/data/hg17/bed/geneSorter
 # Create table that maps between known genes and RefSeq
 hgMapToGene hg17 refGene knownGene knownToRefSeq
 #	may need to build this command in src/hg/near/hgMapToGene
 #	hgsql -e "select count(*) from knownToRefSeq;" hg17
 #	row count changed from 36078 in Hg16 to 36082
 
 # Create table that maps between known genes and LocusLink
 hgsql --skip-column-names -e "select mrnaAcc,locusLinkId from refLink" hg17 \
 	> refToLl.txt
 hgMapToGene hg17 refGene knownGene knownToLocusLink -lookup=refToLl.txt
 #	hgsql -e "select count(*) from knownToLocusLink;" hg17
 #	row count went from 36078 in Hg16 to 36082
 
 # Create table that maps between known genes and Pfam domains
 hgMapViaSwissProt hg17 knownGene name proteinID Pfam knownToPfam
 #	hgsql -e "select count(*) from knownToPfam;" hg17
 #	row count dropped from 30467 in Hg16 to 29725
 
 
 # Create table to map between known genes and GNF Atlas2
 # expression data.
     hgMapToGene hg17 gnfAtlas2 knownGene knownToGnfAtlas2 '-type=bed 12'
 #	hgsql -e "select count(*) from knownToGnfAtlas2;" hg17
 #	row count droppted from 35817 in Hg16 to 35739
 
 # Create expression distance table - takes about an hour
     hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
     	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
 	-lookup=knownToGnfAtlas2
 # Got 35739 unique elements in hgFixed.gnfHumanAtlas2MedianRatio
     #	hgsql -e "select count(*) from gnfAtlas2Distance;" hg17
     #	row count went from 35,817,000 in Hg16 to 35,739,000
     #	real    108m1.671s
     #	user    89m30.680s
     #	sys     3m6.800s
 
 # Create a table that maps between known genes and 
 # the nice affy expression data.
 hgMapToGene "-type=bed 12" hg17 affyUclaNorm knownGene knownToU133
     #	hgsql -e "select count(*) from knownToU133;" hg17
     #	row count went from 37,634 in Hg16 to 36,795
 
 # Create expression distance table.  This will take about 2.5 hours
 cd /tmp
 cp -p ~/kent/src/hg/near/hgExpDistance/affyUcla.weight .
 time hgExpDistance hg17 affyUclaNorm affyUclaExp knownExpDistance \
 	-weights=affyUcla.weight -lookup=knownToU133
     #	211 genes, 42 weights, 26.500000 total wieght
     #	Got 36795 unique elements in affyUclaNorm
     #	real    154m1.058s
     #	user    134m45.000s
     #	sys     3m1.990s
 
 # Create table that maps between known genes and 
 # the GNF data.
 cd /tmp
 hgMapToGene hg17 affyU95 knownGene knownToU95
     #	row count went from 18780 in Hg16 to 18796
     #	hgFixed.gnfHumanU95Exps argument is unused, no need to exist
 hgExpDistance hg17 hgFixed.gnfHumanU95MedianRatio \
 	hgFixed.gnfHumanU95Exps gnfU95Distance  -lookup=knownToU95
     #	row count went from 17711000 in Hg16 to 17710000
     #	real    21m37.703s
     #	user    13m35.110s
     #	sys     0m28.470s
 
 # Create known gene mapping table and expression distance tables
 # for GNF Atlas 2.  (The hgExpDistance takes only 10 minutes.)
 
 hgMapToGene hg17 affyGnf1h knownGene knownToGnf1h
 hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
 	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
 	-lookup=knownToGnf1h
 # Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
 # Got 9756 unique elements in hgFixed.gnfHumanAtlas2MedianRatio
 
 # create table mapping knownGenes to affyU133Plus2 table (2004-10-14, hartera)
 cd /cluster/data/hg17/bed/geneSorter
 hgMapToGene hg17 affyU133Plus2 knownGene knownToU133Plus2
 
 # Make sure that GO database is up to date.
 See README in /cluster/store1/geneOntology.
 #	I update this GO database very carefully, checking that all
 #	structures in it remain the same from release to release and
 #	backing up the current go DB in a backup database.  In this case
 #	the backup is go040107 - when it was loaded for Mm4, and the new
 #	go database is based on data from Dec 17th 2003 and Feb 2004 according
 #	to the time stamp on the fetched data.  This build was done in
 #	/cluster/store1/geneOntology/20040217
 
 cd /cluster/data/hg17/bed/geneSorter
 
 XXX - DO NOT YET HAVE ensGene table - must wait on Ensembl to release that
 XXX - have not created the knownToEnsembl table yet - 2004-07-15 - Hiram
 # Create knownToEnsembl column
 hgMapToGene hg17 ensGene knownGene knownToEnsembl
 #	table row count went from previous version: 36068 to 38251
 
 # Make knownToCdsSnp table (DONE Nov 11, 2004, Heather)
   ssh hgwdev
   nice hgMapToGene hg17 snp knownGene knownToCdsSnp -all -cds
 # row count 165728
 # unique 34013
 # approx. 5 minutes running time
 
 # Make C. elegans ortholog column using blastp on wormpep.
 # First make C. elegans protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This is already done, see makeMm3.doc for procedure
 #	the directory: /cluster/bluearc/ce1/blastp should have data
 
 #	The blast jobs below can be run on the kk or kk9 clusters
 # Create the ceBlastTab
 ssh kk9
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/ce1
 cd /cluster/data/hg17/bed/geneSorter/blastp/ce1
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 	-p blastp -d /cluster/bluearc/ce1/blastp/wormPep \
 	-i $1 -o $2 -e 0.01 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 #	Only takes 10 minutes on an idle cluster
 # Completed: 7749 of 7749 jobs
 # CPU time in finished jobs:      32023s     533.72m     8.90h    0.37d  0.001 y
 # IO & Wait Time:                 20643s     344.05m     5.73h    0.24d  0.001 y
 # Average job time:                   7s       0.11m     0.00h    0.00d
 # Longest job:                      110s       1.83m     0.03h    0.00d
 # Submission to last job:          1911s      31.85m     0.53h    0.02d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/ce1/run/out
 hgLoadBlastTab hg17 ceBlastTab -maxPer=1 *.tab
 #	row count went from 27620 to 27616
 
 # Make mouse ortholog column using blastp on mouse known genes.
 # First make mouse protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This already exists.  See makeMm5.doc for procedure
 #	the directory: /cluster/bluearc/scratch/mus/mm5/blastp should have data
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/mm5
 cd /cluster/data/hg17/bed/geneSorter/blastp/mm5
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 	-p blastp -d /cluster/bluearc/scratch/mus/mm5/blastp/known \
 	-i $1 -o $2 -e 0.001 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #	this echo trick is used because otherwise the command line is
 #	too long and you can not do a simple ls
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7749 of 7749 jobs
 # CPU time in finished jobs:     139041s    2317.34m    38.62h    1.61d  0.004 y
 # IO & Wait Time:                 21227s     353.79m     5.90h    0.25d  0.001 y
 # Average job time:                  21s       0.34m     0.01h    0.00d
 # Longest job:                      260s       4.33m     0.07h    0.00d
 # Submission to last job:          1137s      18.95m     0.32h    0.01d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/mm5/run/out
 hgLoadBlastTab hg17 mmBlastTab -maxPer=1 *.tab
 # Scanning through 7748 files
 #	row count went from 36471 to 36638
 
 # Make rat ortholog column using blastp on rat known genes.
 # First make rat protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This already exists.  See makeRn3.doc for procedure.
 #	Files were put in this directory: /cluster/bluearc/rn3/blastp/
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/rn3
 cd /cluster/data/hg17/bed/geneSorter/blastp/rn3
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
     -p blastp -d /cluster/bluearc/rn3/blastp/known \
     -i $1 -o $2 -e 0.001 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #	this echo trick is used because otherwise the command line is
 #	too long and you can not do a simple ls
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 #Completed: 7749 of 7749 jobs
 #CPU time in finished jobs:      31035s     517.25m     8.62h    0.36d  0.001 y
 #IO & Wait Time:                 38472s     641.20m    10.69h    0.45d  0.001 y
 #Average job time:                   9s       0.15m     0.00h    0.00d
 #Longest job:                       75s       1.25m     0.02h    0.00d
 #Submission to last job:           169s       2.82m     0.05h    0.00d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/rn3/run/out
 hgLoadBlastTab hg17 rnBlastTab -maxPer=1 *.tab
 # Scanning through 7749 files
 #Loading database with 25574 rows
 
 # Make Danio rerio (zebrafish) ortholog column using blastp on Ensembl.
 # First make protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This is already done, see makeMm3.doc for procedure
 #	the directory: /cluster/bluearc/dr1/blastp should have data
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/dr1
 cd /cluster/data/hg17/bed/geneSorter/blastp/dr1
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 	-p blastp -d /cluster/bluearc/dr1/blastp/ensembl \
 	-i $1 -o $2 -e 0.005 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7749 of 7749 jobs
 # CPU time in finished jobs:     100217s    1670.28m    27.84h    1.16d  0.003 y
 # IO & Wait Time:                 23697s     394.95m     6.58h    0.27d  0.001 y
 # Average job time:                  16s       0.27m     0.00h    0.00d
 # Longest job:                      233s       3.88m     0.06h    0.00d
 # Submission to last job:          1667s      27.78m     0.46h    0.02d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/dr1/run/out
 hgLoadBlastTab hg17 drBlastTab -maxPer=1 *.tab
 #	row count went from 32971 to 33023
 
 # Make Saccharomyces cerevisiae (yeast) ortholog column using blastp on RefSeq.
 # First make protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This is already done, see makeMm3.doc for procedure
 #	the directory: /cluster/bluearc/sc1/blastp should have data
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/sc1
 cd /cluster/data/hg17/bed/geneSorter/blastp/sc1
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 	-p blastp -d /cluster/bluearc/sc1/blastp/sgd \
 	-i $1 -o $2 -e 0.01 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7749 of 7749 jobs
 # CPU time in finished jobs:      20738s     345.64m     5.76h    0.24d  0.001 y
 # IO & Wait Time:                 22018s     366.96m     6.12h    0.25d  0.001 y
 # Average job time:                   6s       0.09m     0.00h    0.00d
 # Longest job:                       39s       0.65m     0.01h    0.00d
 # Submission to last job:           572s       9.53m     0.16h    0.01d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/sc1/run/out
 hgLoadBlastTab hg17 scBlastTab -maxPer=1 *.tab
 #	row count went from 18286 to 18265
 
 # Make Drosophila melanagaster ortholog column using blastp on FlyBase.
 # First make SwissProt protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This is already done, see makeMm3.doc for procedure
 #	the directory: /cluster/bluearc/dm1/blastp should have data
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/dm1
 cd /cluster/data/hg17/bed/geneSorter/blastp/dm1
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 	-p blastp -d /cluster/bluearc/dm1/blastp/flyBase \
 	-i $1 -o $2 -e 0.01 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7749 of 7749 jobs
 # CPU time in finished jobs:      82022s    1367.03m    22.78h    0.95d  0.003 y
 # IO & Wait Time:                 21982s     366.37m     6.11h    0.25d  0.001 y
 # Average job time:                  13s       0.22m     0.00h    0.00d
 # Longest job:                      174s       2.90m     0.05h    0.00d
 # Submission to last job:          1439s      23.98m     0.40h    0.02d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/dm1/run/out
 hgLoadBlastTab hg17 dmBlastTab -maxPer=1 *.tab
 #	row count went from 29322 to 29341
 
 ####  Blat knownGene proteins to determine exons (braney 2004-06-20 DONE)
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mkdir blat.hg17KG.2004-06-20
     rm blat.hg17KG
     ln -s  blat.hg17KG.2004-06-20 blat.hg17KG
     cd blat.hg17KG
     pepPredToFa hg17 knownGenePep known.fa
     hgPepPred hg17 generic blastKGPep00 known.fa
     grep ">" known.fa | sed "s/>//" > kgName.lst
     kgName hg17 kgName.lst blastKGRef00
     hgsql hg17 < ~/kent/src/lib/hg/blastRef.sql
     echo "rename table blastRef to blastKGRef00" | hgsql hg17
     echo "load data local infile 'blastKGRef00' into table blastKGRef00" | hgsql hg17
     ssh kk
     cd /cluster/data/hg17/bed/blat.hg17KG
     cat << '_EOF_' > blatSome
 #!/bin/csh -fe
 /cluster/bin/i386/blat -t=dnax -q=prot -out=pslx $1 $2 $3
 '_EOF_'
     # << keep emacs happy
     chmod +x blatSome
     ls -1S /scratch/hg/gs.18/build35/bothMaskedNibs/*.nib > human.lst
     mkdir kgfa
     cd kgfa
     faSplit sequence ../known.fa 3000 kg
     cd ..
     ls -1S kgfa/*.fa > kg.lst
     cat << '_EOF_' > blatGsub
 #LOOP
 blatSome $(path1) {check in line $(path2)} {check out line psl/$(root1)/$(root2).psl}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
     gensub2 human.lst kg.lst blatGsub blatSpec
     mkdir psl
     cd psl
     foreach i (`cat ../human.lst`)
 	mkdir `basename $i .nib`
     end
     cd ..
     para create blatSpec
     para push
 
 # Completed: 133676 of 133676 jobs
 # CPU time in finished jobs:   29661130s  494352.16m  8239.20h  343.30d  0.941 y
 # IO & Wait Time:               2181179s   36352.99m   605.88h   25.25d  0.069 y
 # Average job time:                 238s       3.97m     0.07h    0.00d
 # Longest job:                   105972s    1766.20m    29.44h    1.23d
 
     ssh eieio
     cd /cluster/data/hg17/bed/blat.hg17KG
     pslSort dirs raw.psl /tmp psl/*
     pslReps -nohead -minCover=0.9 -minAli=0.9 raw.psl cooked.psl /dev/null
     pslUniq cooked.psl hg17KG.psl
     pslxToFa hg17KG.psl hg17KG_ex.fa -liftTarget=genome.lft -liftQuery=protein.lft
 
 # BLASTZ MM4 (DONE - 2004-06-22 - Hiram)
     ssh kk
     mkdir -p /cluster/data/hg17/bed/blastz.mm4.2004-06-21
     cd /cluster/data/hg17/bed
     ln -s  blastz.mm4.2004-06-21 blastz.mm4
     cd blastz.mm4
 
     cat << '_EOF_' > DEF
 # human vs. mouse
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/home/angie/schwartzbin:/cluster/home/kent/bin/i386
 
 ALIGN=blastz-run
 BLASTZ=blastz
 BLASTZ_H=2000
 BLASTZ_ABRIDGE_REPEATS=1
 
 # TARGET
 # Human
 SEQ1_DIR=/iscratch/i/gs.18/build35/bothMaskedNibs
 # not used
 SEQ1_RMSK=
 # not used
 SEQ1_FLAG=
 SEQ1_SMSK=/iscratch/i/gs.18/build35/linSpecRep.notInRat
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY
 # Mouse
 SEQ2_DIR=/scratch/mus/mm4/softNib
 # RMSK not currently used
 SEQ2_RMSK=/scratch/mus/mm4/rmsk
 # FLAG not currently used
 SEQ2_FLAG=-rodent
 SEQ2_SMSK=/scratch/mus/mm4/linSpecRep.notInHuman
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=30000000
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/blastz.mm4
 
 DEF=$BASE/DEF
 RAW=$BASE/raw
 CDBDIR=$BASE
 SEQ1_LEN=$BASE/S1.len
 SEQ2_LEN=$BASE/S2.len
 '_EOF_'
     # << this line keeps emacs coloring happy
 
     # prepare first cluster run
     ssh kk
     cd /cluster/data/hg17/bed/blastz.mm4
     /cluster/data/hg17/jkStuff/BlastZ_run0.sh
     cd run.0
     para try, check, push, check, ....
 # Completed: 43648 of 43648 jobs
 # CPU time in finished jobs:   16448001s  274133.36m  4568.89h  190.37d  0.522 y
 # IO & Wait Time:                751666s   12527.76m   208.80h    8.70d  0.024 y
 # Average job time:                 394s       6.57m     0.11h    0.00d
 # Longest job:                     8323s     138.72m     2.31h    0.10d
 # Submission to last job:         44244s     737.40m    12.29h    0.51d
 
     #	the file server to its knees.  Run this on the small cluster.
     ssh kki
     cd /cluster/data/hg17/bed/blastz.mm4
     /cluster/data/hg17/jkStuff/BlastZ_run1.sh
     cd run.1
     para try, check, push, etc ...
 # Completed: 341 of 341 jobs
 # CPU time in finished jobs:       3925s      65.42m     1.09h    0.05d  0.000 y
 # IO & Wait Time:                  6208s     103.46m     1.72h    0.07d  0.000 y
 # Average job time:                  30s       0.50m     0.01h    0.00d
 # Longest job:                      289s       4.82m     0.08h    0.00d
 # Submission to last job:          2800s      46.67m     0.78h    0.03d
 
     #	Third cluster run to convert lav's to axt's
     #	Does not work on kki since /scratch on the iservers is not the
     #	same as /scratch on the other clusters.
     ssh kk
     cd /cluster/data/hg17/bed/blastz.mm4
     /cluster/data/hg17/jkStuff/BlastZ_run2.sh
     cd run.2
     para try, check, push, etc ...
 # Completed: 45 of 46 jobs
 # Crashed: 1 jobs
 # CPU time in finished jobs:       2389s      39.82m     0.66h    0.03d  0.000 y
 # IO & Wait Time:                 13374s     222.90m     3.71h    0.15d  0.000 y
 # Average job time:                 350s       5.84m     0.10h    0.00d
 # Longest job:                     1426s      23.77m     0.40h    0.02d
 # Submission to last job:          1440s      24.00m     0.40h    0.02d
 
     #	chr19 failing due to out of memory.  Run this job individually
     #	on kolossus, adjusting the location of the nib directories:
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.mm4
     sed -e "s/i386/x86_64/g" /cluster/bin/scripts/blastz-chromlav2axt > \
 	x86_64-chromlav2axt
     chmod +x x86_64-chromlav2axt
     time ./x86_64-chromlav2axt \
 	/cluster/data/hg17/bed/blastz.mm4/lav/chr19 \
 	/cluster/data/hg17/bed/blastz.mm4/axtChrom/chr19.axt \
 	/cluster/bluearc/scratch/hg/gs.18/build35/bothMaskedNibs \
 	/cluster/bluearc/scratch/mus/mm4/softNib
     #	real    24m28.955s
     #	user    6m40.990s
     #	sys     1m16.500s
 
     # translate sorted axt files into psl
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm4
     mkdir -p pslChrom
     set tbl = "blastzMm4"
     foreach f (axtChrom/chr*.axt)
       set c=$f:t:r
       echo "Processing chr $c"
       /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
     end
     #	This takes more than an hour.  You can shorten this by changing
     #	that command to a simple echo, put the results into a file,
     #	split the file into four parts and run the four files as shell
     #	scripts on eieio to have four processes running at the same
     #	time.  Load on eieio gets up to about 20 which is reasonable.
 
     # Load database tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm4/pslChrom
     bash
     for F in chr*_blastzMm4.psl
     do
 	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${F}
 	echo "${F} done"
     done
     # this is a 55 minute job
     # exit bash if you are tcsh
 
     # featureBits on blastzMm3 or 4 will not work on hgwdev, runs out of
     # memory.  But if you reset your ~/.hg.conf to use the read-only
     #	user and contact the hgwdev host, then use the x86_64 featureBits
     # featureBits hg16 blastzMm4
     # 1056761609 bases of 2865248791 (36.882%) in intersection
     # featureBits hg17 blastzMm4
     # 1056201417 bases of 2866216770 (36.850%) in intersection
 
 # CHAIN MM4 BLASTZ (DONE - 2004-06-29 - Hiram)
 # redone with the 'axtFilter -notQ_random' removed - 2004-06-23
 
 # The axtChain is best run on the small kluster, or the kk9 kluster
     ssh kk9
     mkdir -p /cluster/data/hg17/bed/blastz.mm4/axtChain/run1
     cd /cluster/data/hg17/bed/blastz.mm4/axtChain/run1
     mkdir out chain
 
     ls -1S /cluster/data/hg17/bed/blastz.mm4/axtChrom/*.axt > input.lst
     cat << '_EOF_' > gsub
 #LOOP
 doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} out/$(root1).out
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
 #  May need -minScore=5000 for all chroms if chr19 won't finish on kolossus
 
     cat << '_EOF_' > doChain
 #!/bin/csh
 axtChain $1 /iscratch/i/gs.18/build35/bothMaskedNibs \
 	/iscratch/i/mm4/softNib $2 > $3
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod a+x doChain
 
     # 46 jobs
     gensub2 input.lst single gsub jobList
     para create jobList
     para try
     para push # ... etc ...
 # Completed: 45 of 46 jobs
 # CPU time in finished jobs:       6575s     109.58m     1.83h    0.08d  0.000 y
 # IO & Wait Time:                  9274s     154.57m     2.58h    0.11d  0.000 y
 # Average job time:                 352s       5.87m     0.10h    0.00d
 # Longest job:                     3121s      52.02m     0.87h    0.04d
 # Submission to last job:          3121s      52.02m     0.87h    0.04d
     #	one job wouldn't finish due to memory usage
     #	run the chr19 job on kolossus, takes an hour, gets up to 4 Gb
     #	memory usage
 
     # now on the file server, sort chains
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm4/axtChain
     time chainMergeSort run1/chain/*.chain > all.chain
     #	real    17m17.639s
     #	user    9m54.240s
     #	sys     1m31.210s
     #	(1.9 Gb result file !)
 
     time chainSplit chain all.chain
     #	real    27m32.278s
     #	user    9m46.970s
     #	sys     2m45.960s
 
     # optionally: rm run1/chain/*.chain
 
     # Load chains into database
     # next machine
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm4/axtChain/chain
     foreach i (*.chain)
         set c = $i:r
         hgLoadChain hg17 ${c}_chainMm4 $i
         echo done $c
     end
 
     #	featureBits hg17 chainMm4
     #	2829135227 bases of 2866216770 (98.706%) in intersection
     #	featureBits hg16 chainMm4
     #	2828363353 bases of 2865248791 (98.713%) in intersection
 
 # NET MM4 (DONE - 2004-06-29 - Hiram)
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm4/axtChain
     mkdir preNet
     cd chain
     foreach i (*.chain)
       echo preNetting $i
       /cluster/bin/i386/chainPreNet $i /cluster/data/hg17/chrom.sizes \
                         /cluster/data/mm4/chrom.sizes ../preNet/$i
     end
 
     cd ..
     mkdir n1
     cd preNet
     foreach i (*.chain)
       set n = $i:r.net
       echo primary netting $i
       /cluster/bin/i386/chainNet $i -minSpace=1 /cluster/data/hg17/chrom.sizes \
 	/cluster/data/mm4/chrom.sizes ../n1/$n /dev/null
     end
 
     cd ..
     cat n1/*.net | /cluster/bin/i386/netSyntenic stdin hNoClass.net
     #	memory usage 2504171520, utime 19373 s/100, stime 5906
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm4/axtChain
     time netClass hNoClass.net hg17 mm4 mouse.net \
 	-tNewR=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInMouse \
 	-qNewR=/cluster/bluearc/scratch/mus/mm4/linSpecRep.notInHuman
     #	real    19m33.421s
     #	user    10m37.130s
     #	sys     1m45.630s
 
     # If things look good do
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm4/axtChain
     rm -r n1 hNoClass.net
     # Make a 'syntenic' subset of these with
     time netFilter -syn mouse.net > mouseSyn.net
     #	real    13m24.885s
     #	user    7m37.100s
     #	sys     1m5.760s
 
     # Load the nets into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm4/axtChain
     netFilter -minGap=10 mouse.net |  hgLoadNet hg17 netMm4 stdin
     netFilter -minGap=10 mouseSyn.net | hgLoadNet hg17 syntenyNetMm4 stdin
     #	real    44m20.735s
     #	user    15m58.620s
     #	sys     1m58.720s
     # check results
     # featureBits hg17 netMm4
     #	2824272033 bases of 2866216770 (98.537%) in intersection
     # featureBits hg16 netMm4
     #	2823565051 bases of 2865248791 (98.545%) in intersection
 
     # featureBits hg17 syntenyNetMm4
     #	2785830955 bases of 2866216770 (97.195%) in intersection
     # featureBits hg16 syntenyNetMm4
     #	2786960572 bases of 2865248791 (97.268%) in intersection
 
     # Add entries for net and chain to mouse/hg17 trackDb
 
     # make net
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm4/axtChain
     mkdir mouseNet
     time netSplit mouse.net mouseNet
     #	real    12m1.478s
     #	user    8m35.050s
     #	sys     1m7.230s
 
     #	extract axt's from net, and convert to maf's (DONE - Kate - 2004-06-24)
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm4/axtChain
     mkdir ../axtNet ../mafNet
 cat > makeMaf.csh << '_EOF_'
     foreach f (mouseNet/chr*.net)
         set c = $f:t:r
         echo "netToAxt: $c.net -> $c.axt"
         rm -f ../axtNet/$c.axt
         netToAxt mouseNet/$c.net chain/$c.chain \
 	    /cluster/data/hg17/nib /cluster/data/mm4/nib stdout | \
 	    axtSort stdin ../axtNet/$c.axt
         axtToMaf ../axtNet/$c.axt \
             /cluster/data/hg17/chrom.sizes /cluster/data/mm4/chrom.sizes \
             ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=mm4.
 	echo "Complete: $c.net -> axtNet/$c.axt -> mafNet/$c.maf"
     end
 '_EOF_'
 # << for emacs
     csh makeMaf.csh >&! makeMaf.log &
     tail -100f makeMaf.log
 
     ssh hgwdev
     mkdir -p /cluster/data/hg17/bed/blastz.mm4/axtBest
     cd /cluster/data/hg17/bed/blastz.mm4/axtBest
     ln -s ../axtNet/chr*.axt .
 
     # copy net axt's to download area
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm4/axtNet
     mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/vsMm4/axtNet
     cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsMm4/axtNet
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsMm4/axtNet
     gzip *.axt
     # add README.txt file to dir (use previous assembly's copy as template)
 
     #  Convert those axt files to psl
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm4
     mkdir pslBest
     foreach a (axtBest/chr*.axt)
 	set c=$a:t:r
 	echo "processing $c.axt -> ${c}_blastzBestMm4.psl"
     /cluster/bin/i386/axtToPsl axtBest/${c}.axt \
 	S1.len S2.len pslBest/${c}_blastzBestMm4.psl
 	echo "Done: ${c}_blastzBestMm4.psl"
     end
 
     # Load tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm4/pslBest
     for I in chr*BestMm4.psl
     do
 	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
 	echo "done ${I}"
     done
 
      # check results
     # featureBits hg17 blastzBestMm4
     #	1017319919 bases of 2866216770 (35.493%) in intersection
     # featureBits hg16 blastzBestMm4
     #	996722004 bases of 2865248791 (34.787%) in intersection
 
     # Make /gbdb links and add them to the axtInfo table:
      mkdir -p /gbdb/hg17/axtBest/Mm4
      cd /gbdb/hg17/axtBest/Mm4
      ln -s /cluster/data/hg17/bed/blastz.mm4/axtNet/chr*.axt .
      cd /cluster/data/hg17/bed/blastz.mm4/axtNet
      rm -f axtInfoInserts.sql
      foreach f (/gbdb/hg17/axtBest/Mm4/chr*.axt)
        set chr=$f:t:r
        echo "INSERT INTO axtInfo (species, alignment, chrom, fileName) \
                 VALUES ('mm4','Blastz Best in Genome','$chr','$f');" \
          >> axtInfoInserts.sql
      end
     hgsql hg17 < ~/kent/src/hg/lib/axtInfo.sql
     #	table axtInfo may already exist, ignore create error.
     hgsql hg17 < axtInfoInserts.sql
 
 # MAKING MOUSE SYNTENY (DONE - 2004-06-29 - Hiram)
 
 ssh hgwdev
 mkdir /cluster/data/hg17/bed/syntenyMm4
 cd /cluster/data/hg17/bed/syntenyMm4
 
 # Copy all the needed scripts from /cluster/data/hg16/bed/syntenyMm3
 cp -p /cluster/data/hg17/bed/syntenyRn3/*.pl .
 
 ./syntenicBest.pl -db=hg17 -table=blastzBestMm4
 ./smooth.pl
 ./joinsmallgaps.pl
 ./fillgap.pl -db=hg17 -table=blastzBestMm4
 ./synteny2bed.pl
 #	The five commands above
 #	real    220m16.227s
 #	user    0m22.940s
 #	sys     0m3.960s
 
 #	Used to load this in syntenyMm4, but that type is misleading to
 #	the table browser and fails the checkTableCoords check.
 #	Better to use this ensRatMusHom type:
 #	Need a new name here for the Mm4 to not conflict with Rn3
 sed -e 's/ensPhusionBlast/ensRatMm4Hom/g' \
       $HOME/kent/src/hg/lib/ensPhusionBlast.sql \
       > ensRatMm4Hom.sql
 hgLoadBed hg17 ensRatMm4Hom ucsc100k.bed -sqlTable=ensRatMm4Hom.sql
     #	featureBits hg17 ensRatMm4Hom
     #	2549307611 bases of 2866216770 (88.943%) in intersection
     #	featureBits hg16 syntenyMm4
     #	2560252977 bases of 2865248791 (89.355%) in intersection
 
 # MAKING MOUSE AXTTIGHT FROM AXTBEST (DONE - 2004-06-29 - Hiram)
     # After creating axtBest alignments above, use subsetAxt to get axtTight:
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm4/axtNet
     mkdir -p ../axtTight
     foreach i (*.axt)
       echo $i
       subsetAxt  $i ../axtTight/$i \
         ~kent/src/hg/mouseStuff/subsetAxt/coding.mat 3400
     end
 
     # translate to psl
     cd ../axtTight
     mkdir ../pslTight
     foreach i (*.axt)
       set c = $i:r
       axtToPsl $i ../S1.len ../S2.len ../pslTight/${c}_blastzTightMm4.psl
       echo "Done: $i"
     end
 
     # Load tables into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm4/pslTight
     for I in chr*TightMm4.psl
     do
 	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
 	echo "done ${I}"
     done
 
     #	Compare results with previous assembly:
     #	featureBits hg17 blastzTightMm4
     #	166569246 bases of 2866216770 (5.811%) in intersection
     #	featureBits hg16 blastzTightMm4
     #	162641577 bases of 2865248791 (5.676%) in intersection
 
     # copy  axt's to download area
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm4/axtTight
     mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/vsMm4/axtTight
     cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsMm4/axtTight
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsMm4/axtTight
     gzip *.axt
     # add README.txt file to dir (use previous assembly's copy as template)
 
 # BLASTZ MM4 CLEAN UP (DONE - 2004-07-02 - Hiram)
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm4
     nice rm -rf raw &
     nice rm axtChain/run1/chain/* &
     nice rm -fr axtChain/n1 axtChain/hNoClass.net &
     nice gzip axtChrom/* pslChrom/* lav/*/* axtChain/all.chain axtChain/*.net &
 
 
 # BLASTZ CHIMP panTro1 (DONE 2004-06-22 kate)
 # NOTE: Ran with abridge repeats=0, although SMSK was set
 # Looked better than running with abridge=1, which had very
 # chopped-up alignments
 
     ssh kk
     cd /cluster/data/hg17/bed
     mkdir -p blastz.panTro1.2004-06-22
     rm -f blastz.panTro1
     cd blastz.panTro1.2004-06-22
 
     cat << 'EOF' > DEF
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin
 
 ALIGN=blastz-run
 BLASTZ=blastz
 BLASTZ_H=2000
 BLASTZ_ABRIDGE_REPEATS=0
 
 # Specific settings for chimp
 BLASTZ_Y=3400
 BLASTZ_T=2
 BLASTZ_K=4500
 BLASTZ_Q=/cluster/data/penn/human_chimp.q
 
 # TARGET: Human
 SEQ1_DIR=/scratch/hg/gs.18/build35/bothMaskedNibs
 # not used 
 SEQ1_RMSK=
 # not used
 SEQ1_FLAG=
 SEQ1_SMSK=/iscratch/i/hg17/linSpecRep.chimp
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY: Chimp
 SEQ2_DIR=/scratch/chimp/panTro1/nib
 # not currently used
 SEQ2_RMSK=/iscratch/i/chimp/panTro1/linSpecRep.human
 # not currently used
 SEQ2_FLAG=
 SEQ2_SMSK=
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=10000000
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/blastz.panTro1.2004-06-22
 
 DEF=$BASE/DEF
 RAW=$BASE/raw
 CDBDIR=$BASE
 SEQ1_LEN=$BASE/S1.len
 SEQ2_LEN=$BASE/S2.len
 
 'EOF'
     # << this line keeps emacs coloring happy
 
     # first cluster run: raw blastz alignments
     ssh kk
     bash # if a csh/tcsh user
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22
     source DEF
     mkdir $RAW run.0
     /cluster/home/angie/hummus/make-joblist $DEF > $BASE/run.0/j
     sh ./xdir.sh
     cd run.0
     sed -e 's@^blastz-run@/cluster/bin/penn/blastz-run@' j > jobList
     para create jobList
         # 160270 jobs written to batc
     para try, check, push, check, ....
 #CPU time in finished jobs:    2399227s   39987.11m   666.45h   27.77d  0.076 y
 #IO & Wait Time:                503100s    8385.00m   139.75h    5.82d  0.016 y
 #Average job time:                  18s       0.30m     0.01h    0.00d
 #Longest running job:                0s       0.00m     0.00h    0.00d
 #Longest finished job:            2073s      34.55m     0.58h    0.02d
 #Submission to last job:         10843s     180.72m     3.01h    0.13d
 
 
     # second cluster run: lift raw alignments -> lav dir
     ssh kki
     bash # if a csh/tcsh user
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22
     source DEF
     mkdir run.1 lav
     /cluster/bin/scripts/blastz-make-out2lav $DEF $BASE > $BASE/run.1/jobList
     cd run.1
     wc -l jobList
     para create jobList
         # 341 jobs
     para try, check, push, etc ...
 # CPU time in finished jobs:       3458s      57.63m     0.96h    0.04d  0.000 y
 # IO & Wait Time:                 57996s     966.60m    16.11h    0.67d  0.002 y
 # Average job time:                 180s       3.00m     0.05h    0.00d
 # Longest job:                      483s       8.05m     0.13h    0.01d
 # Submission to last job:          1498s      24.97m     0.42h    0.02d
 
     # third run: lav -> axt -> psl
     ssh kki
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22
     mkdir axtChrom pslChrom run.2
     cd run.2
     cat << '_EOF_' > do.csh
 #!/bin/csh -ef
 cd $1
 set chr = $1:t
 cat `ls -1 *.lav | sort -g` \
 | /cluster/bin/x86_64/lavToAxt stdin \
     /iscratch/i/hg17/bothMaskedNibs /iscratch/i/chimp/panTro1/nib stdout \
 | /cluster/bin/x86_64/axtSort stdin ../../axtChrom/$chr.axt 
 /cluster/bin/x86_64/axtToPsl ../../axtChrom/$chr.axt ../../S1.len ../../S2.len \
   ../../pslChrom/$chr.psl
 '_EOF_'
     # << this line keeps emacs coloring happy
     chmod a+x do.csh
     cp /dev/null jobList
     for d in ../lav/chr*; do
       echo "do.csh $d" >> jobList
     done
     para create jobList
         # 46 jobs
     para try, check, push, check
 #Completed: 42 of 42 jobs
 #Average job time:                  38s       0.64m     0.01h    0.00d
 #Longest job:                      147s       2.45m     0.04h    0.00d
 #Submission to last job:           147s       2.45m     0.04h    0.00d
 
     # Load database tables (takes an hour or so)
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/pslChrom
 cat > load.csh << 'EOF'
     foreach f (chr*.psl)
 	set table = $f:r_blastzPanTro1
 	echo "loading ${table}"
 	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 -table=$f:r_${table} $f
     end
 'EOF'
 # << for emacs
     csh load.csh >&! load.log & 
     tail -100f load.log
 
 
 # CHAIN CHIMP BLASTZ (6/23/04 kate)
     # Run axtChain on little cluster
     # first copy input to bluearc, as eieo bogs down if even mini-cluster
     # gets input from it !?
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22
     cp -rp axtChrom /cluster/bluearc/hg17/blastz.panTro1.2004-06-22/axtChrom
 
     ssh kki
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22
     mkdir -p axtChain/run1
     cd axtChain/run1
     mkdir out chain
     ls -1S /cluster/bluearc/hg17/blastz.panTro1.2004-06-22/axtChrom/*.axt \
       > input.lst
     cat << '_EOF_' > gsub
 #LOOP
 doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     cat << '_EOF_' > doChain
 #!/bin/csh -fe
 set c = $1:r:t
 axtChain $1 -scoreScheme=/cluster/data/blastz/human_chimp.q \
         /iscratch/i/hg17/bothMaskedNibs \
         /iscratch/i/chimp/panTro1/nib /tmp/$c.chain.$$ > /tmp/$c.out.$$
 set ret = $status
 mv -f /tmp/$c.chain.$$ $2
 mv -f /tmp/$c.out.$$ $3
 exit $status
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod a+x doChain
     gensub2 input.lst single gsub jobList
     para create jobList
     para try, check, push, check...
     # TODO
     rm -fr /cluster/bluearc/hg17/blastz.panTro1.2004-06-22/axtChrom
     echo "remove after 7/1/04" > /cluster/bluearc/hg17/blastz.panTro1.2004-06-22/axtChrom/README
 
     # now on the cluster server, sort chains
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
     chainMergeSort run1/chain/*.chain > all.chain
     chainSplit chain all.chain
     # TODO
     rm run1/chain/*.chain
     echo "remove after 7/1/04" > run1/chain/README
 
     # Load chains into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain/chain
     foreach i (*.chain)
         set c = $i:r
         echo loading $c
         hgLoadChain hg17 ${c}_chainPanTro1 $i
     end
     # TODO
     featureBits hg16 chainPanTro1Link
         #2627280557 bases of 2865248791 (91.695%) in intersection
     featureBits hg17 chainPanTro1Link
         # 2633869032 bases of 2866216770 (91.894%) in intersection
 
 
 # NET CHIMP (DONE 2004-6-24 kate)
     # Redone to make chimp.net on 2004-10-11 kate (other files have
     # new times, but are the same as 6-24 versions)
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
     #chainPreNet all.chain ../S1.len ../S2.len stdout \
     #| chainNet stdin -minSpace=1 ../S1.len ../S2.len stdout /dev/null \
     #| netSyntenic stdin noClass.net
 
     time chainPreNet all.chain ../S1.len ../S2.len stdout \
     | chainNet stdin -minSpace=10 ../S1.len ../S2.len human.net chimp.net
         # 42.860u 2.080s 2:11.11 34.2% 
     netSyntenic human.net noClass.net
 
     # Add classification info using db tables:
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
     netClass noClass.net hg17 panTro1 human.net
     rm noClass.net
 
     # Make a 'syntenic' subset:
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
     # TODO
     #rm noClass.net
     # Make a 'syntenic' subset of these with
     # NOTE: we used -chimpSyn filtering for the reciprocal best nets
     # on hg16 -- perhaps should use for nets here as well
     netFilter -chimpSyn human.net > humanSyn.net
 
     # Load the nets into database 
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
     netFilter -minGap=10 human.net |  hgLoadNet hg17 netPanTro1 stdin
     netFilter -minGap=10 humanSyn.net | hgLoadNet hg17 netSyntenyPanTro1 stdin
     # Add entries for chainPanTro1, netPanTro1 to
     # human/hg17 trackDb
 
     # save chimp net to downloads area
     ssh eieio
     cd /cluster/data/hg17/blastz.panTro1/axtChain
     nice gzip chimp.net
     cp chimp.net.gz /usr/local/apache/htdocs/goldenPath/panTro1/vsHg17
     cd /usr/local/apache/htdocs/goldenPath/panTro1/vsHg17
     md5sum *.gz > md5sum.txt
 
 
 # RECIPROCAL BEST CHAINS FOR ENSEMBL GENE BUILD (DONE 2004-10-11 kate)
 
     # Starting with the chimp-reference net, which contains the best human
     # alignments to chimp, extract the subset of chains in the net.
     # (these are the "best" chains of human alignments to chimp).
     # Net these chains and use the resulting human-reference net (the
     # "reciprocal best" net).  Extract the chains from this net to
     # obtain "reciprocal best" chains of chimp alignments to human.
 
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
     mkdir rBest
     grep chain all.chain | wc -l
 
     # extract "best" chains from the chimp-reference net
     time chainSwap all.chain stdout | \
         netChainSubset chimp.net stdin stdout | \
         chainSort stdin rBest/chimp.best.chain
     grep chain rBest/chimp.best.chain | wc -l
         # 64396 
 
     # for comparison later, extract "best" chains from human-reference net
     netChainSubset human.net all.chain stdout | \
         chainSort stdin rBest/human.best.chain
     cd rBest
 
     # net the best chains from the chimp net and pull the human-ref net
     # (Daryl accidentally deleted human.rbest.net and rebuilt it with the 
     #  same command on 8/14/2005, resulting in a file of the same size)
     time chainPreNet chimp.best.chain ../../S2.len ../../S1.len stdout | \
        chainNet stdin -minSpace=10 ../../S2.len ../../S1.len \
                          /dev/null human.rbest.net
 
     # extract "reciprocal best" chains from the "best" human-reference net
     netChainSubset human.rbest.net ../all.chain stdout | \
         chainSort stdin human.rbest.chain
 
     # take a look
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
     cd rBest
     mkdir rBestChain
     chainSplit rBestChain human.rbest.chain
     hgLoadChain hg17 chr7_rBestChainPanTro1 rBestChain/chr7.chain
         # Loading 1639 chains into hg17.chr7_rBestChainPanTro1
     mkdir bestChain
     chainSplit bestChain human.best.chain
     hgLoadChain hg17 chr7_bestChainPanTro1 bestChain/chr7.chain
         # Loading 6516 chains into hg17.chr7_bestChainPanTro1
     # compare 
     hgsql hg16 -s -e "select count(*) from chr7_rBestChainPanTro1"
         # 2416
     # spot-checked by comparing chr7 best and rbest:
     # 1. for a a chain appearing in rBest,  click thru to human browser, 
     #   then via chimp net back to human browser at same region
     # 2. for a chain in "best", but not rBest, do the same, verify
     #   that it produces a different region in the human browser
 
     # post pre-Q/A file for ensembl download
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain/rBest
     gzip human.rbest.chain
     cp human.rbest.chain.gz \
         /usr/local/apache/htdocs/kate/ensembl/hg17-panTro1.rbest.chain.gz
     cd /usr/local/apache/htdocs/kate/ensembl
     md5sum *.gz > md5sum.txt
     mv hg17-panTro1.rbest.chain.gz /usr/local/apache/htdocs/hg17/vsPanTro1/hg17.panTro1.rbest.chain.gz
 
     # save as reciprocal best liftover chain (2005-02-22 kate)
     gunzip -c  human.rbest.chain.gz > \
         /cluster/data/hg17/bed/liftOver/hg17ToPanTro1.rbest.chain
 
     # cleanup (TODO -- after QA)
     ssh hgwdev
     hgsql hg17 -e "drop table chr7_rBestChainPanTro1"
     hgsql hg17 -e "drop table chr7_bestChainPanTro1"
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
     mv rBest/human.rbest.chain.gz ..
     rm -fr rBest
 
 # RECIPROCAL BEST AXT'S FROM RECIPROCAL BEST CHAIN (2005-08-16 kate)
 #       (requested by Daryl)
     cd /cluster/data/hg17/bed/blastz.panTro1
     mkdir -p axtRBestNet
 cat > makeRbestAxt.csh << 'EOF'
 foreach f (axtChain/rBest/rBestChain/*.chain)
     set c = $f:t:t:r
     echo $c
     chainToAxt $f /cluster/data/hg17/nib  /cluster/data/panTro1/nib stdout \
         | axtSort stdin axtRBestNet/$c.axt
     end
 'EOF'
 # << for emacs
     csh makeRbestAxt.csh >&! makeRbestAxt.log &
 
 
 # GENERATE CHIMP MAF FOR MULTIZ FROM NET (DONE 2004-06-24 kate)
     # Redo to fix overlap problem using 8/05 netToAxt (2005-08-16 kate)
     # Replace bad chr5 axtNet and mafNet (2006-01-05 kate) 
     #   There was apparently a bad chr5 nib for a while...
     ssh kkstore02
     cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
     netSplit human.net net
 
     mkdir axtNet mafNet
 cat > makeMaf.csh << 'EOF'
 foreach f (axtChain/net/*.net)
     set c = $f:t:r
     netToAxt $f axtChain/chain/$c.chain /cluster/data/hg17/nib \
         /cluster/data/panTro1/nib stdout | axtSort stdin axtNet/$c.axt
     axtToMaf axtNet/$c.axt  \
         /cluster/data/hg17/chrom.sizes /cluster/data/panTro1/chrom.sizes \
         mafNet/$c.maf -tPrefix=hg17. -qPrefix=panTro1.
     end
 'EOF'
 # << for emacs
     csh makeMaf.csh >&! makeMaf.log &
     tail -100f makeMaf.log
 
     mkdir -p /cluster/bluearc/hg17/mafNet
     cp -rp mafNet /cluster/bluearc/hg17/mafNet/panTro1
 
 
 # MAKE PANTRO1 DOWNLOADABLES (DONE 2004-09-14 kate)
 #  Redo panTro1.net.gz (it was truncated) 2004-10-07 kate
 # Redo axtNets with non-overlapped versions (2005-08-29 kate)
 # Replace bad chr5 axtNet and mafNet (2006-01-05 kate) 
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.panTro1
     # gzip chains and nets
     mkdir gz
     cd gz
     nice gzip -c ../axtChain/all.chain  > panTro1.chain.gz
     nice gzip -c ../axtChain/human.net > panTro1.net.gz
     wc -l *.gz
     cd ../axtNet
     time nice gzip *.axt
         # 46 mins.
    
     ssh hgwdev
     # copy chains and nets to downloads area
     cd /usr/local/apache/htdocs/goldenPath/hg17
     mkdir -p vsPanTro1
     cd vsPanTro1
     mv /cluster/data/hg17/bed/blastz.panTro1/gz/*.gz .
     md5sum *.gz > md5sum.txt
     # copy in README and edit
     rmdir /cluster/data/hg17/bed/blastz.panTro1/gz
     mkdir -p axtNet
     cd axtNet
     cp /cluster/data/hg17/bed/blastz.panTro1/axtNet/*.axt.gz .
     md5sum *.gz > md5sum.txt
 
 
 # RESCORE CHICKEN BLASTZ (DONE 6/23/04 angie)
     # Webb noticed low scores when using non-default BLASTZ_Q scoring matrix 
     # and repeats abridged --
     # PSU's restore_rpts program rescored alignments with default matrix 
     # instead of BLASTZ_Q matrix.  Rescore them here so the chainer sees 
     # the higher scores:
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14
     mkdir axtChrom.rescore
     foreach f (axtChrom/chr*.axt)
       axtRescore -scoreScheme=/cluster/data/blastz/HoxD55.q \
         $f axtChrom.rescore/$f:t
     end
     mv axtChrom axtChrom.preRescore
     mv axtChrom.rescore axtChrom
 
 
 # CHAIN CHICKEN BLASTZ (DONE 6/23/04 angie)
     # Run axtChain on little cluster
     ssh kki
     cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14
     mkdir -p axtChain/run1
     cd axtChain/run1
     mkdir out chain
     ls -1S /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChrom/*.axt \
       > input.lst
     cat << '_EOF_' > gsub
 #LOOP
 doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     cat << '_EOF_' > doChain
 #!/bin/csh
 axtChain -scoreScheme=/cluster/data/blastz/HoxD55.q \
          -linearGap=/cluster/data/blastz/chickenHumanTuned.gap \
          -minScore=5000 $1 \
     /iscratch/i/hg17/bothMaskedNibs \
     /iscratch/i/galGal2/nib $2 > $3
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod a+x doChain
     gensub2 input.lst single gsub jobList
     para create jobList
     para try, check, push, check...
     # axtChrom/chr18_random.axt is empty, so the {out line +} check failed:
 #Completed: 45 of 46 jobs
 #Crashed: 1 jobs
 #Average job time:                  46s       0.76m     0.01h    0.00d
 #Longest job:                      273s       4.55m     0.08h    0.00d
 #Submission to last job:           519s       8.65m     0.14h    0.01d
 
     # now on the cluster server, sort chains
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
     chainMergeSort run1/chain/*.chain > all.chain
     chainSplit chain all.chain
     rm run1/chain/*.chain
 
     # Load chains into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain/chain
     foreach i (*.chain)
         set c = $i:r
         echo loading $c
         hgLoadChain hg17 ${c}_chainGalGal2 $i
     end
 
 
 # NET CHICKEN BLASTZ (DONE 6/23/04 angie)
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
     chainPreNet all.chain ../S1.len ../S2.len stdout \
     | chainNet stdin -minSpace=1 ../S1.len ../S2.len stdout /dev/null \
     | netSyntenic stdin noClass.net
 
     # Add classification info using db tables:
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
     netClass noClass.net hg17 galGal2 human.net
 
     # Make a 'syntenic' subset:
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
     rm noClass.net
     # Make a 'syntenic' subset of these with
     netFilter -syn human.net > humanSyn.net
 
     # Load the nets into database 
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
     netFilter -minGap=10 human.net |  hgLoadNet hg17 netGalGal2 stdin
     netFilter -minGap=10 humanSyn.net | hgLoadNet hg17 netSyntenyGalGal2 stdin
     # Add entries for chainGalGal2, netGalGal2, syntenyGalGal2 to 
     # human/hg17 trackDb
 
 # XENOPUS BLASTZ/CHAIN/NET (DONE 9/24/04 jk)
 # see makeXenTro1.doc and search for zb.hg17
 # The results of this are also symlinked under hg17/bed
 
 # GENERATE GALGAL2 MAF FOR MULTIZ FROM NET (DONE 6/23/04 angie)
 #       Redo net axt's and maf's to fix overlap problem (use 8/05 netToAxt)
 #               (2005-08-16 kate)
     ssh kkstore02
     cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
     netSplit human.net net
     cd ..
     mkdir axtNet mafNet
 cat > makeMaf.csh << 'EOF'
     foreach f (axtChain/net/*)
       set chr = $f:t:r
       netToAxt $f axtChain/chain/$chr.chain /cluster/data/hg17/nib \
         /cluster/data/galGal2/nib stdout \
       | axtSort stdin axtNet/$chr.axt
       axtToMaf axtNet/$chr.axt \
             /cluster/data/hg17/chrom.sizes /cluster/data/galGal2/chrom.sizes \
             mafNet/$chr.maf -tPrefix=hg17. -qPrefix=galGal2.
     end
 'EOF'
 # << for emacs
     csh makeMaf.csh >&! makeMaf.log &
     mkdir -p /cluster/bluearc/hg17/mafNet
     cp -rp mafNet /cluster/bluearc/hg17/mafNet/galGal2
 
 
 # MAKE VSGALGAL2 DOWNLOADABLES (REDONE 9/13/04 angie)
 #   REDO axtNet's to fix overlaps (2005-09-12 kate)
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
     gzip -c all.chain > /cluster/data/hg17/zip/chicken.chain.gz
     gzip -c human.net > /cluster/data/hg17/zip/chicken.net.gz
     mkdir /cluster/data/hg17/zip/axtNet
     foreach f (axtNet/chr*axt)
       gzip -c $f > /cluster/data/hg17/zip/$f.gz
     end
     # Doh! above for loop didn't work because all axt's have been removed 
     # from this dir!  :|  Just this once, regenerate compressed axtNet on 
     # the fly:
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain/net
     foreach f (*.net)
       set chr = $f:t:r
       echo $chr
       netToAxt $f ../chain/$chr.chain /cluster/data/hg17/nib \
         /cluster/data/galGal2/nib stdout \
       | axtSort stdin stdout \
       | gzip -c > /cluster/data/hg17/zip/axtNet/$chr.axt.gz
     end    
 
     ssh hgwdev
     mkdir /usr/local/apache/htdocs/goldenPath/hg17/vsGalGal2
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsGalGal2
     mv /cluster/data/hg17/zip/chicken*.gz .
     mv /cluster/data/hg17/zip/axtNet .
     md5sum *.gz */*.gz > md5sum.txt
     # Copy over & edit README.txt w/pointers to chain, net formats.
 
     # REDO axtNet downloads to fix overlaps (2005-09-13 kate)
     ssh kkstore02
     cd /cluster/data/hg17/bed/blastz.galGal2/axtNet
     nice gzip *.axt
     md5sum *.axt.gz > md5sum.txt
 
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsGalGal2
     mv axtNet axtNet.old
     ln -s /cluster/data/hg17/bed/blastz.galGal2/axtNet .
 
 
 # 8-WAY MULTIZ MULTIPLE ALIGNMENT WITH MM5 (DONE 2004-07-13 kate)
 #       Redo, below to fix overlapping alignments (2005-08-16 kate)
 
     ssh eieio
     set multizDir = multiz.2004-07-13
     set workingDir = /cluster/bluearc/hg17/$multizDir
     ln -s $workingDir /cluster/bluearc/hg17/multiz8way
     mkdir -p $workingDir
     mkdir -p /cluster/data/hg17/bed/$multizDir
     cd /cluster/data/hg17/bed/$multizDir
 
 # wrapper script for multiz
     # NOTE: first arg is pairwise, 2nd arg is multiple (to add to) 
     # NOTE: next time, modify script so it only needs one arg -- saves the
     # multiple dirname in a file for use by the next run
     cat << 'EOF' > doMultiz.csh
 #!/bin/csh -fe
 mkdir -p $3:h
 /cluster/bin/penn/multiz $1 $2 - > $3
 'EOF'
 # << for emacs
     cat << 'EOF' > gsub
 #LOOP
 ../doMultiz.csh {check in line /cluster/bluearc/hg17/multiz.2004-07-13/$(dir1)/$(root2).maf} {check in line /cluster/bluearc/hg17/multiz.2004-07-13/$(root1)/$(root2).maf} {check out line+ /cluster/bluearc/hg17/multiz.2004-07-13/$(root1)$(dir1)/$(root2).maf}
 #ENDLOOP
 'EOF'
 # << for emacs
     chmod +x doMultiz.csh
 
     ssh eieio
     set workingDir = /cluster/bluearc/hg17/multiz.2004-07-13
 
     # copy mafs to bluearc -- chimp
     mkdir $workingDir/panTro1
     cp /cluster/data/hg17/bed/blastz.panTro1/mafNet/*.maf \
                 $workingDir/panTro1
     ls $workingDir/panTro1/*.maf > chrom.lst
 
     # mouse
     mkdir $workingDir/mm5
     cp /cluster/data/hg17/bed/blastz.mm5/mafNet/chr*.maf $workingDir/mm5
 
     # rat
     mkdir $workingDir/rn3
     cp /cluster/data/hg17/bed/blastz.rn3/mafNet/chr*.maf $workingDir/rn3
 
     # dog
     mkdir $workingDir/canFam1
     foreach f (/cluster/data/hg17/bed/blastz.canFam1.2004-07-08/mafNet/chr*.maf)
         set c = $f:r:r:t
         echo $c
         cp $f $workingDir/canFam1/$c.maf
     end
 
     # chicken
     mkdir $workingDir/galGal2
     foreach f (/cluster/data/hg17/bed/blastz.galGal2/mafNet/chr*.maf)
         set c = $f:r:r:t
         cp $f $workingDir/galGal2/$c.maf
     end
 
     # fugu
     mkdir $workingDir/fr1
     cp /cluster/data/hg17/bed/blastz.fr1/mafNet/chr*.maf $workingDir/fr1
 
     # zebrafish
     mkdir $workingDir/danRer1
     cp /cluster/data/hg17/bed/blastz.danRer1.swap/mafNet/chr*.maf \
                         $workingDir/danRer1
 
     # first multiz - add in mm5 mouse to human/chimp
     # 
     ssh kki
     set multizDir = multiz.2004-07-13
     set workingDir = /cluster/bluearc/hg17/$multizDir
     cd /cluster/data/hg17/bed/$multizDir
     mkdir run.mm5
     cd run.mm5
     echo "mm5/panTro1" > species.lst
     gensub2 species.lst ../chrom.lst ../gsub jobList
     para create jobList
         # 46 jobs
     para try, check, push, check
 # CPU time in finished jobs:       6620s     110.33m     1.84h    0.08d  0.000 y
 # IO & Wait Time:                  3685s      61.42m     1.02h    0.04d  0.000 y
 # Average job time:                 224s       3.73m     0.06h    0.00d
 # Longest job:                      819s      13.65m     0.23h    0.01d
 # Submission to last job:          1474s      24.57m     0.41h    0.02d
     cd ..
 
     # rat
     mkdir run.rn3
     cd run.rn3
     echo "rn3/panTro1mm5" > species.lst
     gensub2 species.lst ../chrom.lst ../gsub jobList
     para create jobList
         # 46 jobs
     para try, check, push, check
     cd ..
 
     # dog
     mkdir run.canFam1
     cd run.canFam1
     echo "canFam1/panTro1mm5rn3" > species.lst
     gensub2 species.lst ../chrom.lst ../gsub jobList
     para create jobList
         # 46 jobs
     para try, check, push, check
     cd ../
 
     # chicken
     mkdir run.galGal2
     cd run.galGal2
     echo "galGal2/panTro1mm5rn3canFam1" > species.lst
     gensub2 species.lst ../chrom.lst ../gsub jobList
     # no alignment file for chr18_random -- create one so we can create jobList
     touch $workingDir/galGal2/chr18_random.maf
     para create jobList
         # 46 jobs
     para try, check, push, check
     # 1 crashed job for empty file chr18_random
     cd ..
 
     # fugu
     mkdir run.fr1
     cd run.fr1
     echo "fr1/panTro1mm5rn3canFam1galGal2" > species.lst
     gensub2 species.lst ../chrom.lst ../gsub jobList
     # create empty alignment file for missing one (no alignments)
     touch /cluster/bluearc/hg17/multiz.2004-07-13/fr1/chr6_hla_hap1.maf
     para create jobList
         # 46 jobs
     para try, check, push, check
     # 1 crashed job for empty file chr6_hla_hap1
     cd ..
 
     # zebrafish
     mkdir run.danRer1
     cd run.danRer1
     echo "danRer1/panTro1mm5rn3canFam1galGal2fr1" > species.lst
     gensub2 species.lst ../chrom.lst ../gsub jobList
     para create jobList
         # 46 jobs
     para try, check, push, check
     cd ..
 
     # copy 8-way mafs to build directory
     ssh eieio
     set multizDir = multiz.2004-07-13
     set workingDir = /cluster/bluearc/hg17/$multizDir
     ln -s $workingDir/panTro1mm5rn3canFam1galGal2fr1danRer1 $workingDir/maf
     cd /cluster/data/hg17/bed/multiz.2004-07-13
     mkdir maf
     cp $workingDir/maf/*.maf maf
 
     # copy to download area (2004-07-27 angie)
     # moved gzipped files to mafDownload dir and recreated symlinks 
     #   (2006-04-23 kate)
     cd /usr/local/apache/htdocs/goldenPath/hg17
     mkdir -p mzPt1Mm5Rn3Cf1Gg2Fr1Dr1
     # gzipped & copied maf files from /cluster/data/hg17/bed/multiz8way/maf
     # dumped table and gzipped for download (user request to after file
     # removed when the track was replaced by 18way).
     cd /cluster/data/hg17/bed/multiz8way/mafDownloads
     hgsqldump --all -c --tab=. hg17 multiz8way
     ssh kkstore02 \
         'gzip /cluster/data/hg17/bed/multiz8way/mafDownloads/multiz8way.{sql,txt}'
     ln -s /cluster/data/hg17/bed/multiz8way/mafDownloads/multiz8way.{sql,txt}.gz \
         /usr/local/apache/htdocs/goldenPath/hg17/multiz8way
 
     # load summary table (2005-09-27)
     cd /cluster/data/hg17/bed/multiz.2004-07-13/maf
     time cat chr*.maf | hgLoadMafSummary hg17 multiz8waySummary stdin
     # 30 minutes ?
     # NOTE: this didn't improve track display time at 5MB, so
     #   I'm leaving out of trackDb (sticking with pairwise maf's) for now
     #   It may be that this helps performance only with larger numbers
     #   of species.
 
     # Create upstream files for download (2004-09-13 kate)
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz8way
     echo hg17 panTro1 mm5 rn3 canFam1 galGal2 fr1 danRer1 > org.txt
     # mafFrags takes a while
     foreach i (1000 2000 5000)
         echo "making upstream$i.maf"
         featureBits hg17 refGene:upstream:$i -fa=/dev/null -bed=up.bad
         awk -F '\t' '{printf("%s\t%s\t%s\t%s\t%s\t%s\n", $1, $2, $3, substr($4, 0, 9), 0, $5)}' up.bad > up.bed
         rm up.bad
         mafFrags hg17 multiz8way up.bed upstream$i.maf -orgs=org.txt
         rm up.bed
     end
     ssh eieio
     cd /cluster/data/hg17/bed/multiz8way
     nice gzip upstream{1000,2000,5000}.maf
         # 6 mins.
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17
     ln -s mzPt1Mm5Rn3Cf1Gg2Fr1Dr1 multiz8way
     mv /cluster/data/hg17/bed/multiz8way/upstream*.maf.gz multiz8way
 
 
 
 # PHYLO-HMM (PHASTCONS) CONSERVATION FOR 8-WAY WITH MM5 (DONE 2004-07-20 kate)
 # (this was partially redone by acs using the new phastCons, 08-28;
 # I've tried to merge the two sets of docs into one cohesive
 # description)
 # More revisions, acs, 09-13
 
     ssh eieio
     set path = ($path /cluster/bin/phast)
     cd /cluster/data/hg17/bed/multiz.2004-07-13
     mkdir cons
     cd cons
 
     #break up the genome-wide MAFs into pieces
     mkdir /cluster/bluearc/hg17/chrom
     cd /cluster/data/hg17
     foreach f (`cat chrom.lst`)
         echo $f
         cp $f/*.fa /cluster/bluearc/hg17/chrom
     end
 
     ssh kki
     cd /cluster/data/hg17/bed/multiz.2004-07-13/cons
     mkdir run.split
     cd run.split
     set WINDOWS = /cluster/bluearc/hg17/multiz.2004-07-13/cons/WINDOWS
     rm -fr $WINDOWS
     mkdir -p $WINDOWS
     cat << 'EOF' > doSplit.sh
 #!/bin/sh
 
 PHAST=/cluster/bin/phast
 FA_SRC=/cluster/bluearc/hg17/chrom
 WINDOWS=/cluster/bluearc/hg17/multiz.2004-07-13/cons/WINDOWS
 
 maf=$1
 c=`basename $maf .maf`
 echo $c
 mkdir -p /scratch/msa_split
 ${PHAST}/msa_split $maf -i MAF -M ${FA_SRC}/$c.fa -O hg17,panTro1,mm5,rn3,canFam1,galGal2,fr1,danRer1 -w 1000000,0 -r /scratch/msa_split/$c -o SS -I 1000 -B 5000
 [ $? -eq 0 ] || exit 1
 echo "Copying..."
 cd /scratch/msa_split
 for file in $c.*.ss ; do gzip -c $file > ${WINDOWS}/$file.gz ; done
 [ $? -eq 0 ] || exit 1
 rm -f /scratch/msa_split/$c.*.ss
 echo "Done copying"
 echo "Done" >> ${WINDOWS}/$c.done
 'EOF'
 # << for emacs
     chmod +x doSplit.sh
     rm -f jobList
     foreach file (/cluster/bluearc/hg17/multiz.2004-07-13/maf/*.maf) 
         set c = $file:t:r
 	echo "doSplit.sh $file {check out line+ $WINDOWS/$c.done}" >> jobList
     end
     
     para create jobList
         # 46 jobs
     para try
     para check
     para push
         # 2 crashed jobs -- due to no alignments in input maf
         # chr18_random, chr6_hla_hap1
     cd ..
 
     # now generate conservation scores and predicted elements
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz.2004-07-13/cons
     mkdir run.elements   
     # despite the name, I've put the elements and the new conservation
     # scores here
 
     # first produce a rough starting model; in this case, we can just
     # use the model previously estimated (see the entry below on PHYLOFIT/PHASTCONS)
     cp /cluster/bluearc/hg17/multiz.2004-07-13/panTro1mm5rn3canFam1/hg17panTro1rn3mm5canFam1galGal2fr1danRer1.mod starting-tree.mod
     # In other cases, it would be sufficient to choose an arbitrary
     # input file from the WINDOWS directory (choose one with plenty of
     # data, i.e., large NTUPLES) and run phyloFit on it with the
     # correct tree topology, e.g.,
     # phyloFit -i SS datafile.ss --tree \
     #    "(((((hg17,panTro1),(mm5,rn3)),canFam1),galGal2),(fr1,danRer1))" \
     #    --out-root starting-tree
 
     # Get genome-wide average GC content (for all species together,
     # not just the reference genome).  If you have a globally
     # estimated tree model, as above, you can get this from the
     # BACKGROUND line in the .mod file.  E.g.,
 # ALPHABET: A C G T
 # ...
 # BACKGROUND: 0.294633 0.205082 0.205189 0.295097
     # This implies a GC content of 0.205 + 0.205 = 0.410
 
     # If you do *not* have a global tree model and you do not know
     # your GC content, you can get it directly from the MAFs with
     # a command like:
     #	msa_view --aggregate hg17,panTro1,rn3,mm5,canFam1,galGal2,danRer1,fr1 \
     #       -i MAF --summary-only /cluster/data/hg17/bed/multiz.2004-07-13/maf/chr*.maf\
     #	    > maf_summary.txt
     # This will take a little while (30-60 min).  Run on eieio. 
 
     # now set up cluster job to estimate model parameters.  Parameters
     # will be estimated separately for each alignment fragment then
     # will be combined across fragments
     cat << 'EOF' > doEstimate.sh
 #!/bin/sh
 zcat $1 | /cluster/bin/phast/phastCons - starting-tree.mod --gc 0.410 --nrates 1,1 --no-post-probs --ignore-missing --expected-lengths 12 --target-coverage 0.17 --quiet --log $2 --estimate-trees $3
 EOF
     # Be sure to substitute in the right G+C content. Also, notice the
     # target coverage of 0.17.  We actually want 5% coverage here but
     # the final (posterior) coverage is only indirectly related to the
     # expected (prior) coverage.  One thing to consider is that we
     # only have about 40% alignment coverage (excluding chimp, which
     # doesn't help us much in identifying conserved regions).  As far
     # as phastCons is concerned, we want to aim for about 0.05 / 0.4 =
     # 0.125 coverage.  In this case, though, --target-coverage
     # 0.125 resulted in only about 4.1% coverage.  I had to iterate
     # a couple of times (using only chromosome 1) to find a value that
     # got me close to the target of 5%
 
     chmod u+x doEstimate.sh
 
     rm -fr LOG TREES
     mkdir -p LOG TREES
     rm -f jobs.lst
     # watch out: bash assumed below in a few places
     for f in /cluster/bluearc/hg17/multiz.2004-07-13/cons/WINDOWS/*.ss.gz ; do \
 	root=`basename $f .ss.gz` ;\
 	echo doEstimate.sh $f LOG/$root.log TREES/$root >> jobs.lst ;\
     done
 
     # run cluster job
     ssh kk, cd /cluster/data/hg17/bed/multiz.2004-07-13/cons/run.elements, para create, ...
     # takes about an hour
 
     # Now combine parameter estimates.  We can average the .mod files
     # using phyloBoot.  This must be done separately for the conserved
     # and nonconserved models
     ls TREES/*.cons.mod > cons.txt
     phyloBoot --read-mods '*cons.txt' --output-average ave.cons.mod > cons_summary.txt
     ls TREES/*.noncons.mod > noncons.txt
     phyloBoot --read-mods '*noncons.txt' --output-average ave.noncons.mod > noncons_summary.txt
     # look over the files cons_summary.txt and noncons_summary.txt.
     # The means and medians should be roughly equal and the stdevs
     # should be reasonably small compared to the means, particularly
     # for rate matrix parameters (at bottom) and for branches to the
     # leaves of the tree.  The stdevs may be fairly high for branches
     # near the root of the tree; that's okay.  Some min values may be
     # 0 for some parameters.  That's okay, but watch out for very large
     # values in the max column, which might skew the mean.  If you see
     # any signs of bad outliers, you may have to track down the
     # responsible .mod files and throw them out.  I've never had to do
     # this; the estimates generally seem pretty well behaved.
 
     # NOTE: Actually, a random sample of several hundred to a thousand
     # alignment fragments (say, a number equal to the number of
     # available cluster nodes) should be more than adequate for
     # parameter estimation.  If pressed for time, use this strategy.
 
     # Now we are ready to set up the cluster job for computing the
     # conservation scores and predicted elements.  It's all downhill
     # from here.
 
     cat << 'EOF' > doPhastCons.sh
 #!/bin/sh
 
 mkdir -p /cluster/bluearc/hg17/phastCons/POSTPROBS /cluster/bluearc/hg17/phastCons/ELEMENTS
 pref=`basename $1 .ss.gz`
 chr=`echo $pref | awk -F\. '{print $1}'`
 tmpfile=/scratch/phastCons.$$
 zcat $1 | /cluster/bin/phast/phastCons - ave.cons.mod,ave.noncons.mod --expected-lengths 12 --target-coverage 0.17 --quiet --seqname $chr --idpref $pref --viterbi /cluster/bluearc/hg17/phastCons/ELEMENTS/$pref.bed --score --require-informative 0 > $tmpfile
 gzip -c $tmpfile > /cluster/bluearc/hg17/phastCons/POSTPROBS/$pref.pp.gz
 rm $tmpfile
 EOF
     chmod u+x doPhastCons.sh
 
     rm -fr /cluster/bluearc/hg17/phastCons/POSTPROBS /cluster/bluearc/hg17/phastCons/ELEMENTS
     rm -f jobs2.lst
     for f in /cluster/bluearc/hg17/multiz.2004-07-13/cons/WINDOWS/*.ss.gz ; do echo doPhastCons.sh $f >> jobs2.lst ; done
 
     # run cluster job
     ssh kk, cd /cluster/data/hg17/bed/multiz.2004-07-13/cons/run.elements, para create, ...
     logout
     # takes about 20 minutes
 
     # combine predictions and transform scores to be in 0-1000 interval
     # do in a way that avoids limits on numbers of args
     find /cluster/bluearc/hg17/phastCons/ELEMENTS -name "*.bed" > files
     rm -f splitfiles* all.raw.bed
     split files splitfiles
     for s in splitfiles* ; do awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", $1, $2, $3, $5, $5;}' `cat $s` >> all.raw.bed ; done
     /cluster/bin/scripts/lodToBedScore all.raw.bed > all.bed
     rm files splitfiles* 
 
     hgLoadBed hg17 phastConsElements all.bed
     hgLoadBed -chrom=chr1 hg17 phastConsElements all.bed
 
     # check coverage
     featureBits hg17 phastConsElements
 #137850739 bases of 2866216770 (4.810%) in intersection
     # This should be close enough.  If necessary, you can rerun the
     # steps above with a different target coverage.  When hitting the
     # target is important, you may want to perform several iterations
     # using a representative subset of the entire dataset (human chr1
     # seems to work pretty well)
 
     # set up wiggle
     mkdir -p /cluster/bluearc/hg17/phastCons/wib
     cat << 'EOF' > doWigAsciiToBinary.sh
 #!/bin/sh
 chr=$1
 zcat `ls /cluster/bluearc/hg17/phastCons/POSTPROBS/$chr.*.pp.gz | sort -t\. -k2,2n` | wigAsciiToBinary -chrom=$chr -wibFile=/cluster/bluearc/hg17/phastCons/wib/${chr}_phastCons stdin 
 EOF
     chmod u+x doWigAsciiToBinary.sh
 
     rm -f jobs3.lst
     for chr in `ls /cluster/bluearc/hg17/phastCons/POSTPROBS | awk -F\. '{print $1}' | sort -u` ; do echo doWigAsciiToBinary.sh $chr >> jobs3.lst ; done
 
     # run a little wigAsciiToBinary cluster job
     ssh kk, etc.
 
     # copy wibs and wigs from bluearc
     rsync -av /cluster/bluearc/hg17/phastCons/wib .
 
     # load track
     hgLoadWiggle hg17 phastCons -pathPrefix=/gbdb/hg17/phastCons/wib \
                 wib/chr*_phastCons.wig
     mkdir -p /gbdb/hg17/phastCons/wib
     rm -f /gbdb/hg17/phastCons/wib/chr*phastCons.wib
     ln -s /cluster/data/hg17/bed/multiz.2004-07-13/cons/run.elements/wib/*.wib /gbdb/hg17/phastCons/wib
     chmod 775 . wib /gbdb/hg17/phastCons /gbdb/hg17/phastCons/wib
     chmod 664 wib/*.wib
 
     # move postprobs over and clean up bluearc 
     rsync -av /cluster/bluearc/hg17/phastCons/POSTPROBS .
     # (people sometimes want the raw scores)
     rm -r /cluster/bluearc/hg17/phastCons/ELEMENTS /cluster/bluearc/hg17/phastCons/POSTPROBS /cluster/bluearc/hg17/phastCons/wib
 
     # set up full alignment/conservation track ("multiz8way")
     # load multiz maf tables 
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz.2004-07-13
     set mafDir = /gbdb/hg17/multiz8way/maf
     set table = multiz8way
     mkdir -p $mafDir/$table
     ln -s `pwd`/maf/*.maf $mafDir/$table
     cd maf
     hgLoadMaf hg17 -warn multiz8way -pathPrefix=$mafDir/$table/maf
 
     # someone dropped from hgwdev
     # reload (2007-03-19 kate)
     nice hgLoadMaf hg17 -warn multiz8way -pathPrefix=/gbdb/hg17/multiz8wayFixed
     cat /gbdb/hg17/multiz8wayFixed/*.maf | \
         nice hgLoadMafSummary hg17 -minSize=30000 -mergeGap=1500 -maxSize=200000  \
             multiz8waySummary stdin
 
     # load blastz maf tables
     # TODO: change mafWiggle to use db names instead of species names
     # in speciesOrder 
     # link files into /gbdb table dir
     ln -s /cluster/data/hg17/bed/blastz.panTro1/mafNet $mafDir/chimp_netBlastz
     ln -s /cluster/data/hg17/bed/blastz.mm5/mafNet $mafDir/mouse_netBlastz
     ln -s /cluster/data/hg17/bed/blastz.rn3/mafNet $mafDir/rat_netBlastz
     ln -s /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/mafNet $mafDir/dog_netBlastz
     ln -s /cluster/data/hg17/bed/blastz.galGal2/mafNet $mafDir/chicken_netBlastz
     ln -s /cluster/data/hg17/bed/blastz.fr1/mafNet $mafDir/fugu_netBlastz
     ln -s /cluster/data/hg17/bed/blastz.danRer1.swap/mafNet $mafDir/zebrafish_netBlastz
 
     # remove empty file, disliked by hgLoadMaf
     # NOTE: these shouldn't be empty -- next time, make sure previous
     # alignments are copied over to output maf (multiz won't if there's
     # an empty input file).
     rm chicken/chr18_random.maf
     rm fugu/chr6_hla_hap1.maf
 
     # load tables
     foreach s (chimp mouse rat dog chicken fugu zebrafish)
         set table = ${s}_netBlastz
         echo "$s $mafDir/$table"
         ~kate/bin/i386/hgLoadMaf hg17 -warn ${s}_netBlastz -pathPrefix=$mafDir/$table
     end
 
     # trackDb entry:
 # track multiz8way
 # shortLabel Conservation
 # longLabel Chimp/Mouse/Rat/Dog/Chicken/Fugu/Zebrafish Multiz Alignments & Conservation
 # group compGeno
 # priority 149
 # visibility pack
 #color 0, 10, 100
 # type wigMaf 0.0 1.0
 # maxHeightPixels 100:40:11
 # wiggle phastCons
 # yLineOnOff Off
 # autoScaleDefault Off
 # pairwise netBlastz
 # speciesOrder chimp mouse rat dog chicken fugu zebrafish
 
 
 # PHASTCONS SCORES DOWNLOADABLES (REDONE 6/15/05 angie)
     # Initially done 10/11/04, but using scores from run.cons -- which 
     # had been replaced by scores in run.elements, where I did not think 
     # to look for scores.  :(  !
     ssh eieio
     mkdir /cluster/data/hg17/zip/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1
     cd /cluster/data/hg17/bed/multiz8way/cons/run.elements/POSTPROBS
     foreach chr (`awk '{print $1;}' /cluster/data/hg17/chrom.sizes`)
       echo $chr
       nice zcat `ls -1 $chr.*.pp.gz | sort -t\. -k2,2n` \
       | nice gzip -c \
       > /cluster/data/hg17/zip/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1/$chr.gz
     end
     ssh hgwdev
     mkdir /usr/local/apache/htdocs/goldenPath/hg17/phastCons
     # Doh!  /cluster/data/hg17/zip/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1 is 11G now -- 
     # too much to dump on hgwdev's / which is at 94%.  So don't do this:
     #mv /cluster/data/hg17/zip/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1 .
     # make symbolic links instead:
     mkdir /usr/local/apache/htdocs/goldenPath/hg17/phastCons/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1
     cd /usr/local/apache/htdocs/goldenPath/hg17/phastCons/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1
     ln -s /cluster/data/hg17/zip/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1/* .
     md5sum *.gz > md5sum.txt
     # make a README.txt.
 
 
 # PHYLOFIT AND TREE-DOCTOR FOR 8-WAY: ESTIMATE PHYLOGENETIC TREE (acs)
 # (This was originally done for phastCons but is not necessary with
 # the new version.  However, it may be useful for other purposes, so
 # I'm leaving it in as a separate entry.)
 
     # first estimate a model for the mammals
     ssh eieio
     cd /cluster/bluearc/hg17/multiz.2004-07-13/panTro1mm5rn3canFam1
 
     # collect sufficient stats (takes maybe an hour)
     for file in *.maf ; do echo $file ; msa_view -i MAF $file -o SS --order hg17,panTro1,rn3,mm5,canFam1 > `basename $file .maf`.ss ; done
     ls *.ss | grep -v chr6_hla_hap2 > files
     msa_view '*files' --aggregate hg17,panTro1,rn3,mm5,canFam1 -i SS -o SS > all.ss
     # BTW, this can now be done in one step using something like:
     # msa_view --aggregate hg17,panTro1,rn3,mm5,canFam1 -i MAF -o SS *.maf > all.ss
     # (modify to exclude certain files if necessary)
 
     # estimate model, with rate variation (takes about a minute)
     phyloFit all.ss --nrates 10 --tree "(((hg17,panTro1),(rn3,mm5)),canFam1)" --alpha 4.4 --EM --log log -i SS --out-root hprmc-rev-dg
     # (Actually, --nrates 4 should be more than adequate for most purposes)
 
     cat hprmc-rev-dg.mod
 #ALPHABET: A C G T
 #ORDER: 0
 #SUBST_MOD: REV
 #NRATECATS: 10
 #ALPHA: 4.658942
 #TRAINING_LNL: -6889216721.159384
 #BACKGROUND: 0.294633 0.205082 0.205189 0.295097
 #RATE_MAT:
 #  -0.865237    0.159990    0.554805    0.150442
 #   0.229851   -1.194646    0.168269    0.796526
 #   0.796651    0.168182   -1.194919    0.230086
 #   0.150205    0.553556    0.159985   -0.863747
 #TREE: (((1:0.006523,2:0.007997):0.103779,(3:0.104867,4:0.078911):0.265676):0.112364,5:0.112364);
 
     # now extrapolate to fish and chicken using tree_doctor and the CFTR 25 tree 
     # (replace numbers with names in hprmc-rev-dg.mod; this won't be necessary in the future)
     tree_doctor --rename "1->hg17;2->panTro1;3->rn3;4->mm5;5->canFam1" hprmc-rev-dg.mod > hprmc-rev-dg.names.mod
     # (obtain 8-way subtree from cftr25_hybrid.nh; also map names as necessary to match above)
     tree_doctor /cluster/data/nisc/targets/cftr/phyloHMMcons25/cftr25_hybrid.nh --prune-all-but hg16,chimp,mm3,rn3,dog,chicken,fr1,zfish --rename "hg16->hg17;mm3->mm5;chimp->panTro1;dog->canFam1;chicken->galGal2;zfish->danRer1" > cftr8way.nh
     # now merge (see tree_doctor help page for explanation)
     tree_doctor hprmc-rev-dg.names.mod --merge cftr8way.nh > hg17panTro1rn3mm5canFam1galGal2fr1danRer1.mod
 
     cat hg17panTro1rn3mm5canFam1galGal2fr1danRer1.mod
 #ALPHABET: A C G T
 #ORDER: 0
 #SUBST_MOD: REV
 #NRATECATS: 10
 #ALPHA: 4.658942
 #BACKGROUND: 0.294633 0.205082 0.205189 0.295097
 #RATE_MAT:
 #  -0.865237    0.159990    0.554805    0.150442
 #   0.229851   -1.194646    0.168269    0.796526
 #   0.796651    0.168182   -1.194919    0.230086
 #   0.150205    0.553556    0.159985   -0.863747
 #TREE: (((((hg17:0.006523,panTro1:0.007997):0.103779,(rn3:0.104867,mm5:0.078911):0.265676):0.019461,canFam1:0.205267):0.377150,galGal2:0.511134):0.536627,(danRer1:0.905323,fr1:0.922995):0.536627);
 
 
 # CONSERVED NON-CODING (CNS) TRACK (acs 08/29/04)
 # (depends on phastConsElements)
 
     cd /cluster/data/hg17/bed/multiz.2004-07-13/cons/run.elements
     featureBits hg17 -bed=possibleCoding.bed -or twinscan:exon xenoMrna mrna intronEst 
     # (add SGP, exoniphy, possib. others if available)
     # now filter out all phastCons elements that overlap
     overlapSelect -nonOverlapping possibleCoding.bed all.bed cns.bed
     hgLoadBed hg17 cns cns.bed
 
 # track cns
 # shortLabel CNS
 # longLabel Conserved Non-Coding (Cons Elements Minus Predicted Coding)
 # priority 109.11
 # group compGeno
 # visibility hide
 # type bed 5 .
 
     
 # PRODUCING GENSCAN PREDICTIONS (DONE - 2004-07-08 - Hiram)
 #	Needed to download a new binary for this run.  Our Linux systems
 #  XXX - I thought a new binary was needed.  Turned out it was already
 #  here in our hg3rdParty CVS project.  All of this discussed here can
 #  be simply fetched from cvs:  cvs co hg3rdParty/genscanlinux
 #	have been updated since the last time, the old binary would not
 #	run.  Go to: http://genes.mit.edu/GENSCAN.html
 #	and then to: http://genes.mit.edu/license.html
 #	Fill in the license agreement and you can then pick up the
 #	README and the Linux version: genscanlinux.tar.uue.tgz
 #	To uudecode that file, go to one of the Solaris home machines
 #	and use the uudecode command:
 #	uudecode genscanlinux.tar.uue.tgz
 #	That produces the file: genscanlinux.tar
 #	Which contains the files:
 # drwxr-xr-x chris/burgelab    0 2003-02-17 11:48:44 ./
 # -rw-r--r-- chris/burgelab 219056 2000-09-07 12:39:26 ./Arabidopsis.smat
 # -rw-r--r-- chris/burgelab   6622 2000-09-07 12:39:26 ./HUMRASH
 # -rw-r--r-- chris/burgelab    849 2000-09-07 12:39:26 ./HUMRASH.sample
 # -rw-r--r-- chris/burgelab 219050 2000-09-07 12:39:26 ./HumanIso.smat
 # -rw-r--r-- chris/burgelab 155735 2000-09-07 12:39:26 ./Maize.smat
 # -rw-r--r-- chris/burgelab  24465 2000-09-07 12:39:26 ./README
 # -rw-r--r-- chris/burgelab   6344 2000-09-07 12:39:27 ./HUMRASH.ps
 # -rwxr-xr-x chris/burgelab 126365 2003-02-17 11:48:44 ./genscan
 #
 #	I placed these currently in: /cluster/home/hiram/GENSCAN/
 #	I'll check with Angie where it should properly live ...
 #  XXX - it already lives in 'cvs co hg3rdParty/genscanlinux'
 #  These instructions should simple check it out right here in
 #  bed/genscan and make the gsub command refer to these copies.
 
     ssh hgwdev
     mkdir /cluster/data/hg17/bed/genscan
     cd /cluster/data/hg17/bed/genscan
     cvs co hg3rdParty/genscanlinux
 
     ssh eieio
     cd /cluster/data/hg17/bed/genscan
     # Make 3 subdirectories for genscan to put their output files in
     mkdir gtf pep subopt
     # Generate a list file, genome.list, of all the contigs
     # *that do not have pure Ns* (due to heterochromatin, unsequencable 
     # stuff) which would cause genscan to run forever.
     rm -f genome.list
     bash
     for f in `cat /cluster/data/hg17/contig.lst`
     do
       egrep '[ACGT]' /cluster/data/hg17/$f.masked > /dev/null
 	if [ $? = 0 ]; then
 	    echo /cluster/data/hg17/$f.masked >> genome.list
 	fi
     done
     # exit your bash shell if you are [t]csh ...
     #	This egrep matched all the contigs in hg17.  I guess none of
     #	them are complete Ns* at this point.
 
     # Log into kki (not kk !).  kki is the driver node for the small
     # cluster (kkr2u00 -kkr8u00. Genscan has problem running on the
     # big cluster, due to limitation of memory and swap space on each
     # processing node).
     ssh kki
     cd /cluster/data/hg17/bed/genscan
     # Create template file, gsub, for gensub2.  For example (3-line file):
     cat << '_EOF_' > gsub
 #LOOP
 /cluster/bin/x86_64/gsBig {check in line+ $(path1)} {check out line gtf/$(root1).gtf} -trans={check out line pep/$(root1).pep} -subopt={check out line subopt/$(root1).bed} -exe=hg3rdParty/genscanlinux/genscan -par=hg3rdParty/genscanlinux/HumanIso.smat -tmp=/tmp -window=2400000
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
     gensub2 genome.list single gsub jobList
     para create jobList
     para try
     para check
     para push ... etc ...
 # Completed: 379 of 380 jobs
 # Crashed: 1 jobs
 # CPU time in finished jobs:      79998s    1333.30m    22.22h    0.93d  0.003 y
 # IO & Wait Time:                  2989s      49.82m     0.83h    0.03d  0.000 y
 # Average job time:                 219s       3.65m     0.06h    0.00d
 # Longest job:                     2999s      49.98m     0.83h    0.03d
 # Submission to last job:          8324s     138.73m     2.31h    0.10d
 
     #	Running the single failed job on kolossus with a smaller window:
 /cluster/bin/x86_64/gsBig /cluster/data/hg17/5/NT_006576/NT_006576.fa.masked \
         gtf/NT_006576.fa.gtf -trans=pep/NT_006576.fa.pep \
         -subopt=subopt/NT_006576.fa.bed -exe=hg3rdParty/genscanlinux/genscan \
         -par=hg3rdParty/genscanlinux/HumanIso.smat -tmp=/tmp -window=2000000
 
     # If there were out-of-memory problems (run "para problems"), then 
     # re-run those jobs by hand but change the -window arg from 2400000
     # something lower.  In build33, this was 22/NT_011519
     #  In build34 there were NO failures !
 
     # Convert these to chromosome level files as so:     
     ssh eieio
     cd /cluster/data/hg17/bed/genscan
     $HOME/bin/i386/liftUp genscan.gtf ../../jkStuff/liftAll.lft warn gtf/N*.gtf
     $HOME/bin/i386/liftUp genscanSubopt.bed ../../jkStuff/liftAll.lft \
 	warn subopt/N*.bed
     cat pep/*.pep > genscan.pep
 
     # Load into the database as so:
     ssh hgwdev
     cd /cluster/data/hg17/bed/genscan
     ldHgGene hg17 genscan genscan.gtf
     #	35 minute job
     #	Read 42807 transcripts in 325994 lines in 1 files
     #	  42807 groups 46 seqs 1 sources 1 feature types
 
     hgPepPred hg17 generic genscanPep genscan.pep
     #	Processing genscan.pep
     hgLoadBed hg17 genscanSubopt genscanSubopt.bed
     #	Reading genscanSubopt.bed
     #	Loaded 517157 elements of size 6
     #	Sorted
     #	Creating table definition for 
     #	Saving bed.tab
     #	Loading hg17
 
     #	featureBits hg17 genscan
     #	55323340 bases of 2866216770 (1.930%) in intersection
 
     #	featureBits hg16 genscan
     #	55333689 bases of 2865248791 (1.931%) in intersection
 
     #	featureBits hg17 genscanSubopt
     #	55986178 bases of 2866216770 (1.953%) in intersection
     #	featureBits hg16 genscanSubopt
     #	56082952 bases of 2865248791 (1.957%) in intersection
 
     #	Should be zero intersection with rmsk
     #	featureBits -chrom=chr1 hg17 genscan rmsk
     #	794 bases of 222827847 (0.000%) in intersection
 
 
 # EXTRACT LINEAGE-SPECIFIC REPEATS FOR DOG (DONE 8/1/05 angie)
     # Originally done 7/1/04 for canFam1 -- redone 8/1/05 for canFam2.
     ssh kolossus
     cd /san/sanvol1/scratch/hg17/rmsk
     # Run Arian's DateRepsinRMoutput.pl to add extra columns telling 
     # whether repeats in -query are also expected in -comp species.  
     # Even though we already have the human-mouse linSpecReps,
     # extractLinSpecReps requires two columns of DateRepsinRMoutput.pl
     # additions.  So add mouse, then ignore it.  
     # Dog in extra column 1, Mouse in extra column 2
     foreach outfl ( *.out )
         echo "$outfl"
         /cluster/bluearc/RepeatMasker/DateRepeats \
           ${outfl} -query human -comp dog -comp mouse
     end
     # Now extract dog (extra column 1), ignore mouse.
     cd ..
     mkdir linSpecRep.notInDog
     foreach f (rmsk/*.out_canis-familiaris_mus-musculus)
         set base = $f:t:r:r
         echo $base.out.spec
         /cluster/bin/scripts/extractLinSpecReps 1 $f > \
                         linSpecRep.notInDog/$base.out.spec
     end
     # Clean up.
     rm rmsk/*.out_canis*
     rsync -av /san/sanvol1/scratch/hg17/linSpecRep.notInDog \
       /cluster/bluearc/scratch/hg/gs.18/build35/
     # Ask cluster-admin for an rsync.
 
 
 # BLASTZ DOG (CANFAM1) (DONE 7/8/04 angie)
     ssh kk
     # space is awful tight on store4 -- use store7.  
     mkdir -p /cluster/store7/hg17/bed/blastz.canFam1.2004-07-08
     ln -s /cluster/store7/hg17/bed/blastz.canFam1.2004-07-08 \
       /cluster/data/hg17/bed/
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08
     # Use default (Human-Mouse) settings for starters.
     cat << '_EOF_' > DEF
 # human vs. dog
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin
 
 ALIGN=blastz-run
 BLASTZ=blastz
 
 # Default
 BLASTZ_H=2000
 BLASTZ_ABRIDGE_REPEATS=1
 
 # TARGET: Human
 SEQ1_DIR=/scratch/hg/gs.18/build35/bothMaskedNibs
 SEQ1_RMSK=
 SEQ1_FLAG=
 SEQ1_SMSK=/scratch/hg/gs.18/build35/linSpecRep.notInDog
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY: Dog
 SEQ2_DIR=/scratch/hg/canFam1/nib
 SEQ2_RMSK=
 SEQ2_FLAG=
 SEQ2_SMSK=/scratch/hg/canFam1/linSpecRep.notInHuman
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=10000000
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/blastz.canFam1.2004-07-08
 
 DEF=$BASE/DEF
 RAW=$BASE/raw
 CDBDIR=$BASE
 SEQ1_LEN=$BASE/S1.len
 SEQ2_LEN=$BASE/S2.len
 '_EOF_'
     # << this line keeps emacs coloring happy
 
     # first cluster run: raw blastz alignments
     ssh kk
     bash # if a csh/tcsh user
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08
     source DEF
     mkdir $RAW run.0
     /cluster/home/angie/hummus/make-joblist $DEF > $BASE/run.0/j
     sh ./xdir.sh
     cd run.0
     sed -e 's@^blastz-run@/cluster/bin/penn/blastz-run@' j > jobList
     para create jobList
     para try, check, push, check, ....
     # Moving the human chr19 jobs up to the top of the jobList probably 
     # would have shaved 4 hours off the total time!  It was almost done 
     # after 6 hours, except for a few chr19 stragglers.
 #Completed: 93775 of 93775 jobs
 #Average job time:                 202s       3.37m     0.06h    0.00d
 #Longest job:                    17806s     296.77m     4.95h    0.21d
 #Submission to last job:         35523s     592.05m     9.87h    0.41d
 
     # second cluster run: lift raw alignments -> lav dir
     ssh kki
     bash # if a csh/tcsh user
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08
     source DEF
     mkdir run.1 lav
     /cluster/bin/scripts/blastz-make-out2lav $DEF $BASE > $BASE/run.1/jobList
     cd run.1
     wc -l jobList
     para create jobList
     para try, check, push, etc ...
 #Completed: 341 of 341 jobs
 #Average job time:                  36s       0.61m     0.01h    0.00d
 #Longest job:                      302s       5.03m     0.08h    0.00d
 #Submission to last job:          1143s      19.05m     0.32h    0.01d
 
     # third run: lav -> axt
     # (if non-default BLASTZ_Q is used in the future, put axtRescore in 
     # the pipe after lavToAxt)
     ssh kki
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08
     mkdir axtChrom pslChrom run.2
     cd run.2
     cat << '_EOF_' > do.csh
 #!/bin/csh -ef
 cd $1
 set chr = $1:t
 cat `ls -1 *.lav | sort -g` \
 | $HOME/bin/x86_64/lavToAxt stdin \
     /iscratch/i/gs.18/build35/bothMaskedNibs /iscratch/i/canFam1/nib stdout \
 | $HOME/bin/x86_64/axtSort stdin ../../axtChrom/$chr.axt 
 $HOME/bin/x86_64/axtToPsl ../../axtChrom/$chr.axt ../../S1.len ../../S2.len \
   ../../pslChrom/$chr.psl
 '_EOF_'
     # << this line keeps emacs coloring happy
     chmod a+x do.csh
     cp /dev/null jobList
     foreach d (../lav/chr*)
       echo "do.csh $d" >> jobList
     end
     para create jobList
     para try, check, push, check
 #Completed: 46 of 46 jobs
 #Average job time:                 300s       5.00m     0.08h    0.00d
 #Longest job:                     1669s      27.82m     0.46h    0.02d
 #Submission to last job:          1689s      28.15m     0.47h    0.02d
 
 
 # CHAIN DOG BLASTZ (DONE 7/9/04 angie)
     # Run axtChain on little cluster
     ssh kki
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08
     mkdir -p axtChain/run1
     cd axtChain/run1
     mkdir out chain
     ls -1S /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChrom/*.axt \
       > input.lst
     cat << '_EOF_' > gsub
 #LOOP
 doChain {check in line+ $(path1)} {check out line+ chain/$(root1).chain} {check out exists out/$(root1).out}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     cat << '_EOF_' > doChain
 #!/bin/csh
 axtChain $1 \
     /iscratch/i/gs.18/build35/bothMaskedNibs \
     /iscratch/i/canFam1/nib $2 > $3
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod a+x doChain
     gensub2 input.lst single gsub jobList
     para create jobList
     para try, check, push, check...
 #Completed: 46 of 46 jobs
 #Average job time:                 266s       4.43m     0.07h    0.00d
 #Longest job:                     3578s      59.63m     0.99h    0.04d
 #Submission to last job:          3578s      59.63m     0.99h    0.04d
     # now on the cluster server, sort chains
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
     chainMergeSort run1/chain/*.chain > all.chain
     chainSplit chain all.chain
     rm run1/chain/*.chain
 
     # take a look at score distr's
     foreach f (chain/*.chain)
       grep chain $f | awk '{print $2;}' | sort -nr > /tmp/score.$f:t:r
       echo $f:t:r
       textHistogram -binSize=10000 /tmp/score.$f:t:r
       echo ""
     end
 
     # Lots of chaff with scores in the 3000's.  Many very-high-scoring 
     # chains.  So filter the chain down somewhat...
     mv all.chain all.chain.unfiltered
     chainFilter -minScore=5000 all.chain.unfiltered > all.chain
     rm chain/*
     chainSplit chain all.chain
     gzip all.chain.unfiltered
 
     # Load chains into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain/chain
     foreach i (*.chain)
         set c = $i:r
         hgLoadChain hg17 ${c}_chainCanFam1 $i
     end
     # Coverage is significantly higher than mouse:
     featureBits hg17 -chrom=chr1 chainCanFam1Link
 #123999291 bases of 222827847 (55.648%) in intersection
 # before filtering: 124750124 bases of 222827847 (55.985%) in intersection
     featureBits hg17 -chrom=chr1 chainMm5Link
 #83773012 bases of 222827847 (37.595%) in intersection
 
 
 # NET DOG BLASTZ (DONE 7/9/04 angie)
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
     chainPreNet all.chain ../S1.len ../S2.len stdout \
     | chainNet stdin -minSpace=1 ../S1.len ../S2.len stdout /dev/null \
     | netSyntenic stdin noClass.net
 
     # Add classification info using db tables:
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
     netClass noClass.net hg17 canFam1 dog.net
 
     # Make a 'syntenic' subset:
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
     rm noClass.net
     # Make a 'syntenic' subset of these with
     netFilter -syn dog.net > dogSyn.net
 
     # Load the nets into database 
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
     netFilter -minGap=10 dog.net |  hgLoadNet hg17 netCanFam1 stdin
     netFilter -minGap=10 dogSyn.net | hgLoadNet hg17 syntenyNetCanFam1 stdin
     # Add entries for chainCanFam1, netCanFam1 to human/hg17 trackDb
 
 
 # MAKE VSCANFAM1 DOWNLOADABLES (DONE 9/17/04 kate)
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
     ln -s all.chain dog.chain
     mkdir gz
     cd gz
     gzip -c ../dog.chain > dog.chain.gz
     gzip -c ../dog.net > dog.net.gz
     gzip ../dogSyn.net > dogSyn.net.gz
     # Angie's notes...
     # Mike Zody asked for raw blastz in chain format, so figure out some 
     # way to translate axt or psl to chain and put it out there.  
     # Actually, it's probably just hg16-canFam1 that he wants for now -- ?
     # Ask when we get to this point.
 
     cd ../axtNet
     time gzip *.axt
 
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17
     mkdir -p vsCanFam1
     cd vsCanFam1
     mv /cluster/data/hg17/bed/blastz.canFam1/axtChain/gz/*.gz .
     md5sum *.gz > md5sum.txt
     mkdir -p axtNet
     cd axtNet
     cp /cluster/data/hg17/bed/blastz.canFam1/axtNet/*.axt.gz .
     md5sum *.gz > md5sum.txt
     # Copy over & edit README.txt w/pointers to chain, net formats.
 
     # REDO downloads of axtNet's to fix overlaps (2005-09-13 kate)
     # Finally, replace bad chr5 files (2006-01-05 kate)
     ssh kkstore02
     cd /cluster/data/hg17/bed/blastz.canFam1/axtNet
     nice gzip *.axt
     md5sum *.axt.gz > md5sum.txt
 
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsCanFam1
     mv axtNet axtNet.old
     ln -s /cluster/data/hg17/bed/blastz.canFam1/axtNet .
 
 
 # GENERATE CANFAM1 MAF FOR MULTIZ FROM NET (DONE 7/9/04 angie)
 # Redo net axt's and maf's to fix overlaps (use 8/5 netToAxt)
 #       (2005-08-16 kate)
 # and replace bad chr5 files (2006-01-05 kate)
     ssh kkstore02
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
     netSplit dog.net net
     cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08
     mkdir axtNet mafNet
 cat > makeMaf.csh << 'EOF'
     foreach f (axtChain/net/*)
       set chr = $f:t:r
       echo $chr
       netToAxt $f axtChain/chain/$chr.chain /cluster/data/hg17/nib \
         /cluster/data/canFam1/nib stdout \
       | axtSort stdin axtNet/$chr.axt
       axtToMaf axtNet/$chr.axt \
             /cluster/data/hg17/chrom.sizes /cluster/data/canFam1/chrom.sizes \
             mafNet/$chr.maf -tPrefix=hg17. -qPrefix=canFam1.
     end
 'EOF'
     csh makeMaf.csh >&! makeMaf.log &
     mkdir -p /cluster/bluearc/hg17/mafNet
     cp -rp mafNet /cluster/bluearc/hg17/mafNet/canFam1
 
 
 # BLASTZ MM5 (DONE - 2004-06-22 - Hiram)
     ssh kk
     mkdir -p /cluster/data/hg17/bed/blastz.mm5.2004-07-01
     cd /cluster/data/hg17/bed
     ln -s  blastz.mm5.2004-07-01 blastz.mm5
     cd blastz.mm5
 
     cat << '_EOF_' > DEF
 # human vs. mouse
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/home/angie/schwartzbin:/cluster/home/kent/bin/i386
 
 ALIGN=blastz-run
 BLASTZ=blastz
 BLASTZ_H=2000
 BLASTZ_ABRIDGE_REPEATS=1
 
 # TARGET
 # Human
 SEQ1_DIR=/iscratch/i/gs.18/build35/bothMaskedNibs
 # not used
 SEQ1_RMSK=
 # not used
 SEQ1_FLAG=
 SEQ1_SMSK=/iscratch/i/gs.18/build35/linSpecRep.notInRat
 # notInRat OK as it is identical to notInMouse
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY
 # Mouse
 SEQ2_DIR=/scratch/mus/mm5/softNib
 # RMSK not currently used
 SEQ2_RMSK=/scratch/mus/mm5/rmsk
 # FLAG not currently used
 SEQ2_FLAG=-rodent
 SEQ2_SMSK=/scratch/mus/mm5/linSpecRep.notInHuman
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=30000000
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/blastz.mm5
 
 DEF=$BASE/DEF
 RAW=$BASE/raw
 CDBDIR=$BASE
 SEQ1_LEN=$BASE/S1.len
 SEQ2_LEN=$BASE/S2.len
 '_EOF_'
     # << this line keeps emacs coloring happy
 
 
     # prepare first cluster run
     ssh kk
     cd /cluster/data/hg17/bed/blastz.mm5
     /cluster/data/hg17/jkStuff/BlastZ_run0.sh
     cd run.0
     para try, check, push, check, ....
 # Completed: 44330 of 44330 jobs
 # CPU time in finished jobs:   16250628s  270843.80m  4514.06h  188.09d  0.515 y
 # IO & Wait Time:                387936s    6465.60m   107.76h    4.49d  0.012 y
 # Average job time:                 375s       6.26m     0.10h    0.00d
 # Longest job:                     4417s      73.62m     1.23h    0.05d
 # Submission to last job:         43754s     729.23m    12.15h    0.51d
 
     #	the file server to its knees.  Run this on the small cluster.
     ssh kki
     cd /cluster/data/hg17/bed/blastz.mm5
     /cluster/data/hg17/jkStuff/BlastZ_run1.sh
     cd run.1
     para try, check, push, etc ...
 # Completed: 341 of 341 jobs
 # CPU time in finished jobs:       2189s      36.48m     0.61h    0.03d  0.000 y
 # IO & Wait Time:                  7714s     128.57m     2.14h    0.09d  0.000 y
 # Average job time:                  29s       0.48m     0.01h    0.00d
 # Longest job:                      165s       2.75m     0.05h    0.00d
 # Submission to last job:           830s      13.83m     0.23h    0.01d
 
 
     #	Third cluster run to convert lav's to axt's
     #	Does not work on kki since /scratch on the iservers is not the
     #	same as /scratch on the other clusters.
     ssh kk
     cd /cluster/data/hg17/bed/blastz.mm5
     /cluster/data/hg17/jkStuff/BlastZ_run2.sh
     cd run.2
     para try, check, push, etc ...
 # Completed: 45 of 46 jobs
 # Crashed: 1 jobs
 # CPU time in finished jobs:       1638s      27.30m     0.46h    0.02d  0.000 y
 # IO & Wait Time:                 12068s     201.13m     3.35h    0.14d  0.000 y
 # Average job time:                 305s       5.08m     0.08h    0.00d
 # Longest job:                     1124s      18.73m     0.31h    0.01d
 # Submission to last job:          2519s      41.98m     0.70h    0.03d
     #	chr19 takes too long, the axtSort becomes too large and the poor
     #	node ends up swapping forever.  When you are down to that last
     #	job running, stop it and go to kolossus.
     #	Adjusting the location of the nib directories, and fixing the
     #	MACHTYPE on the commands in the blastz script:
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.mm5
     sed -e "s/i386/x86_64/g" /cluster/bin/scripts/blastz-chromlav2axt > \
 	x86_64-chromlav2axt
     chmod +x x86_64-chromlav2axt
     time ./x86_64-chromlav2axt \
 	/cluster/data/hg17/bed/blastz.mm5/lav/chr19 \
 	/cluster/data/hg17/bed/blastz.mm5/axtChrom/chr19.axt \
 	/cluster/bluearc/scratch/hg/gs.18/build35/bothMaskedNibs \
 	/cluster/bluearc/scratch/mus/mm5/softNib
     #	real    7m41.719s
     #	user    2m2.850s
     #	sys     0m23.070s
 
     # translate sorted axt files into psl
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm5
     mkdir -p pslChrom
     set tbl = "blastzMm5"
     foreach f (axtChrom/chr*.axt)
       set c=$f:t:r
       echo "Processing chr $c"
       /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
     end
     #	This takes more than an hour.  You can shorten this by changing
     #	that command to a simple echo, put the results into a file,
     #	split the file into four parts and run the four files as shell
     #	scripts on eieio to have four processes running at the same
     #	time.  Load on eieio gets up to about 20 which is reasonable.
 
     # Load database tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm5/pslChrom
     bash		#	for tcsh users
     for F in chr*_blastzMm5.psl
     do
 	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${F}
 	echo "${F} done"
     done
     # this is a 40 minute job
     # exit bash if you are tcsh
 
     # featureBits on blastzMm3 or 4 will not work on hgwdev, runs out of
     # memory.  But if you reset your ~/.hg.conf to use the read-only
     #	user and contact the hgwdev host, then use the x86_64 featureBits
     # featureBits hg16 blastzMm5
     # 1056761609 bases of 2865248791 (36.882%) in intersection
     # featureBits hg17 blastzMm5
     #	1052077141 bases of 2866216770 (36.706%) in intersection
     # featureBits hg17 blastzMm4
     #  1056201417 bases of 2866216770 (36.850%) in intersection
 
 # CHAIN MM5 BLASTZ (DONE - 2004-07-02 - Hiram)
 
 # The axtChain is best run on the small kluster, or the kk9 kluster
     ssh kki
     mkdir -p /cluster/data/hg17/bed/blastz.mm5/axtChain/run1
     cd /cluster/data/hg17/bed/blastz.mm5/axtChain/run1
     mkdir out chain
 
     ls -1S /cluster/data/hg17/bed/blastz.mm5/axtChrom/*.axt > input.lst
     cat << '_EOF_' > gsub
 #LOOP
 doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} out/$(root1).out
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
 #  May need -minScore=5000 for all chroms if chr19 won't finish on kolossus
 
     cat << '_EOF_' > doChain
 #!/bin/csh
 axtChain $1 /iscratch/i/gs.18/build35/bothMaskedNibs \
 	/iscratch/i/mus/mm5/softNib $2 > $3
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod a+x doChain
 
     # 46 jobs
     gensub2 input.lst single gsub jobList
     para create jobList
     para try
     para push # ... etc ...
 # Completed: 46 of 46 jobs
 # CPU time in finished jobs:       4856s      80.94m     1.35h    0.06d  0.000 y
 # IO & Wait Time:                 20083s     334.71m     5.58h    0.23d  0.001 y
 # Average job time:                 542s       9.04m     0.15h    0.01d
 # Longest job:                     2929s      48.82m     0.81h    0.03d
 # Submission to last job:          2929s      48.82m     0.81h    0.03d
 
 
     # now on the file server, sort chains
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm5/axtChain
     time chainMergeSort run1/chain/*.chain > all.chain
     #	real    8m42.853s
     #	user    5m59.100s
     #	sys     0m40.320s
 
     time chainSplit chain all.chain
     #	real    10m52.224s
     #	user    5m52.360s
     #	sys     0m34.870s
 
     # optionally: rm run1/chain/*.chain
 
     # Load chains into database
     # next machine
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm5/axtChain/chain
     bash	#	for tcsh users
     for i in *.chain
     do
         c=${i/.chain/}
         hgLoadChain hg17 ${c}_chainMm5 $i
         echo done $c
     done
     # exit bash if you are tcsh
     #	This is a 50 minute job
 
     #	featureBits hg17 chainMm5
     #	2834490112 bases of 2866216770 (98.893%) in intersection
     #	featureBits hg17 chainMm4
     #	2829135227 bases of 2866216770 (98.706%) in intersection
     #	featureBits hg16 chainMm4
     #	2828363353 bases of 2865248791 (98.713%) in intersection
 
 
 # NET MM5 (DONE - 2004-07-02 - Hiram)
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm5/axtChain
     mkdir preNet
     cd chain
     bash	#	for tcsh users
     for i in *.chain
     do
       echo preNetting $i
       /cluster/bin/i386/chainPreNet $i /cluster/data/hg17/chrom.sizes \
                         /cluster/data/mm5/chrom.sizes ../preNet/$i
     done
     # exit bash if you are tcsh
     #	15 minute job
 
     cd ..
     mkdir n1
     cd preNet
     bash	#	for tcsh users
     for i in *.chain
     do
       n=${i/.chain/}.net
       echo primary netting $i $n
       /cluster/bin/i386/chainNet $i -minSpace=1 /cluster/data/hg17/chrom.sizes \
 	/cluster/data/mm5/chrom.sizes ../n1/$n /dev/null
     done
     # exit bash if you are tcsh
     #	9 minute job
 
     cd ..
     cat n1/*.net | /cluster/bin/i386/netSyntenic stdin hNoClass.net
     #	memory usage 2546110464, utime 16327 s/100, stime 3546
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm5/axtChain
     time netClass hNoClass.net hg17 mm5 mouse.net \
 	-tNewR=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInMouse \
 	-qNewR=/cluster/bluearc/scratch/mus/mm5/linSpecRep.notInHuman
     #	real    16m38.098s
     #	user    11m38.490s
     #	sys     1m48.470s
 
     # If things look good do
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm5/axtChain
     rm -r n1 hNoClass.net
     # Make a 'syntenic' subset of these with
     time netFilter -syn mouse.net > mouseSyn.net
     #	real    12m3.701s
     #	user    8m44.180s
     #	sys     1m1.610s
 
     # Load the nets into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm5/axtChain
     netFilter -minGap=10 mouse.net |  hgLoadNet hg17 netMm5 stdin
     netFilter -minGap=10 mouseSyn.net | hgLoadNet hg17 syntenyNetMm5 stdin
 
     # check results
     #	featureBits hg17 netMm5
     #	2830625630 bases of 2866216770 (98.758%) in intersection
 
     #	featureBits hg17 netMm4
     #	2824272033 bases of 2866216770 (98.537%) in intersection
     # featureBits hg16 netMm5
     #	2823565051 bases of 2865248791 (98.545%) in intersection
 
     # featureBits hg17 syntenyNetMm5
     #	2799194300 bases of 2866216770 (97.662%) in intersection
     # featureBits hg17 syntenyNetMm4
     #	2785830955 bases of 2866216770 (97.195%) in intersection
     # featureBits hg16 syntenyNetMm5
     #	2786960572 bases of 2865248791 (97.268%) in intersection
 
     # Add entries for net and chain to mouse/hg17 trackDb
 
     # make net
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm5/axtChain
     mkdir mouseNet
     time netSplit mouse.net mouseNet
     #	real    11m45.243s
     #	user    8m48.490s
     #	sys     1m13.490s
 
     #	extract axt's from net, and convert to maf's
     # NOTE: Redo the net axt's and maf's using 8/05 netToAxt
     # in order to remove overlaps (2005-08-16 kate)
     ssh kkstore02
     cd /cluster/data/hg17/bed/blastz.mm5/axtChain
     mkdir ../axtNet ../mafNet
 cat > makeMaf.csh << '_EOF_'
 #!/bin/csh -ef
     foreach f (mouseNet/chr*.net)
         set c = $f:t:r
         echo "netToAxt: $c.net -> $c.axt"
         rm -f ../axtNet/$c.axt
         netToAxt mouseNet/$c.net chain/$c.chain \
 	    /cluster/data/hg17/nib /cluster/data/mm5/nib stdout | \
 	    axtSort stdin ../axtNet/$c.axt
         axtToMaf ../axtNet/$c.axt \
             /cluster/data/hg17/chrom.sizes /cluster/data/mm5/chrom.sizes \
             ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=mm5.
 	echo "Complete: $c.net -> axtNet/$c.axt -> mafNet/$c.maf"
     end
 '_EOF_'
 # << for emacs
     csh makeMaf.csh >&! makeMaf.log &
     tail -100f makeMaf.log
     mkdir -p /cluster/bluearc/hg17/mafNet
     cp -rp ../mafNet /cluster/bluearc/hg17/mafNet/mm5
 
     ssh hgwdev
     mkdir /cluster/data/hg17/bed/blastz.mm5/axtBest
     cd /cluster/data/hg17/bed/blastz.mm5/axtBest
     ln -s ../axtNet/chr*.axt .
 
     # copy net axt's to download area
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm5/axtNet
     mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/axtNet
     cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/axtNet
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/axtNet
     gzip *.axt
     # add README.txt file to dir (use previous assembly's copy as template)
     #	32 minute gzip
     
 
     #  Convert those axt files to psl
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm5
     mkdir pslBest
     foreach a (axtBest/chr*.axt)
 	set c=$a:t:r
 	echo -n "processing $c.axt -> ${c}_blastzBestMm5.psl ..."
     /cluster/bin/i386/axtToPsl axtBest/${c}.axt \
 	S1.len S2.len pslBest/${c}_blastzBestMm5.psl
 	echo "Done"
     end
 
     # Load tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm5/pslBest
     for I in chr*BestMm5.psl
     do
 	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
 	echo "done ${I}"
     done
 
      # check results
     # featureBits hg17 blastzBestMm5
     #	1013348528 bases of 2866216770 (35.355%) in intersection
     # featureBits hg17 blastzBestMm4
     #	1017319919 bases of 2866216770 (35.493%) in intersection
     # featureBits hg16 blastzBestMm5
     #	996722004 bases of 2865248791 (34.787%) in intersection
 
     # Make /gbdb links and add them to the axtInfo table:
      mkdir -p /gbdb/hg17/axtBest/Mm5
      cd /gbdb/hg17/axtBest/Mm5
      ln -s /cluster/data/hg17/bed/blastz.mm5/axtNet/chr*.axt .
      cd /cluster/data/hg17/bed/blastz.mm5/axtNet
      rm -f axtInfoInserts.sql
      foreach f (/gbdb/hg17/axtBest/Mm5/chr*.axt)
        set chr=$f:t:r
        echo "INSERT INTO axtInfo (species, alignment, chrom, fileName) \
                 VALUES ('mm5','Blastz Best in Genome','$chr','$f');" \
          >>! axtInfoInserts.sql
      end
     hgsql hg17 < ~/kent/src/hg/lib/axtInfo.sql
     #	table axtInfo may already exist, ignore create error.
     hgsql hg17 < axtInfoInserts.sql
 
     # REDO: replace downloadable axtNet's to remove overlaps (2005-09-12 kate)
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsMm5
     mv axtNet axtNet.old
     mkdir axtNet
     cd axtNet
     cp /cluster/data/hg17/bed/blastz.mm5/axtNet/*.axt .
     nice gzip *.axt
     md5sum *.axt.gz > md5sum.txt
 
 
 # HG17 TO MM5 LIFTOVER CHAIN (DONE 1/6/05 Andy)
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.mm5/axtChain
     mkdir over
     for file in chain/*.chain; do
        chrom=`basename $file .chain`
        netChainSubset mouseNet/$chrom.net chain/$chrom.chain over/$chrom.over
        cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToMm5.chain
     done
     rm -rf over/
     ssh hgwdev
     mkdir /usr/local/apache/htdocs/goldenPath/hg17/liftOver
     cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver
     cp /cluster/data/hg17/bed/liftOver/hg17ToMm5.chain .
     gzip hg17ToMm5.chain
     ln -s /cluster/data/hg17/bed/liftOver/hg17ToMm5.chain /gbdb/hg17/liftOver/hg17ToMm5.over.chain
     hgAddLiftOverChain -multiple hg17 mm5
 
 # HG17 TO CANFAM1 LIFTOVER CHAIN (DONE 1/7/05 Andy)
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.canFam1/axtChain
     mkdir over
     for file in chain/*.chain; do
        chrom=`basename $file .chain`
        netChainSubset net/$chrom.net chain/$chrom.chain over/$chrom.over
        cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToCanFam1.chain
     done
     rm -rf over/
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver
     cp /cluster/data/hg17/bed/liftOver/hg17ToCanFam1.chain .
     gzip hg17ToCanFam1.chain
     ln -s /cluster/data/hg17/bed/liftOver/hg17ToCanFam1.chain /gbdb/hg17/liftOver/hg17ToCanFam1.over.chain
     hgAddLiftOverChain -multiple hg17 canFam1
 
 # HG17 TO PANTRO1 LIFTOVER CHAIN (DONE 1/20/05 Andy)
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.panTro1/axtChain
     mkdir over
     for file in chain/*.chain; do
        chrom=`basename $file .chain`
        netChainSubset net/$chrom.net chain/$chrom.chain over/$chrom.over
        cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToPanTro1.chain
     done
     rm -rf over/
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver
     cp /cluster/data/hg17/bed/liftOver/hg17ToPanTro1.chain .
     gzip hg17ToPanTro1.chain
     ln -s /cluster/data/hg17/bed/liftOver/hg17ToPanTro1.chain /gbdb/hg17/liftOver/hg17ToPanTro1.over.chain
     hgAddLiftOverChain -multiple hg17 panTro1   
 
 # HG17 TO RN3 LIFTOVER CHAIN (DONE 3/1/05 Andy)
     #ssh kolossus
     #cd /cluster/data/hg17/bed/blastz.rn3/axtChain
     #mkdir over
     #for file in chain/*.chain; do
     #   chrom=`basename $file .chain`
     #   netChainSubset ratNet/$chrom.net chain/$chrom.chain over/$chrom.over
     #   cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToRn3.chain
     #done
     #rm -rf over/
     # Oh fancy that, there's already a hg17ToRn3.over.chain in the /cluster/data/hg17/bed/liftOver
     # directory generated by Angie.  
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver 
     cp /cluster/data/hg17/bed/liftOver/hg17ToRn3.over.chain .
     gzip hg17ToRn3.over.chain
     ln -s /cluster/data/hg17/bed/liftOver/hg17ToRn3.over.chain /gbdb/hg17/liftOver/hg17ToRn3.over.chain
     hgAddLiftOverChain -multiple hg17 rn3   
 
 # HG17 TO GALGAL2 LIFTOVER CHAIN (DONE 3/1/05 Andy)
     # OK there's already a /cluster/data/hg17/bed/liftOver/hg17ToGalGal2.over.chain file generated
     # by Angie.
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver 
     cp /cluster/data/hg17/bed/liftOver/hg17ToGalGal2.over.chain .
     gzip hg17ToGalGal2.over.chain
     ln -s /cluster/data/hg17/bed/liftOver/hg17ToGalGal2.over.chain /gbdb/hg17/liftOver/hg17ToGalGal2.over.chain
     hgAddLiftOverChain -multiple hg17 galGal2       
 
 # HG17 TO MONDOM1 LIFTOVER CHAIN (DONE 3/1/05 Andy)
     ssh kksilo
     cd /cluster/data/monDom1/bed/zb.hg17/axtChain
     netSplit human.net.gz net
     ssh kolossus
     cd /cluster/data/monDom1/bed/zb.hg17/axtChain
     mkdir over
     for file in chain/*.chain.gz; do
        chrom=`basename $file .chain.gz`
        netChainSubset net/$chrom.net chain/$chrom.chain.gz over/$chrom.over
        cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToMonDom1.chain
     done
     rm -rf over/ net/
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver 
     cp /cluster/data/hg17/bed/liftOver/hg17ToMonDom1.chain .
     gzip hg17ToMonDom1.chain
     ln -s /cluster/data/hg17/bed/liftOver/hg17ToMonDom1.chain /gbdb/hg17/liftOver/hg17ToMonDom1.over.chain
     hgAddLiftOverChain -multiple hg17 monDom1 
 
 # HG17 TO DANRER2 LIFTOVER CHAIN (DONE 3/2/05 Andy)
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
     chainSplit chain all.chain.gz
     netSplit zfishdanRer2.net.gz net
     mkdir over
     # FAILED STEPS:
     #for file in chain/*.chain; do
     #   chrom=`basename $file .chain`
     #   netChainSubset net/$chrom.net chain/$chrom.chain over/$chrom.over
     #   cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToDanRer2.chain
     #done
     # Error:
 #read 28019 of 28019 chains in chain/chr1.chain
 #Processing chr1
 #netChainSubset: netChainSubset.c:55: writeChainPart: Assertion `subChain != ((void *)0)' failed.
     # OK instead of using the ones in the chain/ subdir, I'm using the ones in
     # the chainAR/ subdir.  These chain files had an additional step in the process of making
     # them: Rachel used the chainAntiRepeat program.  
     for file in chain/*.chain; do
        chrom=`basename $file .chain`
        if [ $chrom = "chr1" ]; then
            netChainSubset net/$chrom.net chainAR/$chrom.chain over/$chrom.over
        else
            netChainSubset net/$chrom.net chainAR/$chrom.chain.gz over/$chrom.over          
        fi
        cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToDanRer2.chain
     done  
     rm -rf over/
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver 
     cp /cluster/data/hg17/bed/liftOver/hg17ToDanRer2.chain .
     gzip hg17ToDanRer2.chain
     ln -s /cluster/data/hg17/bed/liftOver/hg17ToDanRer2.chain /gbdb/hg17/liftOver/hg17ToDanRer2.over.chain
     hgAddLiftOverChain -multiple hg17 danRer2 
     
 
 # HG17 TO TETNIG1 LIFTOVER CHAIN (DONE 3/1/05 Andy)
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
     mkdir over
     for file in chain/*.chain; do
        chrom=`basename $file .chain`
        netChainSubset tetraodonNet/$chrom.net chain/$chrom.chain over/$chrom.over
        cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToTetNig1.chain
     done
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver 
     cp /cluster/data/hg17/bed/liftOver/hg17ToTetNig1.chain .
     gzip hg17ToTetNig1.chain
     ln -s /cluster/data/hg17/bed/liftOver/hg17ToTetNig1.chain /gbdb/hg17/liftOver/hg17ToTetNig1.over.chain
     hgAddLiftOverChain -multiple hg17 tetNig1      
 
 # HG17 TO BOSTAU1 LIFTOVER CHAIN (DONE Mar. 18, 2004, Heather)
 
     ssh kolossus
     cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
     mkdir over
     for file in chain/*.chain; do
        chrom=`basename $file .chain`
        netChainSubset net/$chrom.net chain/$chrom.chain over/$chrom.over
        cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToBosTau1.chain
     done
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver 
     cp /cluster/data/hg17/bed/liftOver/hg17ToBosTau1.chain .
     gzip hg17ToBosTau1.chain
     ln -s /cluster/data/hg17/bed/liftOver/hg17ToBosTau1.chain /gbdb/hg17/liftOver/hg17ToBosTau1.over.chain
     hgAddLiftOverChain -multiple hg17 bosTau1      
 
 # HG17 TO XENTRO1 LIFTOVER CHAIN (DONE 7/5/05 Andy)
     ssh kolossus
     cd /cluster/data/xenTro1/bed/zb.hg17/axtChain
     mkdir chain net over
     chainSplit chain all.chain
     netSplit human.net net
     for file in chain/*.chain; do
        chrom=`basename $file .chain`
        netChainSubset net/$chrom.net chain/$chrom.chain over/$chrom.over
        cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToXenTro1.chain
     done
     rm -rf over/ chain/ net/
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver
     cp /cluster/data/hg17/bed/liftOver/hg17ToXenTro1.chain .
     gzip hg17ToXenTro1.chain
     ln -s /cluster/data/hg17/bed/liftOver/hg17ToXenTro1.chain /gbdb/hg17/liftOver/hg17ToXenTro1.over.chain
     hgAddLiftOverChain -multiple hg17 xenTro1   
 
 # ADD CHAIN AND NET TO VSMM5 AND VSRN3 DOWNLOAD AREAS (DONE 8/5/04 angie)
     ssh hgwdev
     cp -p /cluster/data/hg17/bed/blastz.mm5/axtChain/all.chain.gz \
       /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/mouse.chain.gz
     cp -p /cluster/data/hg17/bed/blastz.mm5/axtChain/mouse.net.gz \
       /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/mouse.net.gz
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsMm5
     md5sum *.gz */*.gz > md5sum.txt
     # Update the README.txt
     cp -p /cluster/data/hg17/bed/blastz.rn3/axtChain/all.chain.gz \
       /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/rat.chain.gz
     cp -p /cluster/data/hg17/bed/blastz.rn3/axtChain/rat.net.gz \
       /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/rat.net.gz
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsRn3
     md5sum *.gz */*.gz > md5sum.txt
     # Update the README.txt
 
 # ADD CHAIN AND NET TO VSHG17 DOWNLOAD AREAS (DONE Sept. 8th, 2004, heather)
     ssh hgwdev
     cp -p /cluster/data/mm5/bed/blastz.hg17/axtChain/all.chain.gz \
       /usr/local/apache/htdocs/goldenPath/mm5/vsHg17/human.chain.gz
     cp -p /cluster/data/mm5/bed/blastz.hg17/axtChain/human.net.gz \
       /usr/local/apache/htdocs/goldenPath/mm5/vsHg17/human.net.gz
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsRn3
     md5sum *.gz */*.gz > md5sum.txt
     # Update the README.txt
 
 
 # SWAP BLASTZ ZEBRAFISH-HUMAN (danRer1-hg17) to HUMAN-ZEBRAFISH (hg17-danRer1)
 # USE RESCORED ALIGNMENTS (see makeDanRer1.doc)
 # (DONE, 2004-06-22, hartera)
 # CONVERT AXTs TO PSL AND LOAD INTO DATABASE (DONE, 2004-07-08, hartera)
     ssh kolossus
     mkdir /cluster/data/hg17/bed/blastz.danRer1.swap
     cd /cluster/data/hg17/bed/blastz.danRer1.swap
     # use rescored axtChrom from blastzHg17 on danRer1
     set aliDir = /cluster/data/danRer1/bed/blastz.hg17
     cp $aliDir/S1.len S2.len
     cp $aliDir/S2.len S1.len
     mkdir unsorted axtChrom
     cat $aliDir/axtChrom/chr*.axt \
     | axtSwap stdin $aliDir/S1.len $aliDir/S2.len stdout \
     | axtSplitByTarget stdin unsorted
     # Sort the shuffled .axt files.
     foreach f (unsorted/*.axt)
       echo sorting $f:t:r
       axtSort $f axtChrom/$f:t
     end
     du -sh $aliDir/axtChrom unsorted axtChrom
 # 19G     /cluster/data/danRer1/bed/blastz.hg17/axtChrom
 # 19G     unsorted
     rm -r unsorted
    # translate sorted axt files into psl
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.danRer1.swap
     mkdir -p pslChrom
     set tbl = "blastzDanRer1"
     foreach f (axtChrom/chr*.axt)
       set c=$f:t:r
       echo "Processing chr $c"
       /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
     end
     # Load database tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.danRer1.swap/pslChrom
     foreach f (./*.psl)
       /cluster/bin/i386/hgLoadPsl hg17 $f
       echo "$f Done"
     end
 
 # CHAIN ZEBRAFISH (danRer1) BLASTZ (DONE, 2004-06-23, hartera)
     # Run axtChain on little cluster
     ssh kki
     cd /cluster/data/hg17/bed/blastz.danRer1.swap
     mkdir -p axtChain/run1
     cd axtChain/run1
     mkdir out chain
     ls -1S /cluster/data/hg17/bed/blastz.danRer1.swap/axtChrom/*.axt \
                     > input.lst
     cat << '_EOF_' > gsub
 #LOOP
 doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
 #ENDLOOP
 '_EOF_'
 
     # << this line makes emacs coloring happy
 
 # Reuse gap penalties from hg16 vs chicken run.
     cat << '_EOF_' > ../../chickenHumanTuned.gap
 tablesize^V     11
 smallSize^V     111
 position^V      1^V     2^V     3^V     11^V    111^V   2111^V  12111^V 32111^V 72111^V 152111^V        252111
 qGap^V  325^V   360^V   400^V   450^V   600^V   1100^V  3600^V  7600^V  15600^V 31600^V 56600
 tGap^V  325^V   360^V   400^V   450^V   600^V   1100^V  3600^V  7600^V  15600^V 31600^V 56600
 bothGap^V       625^V   660^V   700^V   750^V   900^V   1400^V  4000^V  8000^V  16000^V 32000^V 57000
 '_EOF_'
     # << this line makes emacs coloring happy
 
 cat << '_EOF_' > doChain
 #!/bin/csh
 axtFilter $1 \
 | axtChain -scoreScheme=/cluster/data/blastz/HoxD55.q \
                       -linearGap=../../chickenHumanTuned.gap \
                       -minScore=5000 stdin \
     /iscratch/i/gs.18/build35/bothMaskedNibs \
     /iscratch/i/danRer1/nib $2 > $3
 '_EOF_'
     chmod a+x doChain
     gensub2 input.lst single gsub jobList
     para create jobList
     para try, check, push, check...
 # para time
 # Completed: 45 of 46 jobs
 # Crashed: 1 jobs
 # CPU time in finished jobs:       3559s      59.32m     0.99h    0.04d  0.000 y
 # IO & Wait Time:                   934s      15.56m     0.26h    0.01d  0.000 y
 # Average job time:                 100s       1.66m     0.03h    0.00d
 # Longest job:                      502s       8.37m     0.14h    0.01d
 # Submission to last job:          2969s      49.48m     0.82h    0.03d
 # chr19.axt crashed - out of memory so try again on kolossus
 
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain/run1
     # need to use nibs on bluearc as iscratch not accessible to kolossus
 cat << '_EOF_' > doChain2
 #!/bin/csh
 axtFilter $1 \
 | axtChain -scoreScheme=/cluster/data/blastz/HoxD55.q \
                       -linearGap=../../chickenHumanTuned.gap \
                       -minScore=5000 stdin \
     /cluster/bluearc/hg17/bothMaskedNibs \
     /cluster/bluearc/danRer1/nib $2 >& $3
 '_EOF_'
     chmod +x doChain2
 
     doChain2 \
          /cluster/data/hg17/bed/blastz.danRer1.swap/axtChrom/chr19.axt \
          chain/chr19.chain out/chr19.out
     # now on the cluster server, sort chains
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
     chainMergeSort run1/chain/*.chain > all.chain
     chainSplit chain all.chain
 
     # Load chains into database
     # next machine
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain/chain
     foreach i (*.chain)
         set c = $i:r
         hgLoadChain hg17 ${c}_chainDanRer1 $i
         echo done $c
     end
 # tried minScore = 1000 and minScore = 10000 for axtChain
 # minScore = 5000 was best for reducing low scoring chains but not reducing 
 # overlap with refGene CDS too much
 
 # NET ZEBRAFISH (danRer1) BLASTZ (DONE, 2004-06-24, hartera)
 # REMAKE NET WITHOUT ANCIENT REPEATS (DONE, 2004-07-07, hartera)
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
     mkdir preNet
     cd chain
     foreach i (*.chain)
        echo preNetting $i
        /cluster/bin/i386/chainPreNet $i ../../S1.len ../../S2.len \
                                      ../preNet/$i
     end
 
     cd ..
     mkdir n1
     cd preNet
     foreach i (*.chain)
       set n = $i:r.net
       echo primary netting $i
       /cluster/bin/i386/chainNet $i -minSpace=1 ../../S1.len ../../S2.len \
                                  ../n1/$n /dev/null
     end
 
     cd ..
     mkdir n1
     cd preNet
     foreach i (*.chain)
       set n = $i:r.net
       echo primary netting $i
       /cluster/bin/i386/chainNet $i -minSpace=1 ../../S1.len ../../S2.len \
                                  ../n1/$n /dev/null
     end
 
     cd ..
     cat n1/*.net | /cluster/bin/i386/netSyntenic stdin noClass.net
     # memory usage 149086208, utime 868 s/100, stime 173                                                                            
     # Add classification info using db tables:
     cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
     # netClass looks for ancient repeats in one of the databases
     # hg17 has this table - hand-curated by Arian
     # this is only for human rodent comparisons so use -noAr option
     mkdir -p /cluster/bluearc/danRer1/linSpecRep.notInHuman
     mkdir -p /cluster/bluearc/hg17/linSpecRep.notInZebrafish
     cp /iscratch/i/gs.18/build35/linSpecRep.notInZebrafish/* \
        /cluster/bluearc/hg17/linSpecRep.notInZebrafish
     cp /iscratch/i/danRer1/linSpecRep.notInHuman/* \
        /cluster/bluearc/danRer1/linSpecRep.notInHuman
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
     # add -noAr option
     # mkdir old
     # mv zebrafish.net ./old/zebrafish.net.old
     time netClass noClass.net hg17 danRer1 zebrafish.net \
         -tNewR=/cluster/bluearc/hg17/linSpecRep.notInZebrafish \
         -qNewR=/cluster/bluearc/danRer1/linSpecRep.notInHuman -noAr
     # 83.410u 43.650s 3:09.94 66.8%   0+0k 0+0io 198pf+0w
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
     netFilter -minGap=10 zebrafish.net | hgLoadNet hg17 netDanRer1 stdin
 
 # EXTRACT AXT'S AND MAF'S FROM ZEBRAFISH (danRer1) NET 
 # (DONE, 2004-06-24, hartera) used net where hg17 ancient Repeat table used
 # sorted axts and remade mafs as multiz needs axts to be sorted
 # (DONE, 2004-06-25, kate)
 # Redone to fix overlaps using 8/05 axtToNet (2005-08-16 kate)
 # Replace bad chr5 axtNet and mafNet (2006-01-05 kate) 
     ssh eieio
     # create axts
     cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
     netSplit zebrafish.net zebrafishNet
     mkdir -p ../axtNet ../mafNet
 cat > makeMaf.csh << 'EOF'
     foreach f (zebrafishNet/chr*.net)
         set c = $f:t:r
         echo $c
         netToAxt zebrafishNet/$c.net chain/$c.chain \
         /cluster/data/hg17/nib /cluster/data/danRer1/nib stdout | \
                axtSort stdin ../axtNet/$c.axt
         axtToMaf ../axtNet/$c.axt \
             /cluster/data/hg17/chrom.sizes /cluster/data/danRer1/chrom.sizes \
             ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=danRer1.
     end
 'EOF'
     csh makeMaf.csh >&! makeMaf.log &
     tail -100f makeMaf.log
     mkdir -p /cluster/bluearc/hg17/mafNet
     cp -rp ../mafNet /cluster/bluearc/hg17/mafNet/danRer1
 
 
 # BLASTZ ZEBRAFISH (danRer1) CLEAN UP (DONE, 2004-07-19, hartera)
 # FURTHER CLEANUP (hartera, 2006-09-01, hartera)
      ssh eieio
      cd /cluster/data/hg17/bed/blastz.danRer1.swap
      nice rm axtChain/run1/chain/* &
      nice rm -fr axtChain/n1 axtChain/hNoClass.net &
      nice gzip axtChrom/* pslChrom/* axtChain/all.chain axtChain/*.net &
      # further cleanup (2006-09-01, hartera)
      ssh kkstore02
      cd /cluster/data/hg17/bed/blastz.danRer1.swap
      rm -r axtNet.old axtNet.unsorted mafNet
      cd axtChain
      rm hist*
      # remove chains and nets directories. These can be reconstructed with
      # all.chain.gz and zebrafish.net.gz
      rm -r old chain zebrafishNet preNet
      rm noClass.net.gz
      cd ..
      rm pslChrom/psl.tab.gz
 
 # ZEBRAFISH DANRER1 DOWNLOADS (WORKING 2004-09-17 kate)
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtNet
     gzip *.axt
 
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17
     mkdir -p vsDanRer1
     cd vsDanRer1
     cp /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain/all.chain.gz zebrafish.chain.gz
     cp /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain/zebrafish.net.gz .
     md5sum *.gz > md5sum.txt
     mkdir -p axtNet
     cd axtNet
     cp /cluster/data/hg17/bed/blastz.danRer1.swap/axtNet/*.axt.gz .
     md5sum *.gz > md5sum.txt
     # Copy and edit README.txt
       
  
 # MAKING MOUSE SYNTENY (DONE - 2004-07-03 - Hiram)
 
 ssh hgwdev
 mkdir /cluster/data/hg17/bed/syntenyMm5
 cd /cluster/data/hg17/bed/syntenyMm5
 
 # Copy all the needed scripts from /cluster/data/hg16/bed/syntenyMm3
 cp -p /cluster/data/hg17/bed/syntenyRn3/*.pl .
 
 ./syntenicBest.pl -db=hg17 -table=blastzBestMm5
 ./smooth.pl
 ./joinsmallgaps.pl
 ./fillgap.pl -db=hg17 -table=blastzBestMm5
 ./synteny2bed.pl
     #	The five commands above
     #	real    209m28.161s
     #	user    0m21.040s
     #	sys     0m4.100s
 
 #	Used to load this in syntenyMm5, but that type is misleading to
 #	the table browser and fails the checkTableCoords check.
 #	Better to use this ensRatMusHom type:
 #	Need a new name here for the Mm5 to not conflict with Rn3
 sed -e 's/ensPhusionBlast/ensRatMm5Hom/g' \
       $HOME/kent/src/hg/lib/ensPhusionBlast.sql \
       > ensRatMm5Hom.sql
 hgLoadBed hg17 ensRatMm5Hom ucsc100k.bed -sqlTable=ensRatMm5Hom.sql
     #	featureBits hg17 ensRatMm5Hom
     #	2649530748 bases of 2866216770 (92.440%) in intersection
     #	featureBits hg17 ensRatMm4Hom
     #	2549307611 bases of 2866216770 (88.943%) in intersection
     #	featureBits hg16 syntenyMm5
     #	2560252977 bases of 2865248791 (89.355%) in intersection
 
 # MAKING MOUSE AXTTIGHT FROM AXTBEST (DONE - 2004-07-02 - Hiram)
     # After creating axtBest alignments above, use subsetAxt to get axtTight:
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm5/axtNet
     mkdir -p ../axtTight
     bash	#	for tcsh users
     for I in *.axt
     do
       echo $I
       subsetAxt  $I ../axtTight/$I \
 	~kent/src/hg/mouseStuff/subsetAxt/coding.mat 3400
     done
     # exit bash if you are tcsh
     #	An 8 minute job
 
     # translate to psl
     cd ../axtTight
     mkdir ../pslTight
     bash	#	for tcsh users
     for I in *.axt
     do
       C=${I/.axt/}
       axtToPsl $I ../S1.len ../S2.len ../pslTight/${C}_blastzTightMm5.psl
       echo "Done: $I -> ${C}_blastzTightMm5.psl"
     done
     # exit bash if you are tcsh
 
     # Load tables into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm5/pslTight
     for I in chr*TightMm5.psl
     do
 	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
 	echo "done ${I}"
     done
 
     #	Compare results with previous assembly:
     #	featureBits hg17 blastzTightMm5
     #	165862935 bases of 2866216770 (5.787%) in intersection
     #	featureBits hg17 blastzTightMm4
     #	166569246 bases of 2866216770 (5.811%) in intersection
     #	featureBits hg16 blastzTightMm5
     #	162641577 bases of 2865248791 (5.676%) in intersection
 
     # copy  axt's to download area
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.mm5/axtTight
     mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/axtTight
     cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/axtTight
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/axtTight
     gzip *.axt
     # add README.txt file to dir (use previous assembly's copy as template)
     #	4 minute gzip
 
 # BLASTZ MM5 CLEAN UP (DONE 2004-07-02 - Hiram)
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.mm5
     nice rm -rf raw &
     nice rm -fr axtChain/n1 axtChain/hNoClass.net &
     nice rm axtChain/run1/chain/* &
     nice gzip axtChrom/* pslChrom/* lav/*/* axtChain/all.chain axtChain/*.net &
 
 ##############################################################################
 #  MAKING BLASTZ SELF (DONE - 2004-07-14 - Hiram)
 
     # The procedure for lineage spec business with self is to simply
     # use the actual repeat masker output for this human assembly as
     # the lineage specific repeats for itself.  Thus, merely make
     # symlinks to the repeat masker out files and name them as expected
     # for blastz.  In this case they are called notInHuman but they
     # really mean InHuman.  Yes, it is confusing, but that's just the
     # nature of the game in this case.
 
     ssh eieio
     mkdir /cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInHuman
     cd /cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInHuman
     foreach f (../rmsk/*.fa.out)
 	set base = $f:t:r:r
 	echo $base.out.spec
 	ln -s $f $base.out.spec
     end
     #	Same thing done on iscratch
     #	Not worried about pushing this scratch yet, it will get done
     #	sometime later.  Using the actual /cluster/bluearc/scratch/
     #	location below.
 
     ssh kk
     mkdir /cluster/data/hg17/bed/blastzSelf.2004-07-01
     cd /cluster/data/hg17/bed
     ln -s blastzSelf.2004-07-01 blastzSelf
     cd blastzSelf
 
     cat << '_EOF_' > DEF
 # human vs. human
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/home/angie/schwartzbin:/cluster/home/kent/bin/i386
 
 ALIGN=blastz-run
 BLASTZ=blastz
 BLASTZ_H=2000
 BLASTZ_ABRIDGE_REPEATS=1
 
 # TARGET
 # Human
 SEQ1_DIR=/iscratch/i/gs.18/build35/bothMaskedNibs
 # not used
 SEQ1_RMSK=
 # not used
 SEQ1_FLAG=
 SEQ1_SMSK=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInHuman
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY
 # Human
 SEQ2_DIR=/iscratch/i/gs.18/build35/bothMaskedNibs
 # not currently used
 SEQ2_RMSK=
 # not currently used
 SEQ2_FLAG=
 SEQ2_SMSK=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInHuman
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=10000000
 SEQ2_LAP=10000
 
 BASE=/cluster/data/hg17/bed/blastzSelf
 
 DEF=$BASE/DEF
 RAW=$BASE/raw
 CDBDIR=$BASE
 SEQ1_LEN=$BASE/S1.len
 SEQ2_LEN=$BASE/S2.len
 '_EOF_'
     # << this line makes emacs coloring happy
 
     # prepare first cluster run
     ssh kk
     cd /cluster/data/hg17/bed/blastzSelf
     /cluster/data/hg17/jkStuff/BlastZ_run0.sh
     cd run.0
     para try, check, push, check, ....
     #	you need a -maxPush=200000 on this one, it is more than 100000
     #	jobs the default push limit.  Also be aware of maxQueue limits
     #	on the KK, may need something more than the default of 200000 if
     #	the KK is busy.
 XXX - running 2004-07-01 11:26
 
 
 ##############################################################################
 # LIFTOVER (DROP) CHAINS TO HG16 (IN PROGRESS 2005-01-03 kate)
 
    # swap hg16->hg17 chains 
 
  # LIFTOVER (DROP) CHAINS TO HG16 (IN PROGRESS 2004-07-07 kate)
 
     # run alignment
     # NOTE: split hg16 to /iscratch/i is doc'ed in makeHg16.doc
     ssh kk
     cd /cluster/data/hg17
     makeLoChain-align hg17 /scratch/hg/gs.18/build35/bothMaskedNibs \
                     hg16 /iscratch/i/gs.17/build34/liftOver/split
     # Created parasol job in bed/blat.hg16.2004-07-07/run
     # 1150 jobs
     cd bed/blat.hg16.2004-07-07/run
     para try
     para check
     para push
 
     # GOT HERE
 
     # lift results (use bash)
     cd /cluster/data/hg17/bed/blat.hg16
     for file in /cluster/data/hg16/nib/*.nib; do
        chrom=`basename $file .nib`
        liftUp -pslQ psl/$chrom.psl /cluster/bluearc/hg/gs.17/build34/liftOver/lift/$chrom.lft warn raw/chr*_${chrom}.psl
     done
     # There were some errors from not finding .lft files for the chr_random ones.
     ssh kk9
     cd ../liftOver
     ln -s blat.hg16 blat.hg16.2005-01-22
     makeLoChain-chain hg17 /cluster/data/hg17/nib hg16 /cluster/data/hg16/nib 2>chain.error.log >chain.log
     ssh eieio
     makeLoChain-net hg17 hg16
     ssh hgwdev
     makeLoChain-load hg17 hg16
 
 # DROPUNDER CHAIN TO HG15 (DONE 2005-07-21 Andy)
     # Split things up
     ssh eieio
     cd /cluster/bluearc
     mkdir -p hg15/liftOver/split
     cd hg15/liftOver/split/
     mkdir ../lift
     for c in `cut -f1 /cluster/data/hg15/chrom.sizes`; do
        echo $c
        num=${c%_random}
        num=${num#chr}
        faSplit -lift=../lift/${c}.lft size /cluster/data/hg15/${num}/${c}.fa -oneFile 3000 ${c}
     done  
 
     # Move files to santest
     ssh hgwdev
     cd /santest/scratch
     mkdir hg15
     cd hg15/
     cp -r /cluster/bluearc/hg15/liftOver .
 
     # run alignment
     ssh kk
     cd /cluster/data/hg17
     makeLoChain-align hg17 /scratch/hg/gs.18/build35/bothMaskedNibs \
                     hg15 /santest/scratch/hg15/liftOver/split
     # Created parasol job in bed/blat.hg16.2004-07-07/run
     # 2024 jobs written to batch
     # *** IGNORE the batch created by the script.
     ln -s bed/blat.hg15.2005-07-21 bed/blat.hg15
     cd bed/blat.hg15/
     mv run run.kk
     mkdir run.kk9 run.kki
     cd run.kk/
     sed 's/\.fa\./\./g' spec > tmp; mv tmp spec
     grep Un_random spec > ../run.kki/spec
     grep -v Un_random spec > newspec
     mv newspec spec
     egrep "chr(1|19|X)(\.|_)" spec | grep -v random > ../run.kk9/spec
     grep -Fv -f ../run.kk9/spec spec > newspec
     mv newspec spec
     wc -l spec ../run.kk9/spec ../run.kki/spec
 #   1831 spec
 #    147 ../run.kk9/spec
 #     46 ../run.kki/spec
 #   2024 total
     # Checks out
     
     # Run the thing on all 3 clusters.
     para create spec
     para push
 #Completed: 1831 of 1831 jobs
 #CPU time in finished jobs:    8556066s  142601.10m  2376.69h   99.03d  0.271 y
 #IO & Wait Time:                 60428s    1007.13m    16.79h    0.70d  0.002 y
 #Average job time:                4706s      78.43m     1.31h    0.05d
 #Longest running job:                0s       0.00m     0.00h    0.00d
 #Longest finished job:           46724s     778.73m    12.98h    0.54d
 #Submission to last job:         46725s     778.75m    12.98h    0.54d
     ssh kk9
     cd /cluster/data/hg17/bed/blat.hg15/run.kk9
     para create spec
     para push
 #Completed: 147 of 147 jobs
 #CPU time in finished jobs:    1698424s   28307.07m   471.78h   19.66d  0.054 y
 #IO & Wait Time:                   874s      14.56m     0.24h    0.01d  0.000 y
 #Average job time:               11560s     192.66m     3.21h    0.13d
 #Longest running job:                0s       0.00m     0.00h    0.00d
 #Longest finished job:           31413s     523.55m     8.73h    0.36d
 #Submission to last job:         31413s     523.55m     8.73h    0.36d
     ssh kki
     cd /cluster/data/hg17/bed/blat.hg15/run.kki
     para create spec
     para push
     # OK I don't have para time stuff for this one, but it was the shortest 
     # by far.
 
     # lift results
     cd /cluster/data/hg17/bed/blat.hg15/lift.run
     for chrom in `cut -f1 /cluster/data/hg15/chrom.sizes`; do
        liftUp -pslQ /cluster/bluearc/hg15/liftOver/psl/${chrom}.psl /cluster/bluearc/hg15/liftOver/lift/${chrom}.lft warn raw/chr*_${chrom}.psl
     done
 
     # Chain
     # There's been some problems with store5.  
     ssh kk9
     cd /cluster/store12/store5/gs.18/build35/bed/blat.hg15.2005-07-21
     mkdir chainRun
     mkdir -p /panasas/store/hg15/chainRaw
     ln -s /panasas/store/hg15/chainRaw chainRaw
     cd chainRun/
     ls -1S ../psl/*.psl > in.lst
     cat > chain.sh << "_EOF_"
 #!/bin/bash
 tmp=/scratch/`basename $4`
 axtChain -psl $1 $2 $3 $tmp
 cp $tmp $4
 rm $tmp  
 _EOF_
     chmod +x chain.sh
     cat > gsub << "_EOF_"
 #LOOP
 ./chain.sh $(path1) /scratch/hg/gs.18/build35/bothMaskedNibs /scratch/hg/gs.16/build33/chromTrfMixedNib {check out line+ ../chainRaw/$(root1).chain}
 #ENDLOOP
 _EOF_
     # <<
     cd chainRun/
     gensub2 in.lst single gsub spec
     para create spec
     para push
 #Completed: 44 of 44 jobs
 #CPU time in finished jobs:       7448s     124.13m     2.07h    0.09d  0.000 y
 #IO & Wait Time:                  9591s     159.85m     2.66h    0.11d  0.000 y
 #Average job time:                 387s       6.45m     0.11h    0.00d
 #Longest running job:                0s       0.00m     0.00h    0.00d
 #Longest finished job:            1906s      31.77m     0.53h    0.02d
 #Submission to last job:          1906s      31.77m     0.53h    0.02d
     ssh kolossus
     cd /panasas/store/hg15/chainRaw    
     chainMergeSort *.chain | chainSplit /scratch/andy/chain stdin
     cd /scratch/andy
     mkdir net over
     cd chain/
     for chain in *; do
        c=${chain%.chain}
        echo $c
        chainNet $chain /cluster/store12/store5/gs.18/build35/chrom.sizes \
           /cluster/store12/store5/gs.16/build33/chrom.sizes \
           ../net/${c}.net /dev/null
        netChainSubset ../net/${c}.net $chain ../over/${c}.over
     done
     cd ../over/
     cat * >> ../hg17ToHg15.over.chain
     cd ../
     cp -r hg17* over/ /cluster/store12/store5/gs.18/build35/bed/blat.hg15.2005-07-21/
     cd ../
     rm -rf andy/
     rm -rf /panasas/store/hg15    
     cd /cluster/bluearc/hg15/liftOver/psl
     for psl in *; do
        gzip $psl
     done
     cd ../
 
     
 # Completed: 116281 of 116281 jobs
 # CPU time in finished jobs:   21807388s  363456.46m  6057.61h  252.40d  0.692 y
 # IO & Wait Time:               2319383s   38656.39m   644.27h   26.84d  0.074 y
 # Average job time:                 207s       3.46m     0.06h    0.00d
 # Longest job:                    22063s     367.72m     6.13h    0.26d
 # Submission to last job:         83402s    1390.03m    23.17h    0.97d
 
 
     #	Second cluster run to convert the .out's to .lav's
     #	You do NOT want to run this on the big cluster.  It brings
     #	the file server to its knees.  Run this on the small cluster.
     ssh kki
     cd /cluster/data/hg17/bed/blastzSelf
     /cluster/data/hg17/jkStuff/BlastZ_run1.sh
     cd run.1
     para try, check, push, etc ...
 
 # Completed: 341 of 341 jobs
 # CPU time in finished jobs:       6344s     105.73m     1.76h    0.07d  0.000 y
 # IO & Wait Time:                  5413s      90.22m     1.50h    0.06d  0.000 y
 # Average job time:                  34s       0.57m     0.01h    0.00d
 # Longest job:                      505s       8.42m     0.14h    0.01d
 # Submission to last job:          4521s      75.35m     1.26h    0.05d
 
     #	Third cluster run to convert lav's to axt's
 
     #	These self alignments do not work well as the usual third cluster job.
     #	Instead, a specialized job here that includes a DropSelf
     #	operation, and in individual lav pieces to avoid out of memory
     #	problems during axtSort
     ssh kki
     cd /cluster/data/hg17/bed/blastzSelf
     mkdir axtChrom run.2
     cd run.2
     cat << '_EOF_' > runLavToAxt.sh
 #!/bin/sh
 
 BASE=/cluster/data/hg17/bed/blastzSelf
 SEQ1_DIR=/cluster/bluearc/scratch/hg/gs.18/build35/bothMaskedNibs
 SEQ2_DIR=/cluster/bluearc/scratch/hg/gs.18/build35/bothMaskedNibs
 
 CHR=$1
 OUT=axtChrom/$CHR.axt
 cd ${BASE}/lav/${CHR}
 for D in *.lav
 do
     smallout=$D.axt
     lavToAxt  $D $SEQ1_DIR $SEQ2_DIR stdout \
 	| axtDropSelf stdin stdout \
 	| axtSort stdin $smallout
 done
 cat `ls -1 *.lav.axt | sort -g` > $BASE/$OUT
 '_EOF_'
     # << keep emacs coloring happy
     chmod +x runLavToAxt.sh
 
     cat << '_EOF_' > gsub
 #LOOP
 ./runLavToAxt.sh $(path1) {check out line ../axtChrom/$(path1).axt}
 #ENDLOOP
 '_EOF_'
     # << keep emacs coloring happy
 
     ls ../lav > chrList
     gensub2 chrList single gsub jobList
     para create jobList
     para try
     para push
     #	This is a tough load on eieio.  Managable, but the load should
     #	be monitored to make sure it isn't severe.  I saw about 100 to 150
     #	the chr19 job will not finish, even in parts it takes up too
     #	much memory and the node it runs on ends up swapping endlessly.
     #	Need to go to kolossus to do chr19
     para stop
     para recover jobList chr19JobList
     ssh kolossus
     cd /cluster/data/hg17/bed/blastzSelf/run.2
     time ./runLavToAxt.sh chr19
     #	real    43m14.797s
     #	user    12m56.670s
     #	sys     3m13.590s
 
     # translate sorted axt files into psl
     ssh eieio
     cd /cluster/data/hg17/bed/blastzSelf
     mkdir pslChrom
     set tbl = "blastzSelf"
     foreach f (axtChrom/chr*.axt)
       set c=$f:t:r
       echo "Processing chr $c"
       /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
     end
     #	That takes about 70 minutes
 
     # Load database tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastzSelf/pslChrom
     bash # if a csh/tcsh user
     for I in *.psl
     do
 	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
 	echo "done: ${I}"
     done
     # exit bash if you are tcsh
     #	This is an 80 minute job
 
     #	Check results
     #	featureBits hg17 blastzSelf
     #	252256266 bases of 2866216770 (8.801%) in intersection
     #	real    40m49.573s
     #	user    21m14.200s
     #	sys     2m10.420s
 
     #	featureBits hg16 blastzSelf
     #	254410837 bases of 2865248791 (8.879%) in intersection
 
 
 # CHAIN SELF BLASTZ (DONE - 2004-07-07 - Hiram)
 
 # The axtChain is best run on the small kluster, or the kk9 kluster
     ssh kki
     mkdir -p /cluster/data/hg17/bed/blastzSelf/axtChain/run1
     cd /cluster/data/hg17/bed/blastzSelf/axtChain/run1
     mkdir out chain
 
     ls -1S /cluster/data/hg17/bed/blastzSelf/axtChrom/*.axt > input.lst
     cat << '_EOF_' > gsub
 #LOOP
 doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} out/$(root1).out
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
 #  May need -minScore=5000 for all chroms if chr19 won't finish on kolossus
 
     cat << '_EOF_' > doChain
 #!/bin/csh
 axtChain $1 /iscratch/i/gs.18/build35/bothMaskedNibs \
 	/iscratch/i/gs.18/build35/bothMaskedNibs $2 > $3
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod a+x doChain
 
     # 46 jobs
     gensub2 input.lst single gsub jobList
     para create jobList
     para try
     para push # ... etc ...
 # Completed: 45 of 46 jobs
 # Crashed: 1 jobs
 # CPU time in finished jobs:       8519s     141.98m     2.37h    0.10d  0.000 y
 # IO & Wait Time:                  4795s      79.92m     1.33h    0.06d  0.000 y
 # Average job time:                 296s       4.93m     0.08h    0.00d
 # Longest job:                     2407s      40.12m     0.67h    0.03d
 # Submission to last job:          3540s      59.00m     0.98h    0.04d
 
     #	chr19 did fail, on kolossus, try:
     ssh kolossus
     cd /cluster/data/hg17/bed/blastzSelf/axtChain/run1
     time axtChain /cluster/data/hg17/bed/blastzSelf/axtChrom/chr19.axt \
 	/cluster/data/hg17/nib \
 	/cluster/data/hg17/nib \
 	chain/chr19.chain > out/chr19.out
     #	80 minute job, 1.5 Gb result:
     #	-rw-rw-r--    1 1588795432 Jul  7 21:54 chr19.chain
 
     # now on the file server, sort chains
     ssh eieio
     cd /cluster/data/hg17/bed/blastzSelf/axtChain
     time chainMergeSort run1/chain/*.chain > all.chain
     #	real    27m38.935s
     #	user    23m18.540s
     #	sys     2m39.300s
     #	A 5 Gb file:
     #	-rw-rw-r--    1 5267202936 Jul  7 22:23 all.chain
 
 
     time chainSplit chain all.chain
     #	real    29m27.062s
     #	user    22m48.250s
     #	sys     1m57.910s
 
     # optionally: rm run1/chain/*.chain
 
     # Load chains into database
     # next machine
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastzSelf/axtChain/chain
     bash	#	for tcsh users
     for I in *.chain
     do
         c=${I/.chain/}
         $HOME/bin/i386/hgLoadChain -normScore hg17 ${c}_chainSelf $I
         echo done $c
     done
     # exit bash if you are tcsh
     #	This is almost 3 hours to load
 
     ssh kolossus
     cd /cluster/data/hg17/bed/blastzSelf.2004-07-01
     time HGDB_CONF=~/.hg.conf.read-only featureBits \
 	-noRandom -noHap hg17 chainSelfLink > fb.chainSelfLink 2>&1 &
     #	real    56m34.802s
     #	240976607 bases of 2851352871 (8.451%) in intersection
 
     #	featureBits hg17 chainSelf
     #	682833453 bases of 2866216770 (23.824%) in intersection
     #	featureBits hg16 chainSelf
     #	626345319 bases of 2865248791 (21.860%) in intersection
 
     #	DELIVER these chain files to hgdownload (2005-01-27 - Hiram)
     ssh eieio
     cd /cluster/data/hg17/bed/blastzSelf/axtChain/chain
     gzip chr*.chain
 
     ssh hgwdev
     mkdir /usr/local/apache/htdocs/goldenPath/hg17/vsSelf
     cd /cluster/data/hg17/bed/blastzSelf/axtChain/chain
     cp -p *.chain.gz /usr/local/apache/htdocs/goldenPath/hg17/vsSelf
 
     #	fixup README file, request push
 
 # NET SELF (DONE - 2004-07-13 - Hiram)
 
     ssh eieio
     cd /cluster/data/hg17/bed/blastzSelf/axtChain
     mkdir preNet
     cd chain
     bash	#	for tcsh users
     for I in *.chain
     do
       echo preNetting $I
       /cluster/bin/i386/chainPreNet $I /cluster/data/hg17/chrom.sizes \
                         /cluster/data/hg17/chrom.sizes ../preNet/$I
     done
     #	23 minutes
 
     cd ..
     mkdir n1
     cd preNet
     for I in *.chain
     do
       N=${I/.chain/}.net
       echo primary netting $I
       /cluster/bin/i386/chainNet $I -minSpace=10 \
 	/cluster/data/hg17/chrom.sizes /cluster/data/hg17/chrom.sizes \
 	../n1/$N /dev/null
     done
     # exit bash if you are tcsh
     #	5 minute job
 
     cd ..
     cat n1/*.net | /cluster/bin/i386/netSyntenic stdin hNoClass.net
     #	memory usage 206442496, utime 3009 s/100, stime 252
     #	memory usage 2510467072, utime 19307 s/100, stime 3181
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastzSelf/axtChain
     time netClass hNoClass.net hg17 hg17 human.net \
 	-tNewR=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInHuman \
 	-qNewR=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInHuman
     #	real    9m32.951s
     #	user    2m42.840s
     #	sys     1m23.460s
 
     # If things look good do
     ssh eieio
     cd /cluster/data/hg17/bed/blastzSelf/axtChain
     rm -r n1 hNoClass.net
     # Make a 'syntenic' subset of these with
     time netFilter -syn human.net > humanSyn.net
     #	real    0m29.851s
     #	user    0m27.200s
     #	sys     0m2.120s
 
     # Load the nets into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastzSelf/axtChain
     netFilter -minGap=10 human.net |  hgLoadNet hg17 netSelf stdin
     netFilter -minGap=10 humanSyn.net | hgLoadNet hg17 syntenyNetSelf stdin
 
     # check results
     # featureBits hg17 netSelf
     #	620827374 bases of 2866216770 (21.660%) in intersection
     # featureBits hg16 netSelf
     #	563788850 bases of 2865248791 (19.677%) in intersection
     # featureBits hg15 selfNet
     #	749177799 bases of 2866466359 (26.136%) in intersection
 
 
     # featureBits hg17 syntenyNetSelf
     #	404535376 bases of 2866216770 (14.114%) in intersection
     # featureBits hg16 syntenyNetSelf
     #	340871322 bases of 2865248791 (11.897%) in intersection
 
     # Add entries for net and chain to human/hg17 trackDb
 
     # make net
     ssh eieio
     cd /cluster/data/hg17/bed/blastzSelf/axtChain
     mkdir humanNet
     time netSplit human.net humanNet
     #	real    0m52.106s
     #	user    0m43.350s
     #	sys     0m5.170s
 
     # extract axts from net  - this should be combined with the sort and
     # maf conversion below
     mkdir ../axtNet 
     foreach n (humanNet/chr*.net)
 	set c=$n:t:r
 	echo "netToAxt: $c.net -> $c.axt"
 	rm -f ../axtNet/$c.axt
 	netToAxt humanNet/$c.net chain/$c.chain \
 		/cluster/data/hg17/nib \
 		/cluster/data/hg17/nib stdout > ../axtNet/$c.axt
 	echo "Complete: $c.net -> axtNet/$c.axt"
     end
 
     # sort axt's and convert to maf format
     mkdir ../mafNet
     foreach f (../axtNet/chr*.axt)
         set c=$f:t:r
         echo $c.axt
         mv ../axtNet/$c.axt ../axtNet/$c.unsorted.axt
         axtSort ../axtNet/$c.unsorted.axt ../axtNet/$c.axt
         rm ../axtNet/$c.unsorted.axt
         axtToMaf ../axtNet/$c.axt \
             /cluster/data/hg17/chrom.sizes /cluster/data/hg17/chrom.sizes \
                 ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=hg17.
     end
     # a 3 minute job
 
     XXXX - ! ! !   WE DO NOT NEED the Best and Tight tracks for Self ! ! !
     ssh hgwdev
     mkdir -p /cluster/data/hg17/bed/blastzSelf/axtBest
     cd /cluster/data/hg17/bed/blastzSelf/axtBest
     ln -s ../axtNet/chr*.axt .
 
     # copy net axt's to download area - XXX Do we need this for Self ?
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastzSelf/axtNet
     mkdir /usr/local/apache/htdocs/goldenPath/hg17/vsSelf
     cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsSelf
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsSelf
     nice gzip *.axt
     nice md5sum *.gz > md5sum.txt
     # add README.txt file to dir (use previous assembly's copy as template)
 
     #  Convert those axt files to psl
     ssh eieio
     cd /cluster/data/hg17/bed/blastzSelf
     mkdir pslBest
     foreach a (axtBest/chr*.axt)
 	set c=$a:t:r
 	echo "processing $c.axt -> ${c}_blastzBestSelf.psl"
     /cluster/bin/i386/axtToPsl axtBest/${c}.axt \
 	S1.len S2.len pslBest/${c}_blastzBestSelf.psl
 	echo "Done: ${c}_blastzBestSelf.psl"
     end
 
     # Load tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastzSelf/pslBest
     bash # if a csh/tcsh user
     for I in chr*BestSelf.psl
     do
 	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
 	echo "done ${I}"
     done
     # exit bash if you are tcsh
 
     #	check results
     #	featureBits hg17 blastzBestSelf
     #	233978156 bases of 2866216770 (8.163%) in intersection
     #	featureBits hg16 blastzBestSelf
     #	225819219 bases of 2865248791 (7.881%) in intersection
 
 # MAKING HUMAN AXTTIGHT FROM AXTBEST (NOT TO BE DONE - 2004-07-13 - Hiram)
     #	XXXX - ! ! !  DO NOT NEED axtBest for Self alignments
     #	Been done anyway, Robert and Gill like to see it.
 
 # BLASTZ SELF CLEAN UP (DONE - 2004-07-15 - Hiram)
     ssh eieio
     cd /cluster/data/hg17/bed/blastzSelf
     nice rm -rf raw &
     nice rm axtChain/run1/chain/* &
     nice rm -fr axtChain/n1 axtChain/hNoClass.net &
     nice gzip axtChrom/* pslChrom/* lav/*/* axtChain/all.chain axtChain/*.net &
 
 
 # CREATING BIG ZIPS (DONE - 2004-07-23 - Hiram)
     ssh eieio
     cd /cluster/data/hg17/jkStuff
     time ./zipAll.sh > zipAll.out 2>&1
 
     ssh hgwdev
     #  This stuff has to work in a different way because this stuff
     #	updates on a daily basis.
     cd /usr/local/apache/htdocs/goldenPath/hg17/bigZips
     featureBits hg17 refGene:upstream:1000 -fa=upstream1000.fa
     zip upstream1000.zip upstream1000.fa
     rm upstream1000.fa
     featureBits hg17 refGene:upstream:2000 -fa=upstream2000.fa
     zip upstream2000.zip upstream2000.fa
     rm upstream2000.fa
     featureBits hg17 refGene:upstream:5000 -fa=upstream5000.fa
     zip upstream5000.zip upstream5000.fa
     rm upstream5000.fa
 
 
 # ENCODE REGIONS (DONE 2004-07-28 kate)
 
     ssh eieio
     cd /cluster/data/hg17/bed
     mkdir encodeRegions
     cd encodeRegions
     liftOver /cluster/data/hg16/bed/encodeRegions/encodeRegions.bed \
         /cluster/data/hg16/bed/liftOver/hg16ToHg17.over.chain \
                 encodeRegions.bed encodeRegions.unmapped
     wc -l encodeRegions.*
     # 44 encodeRegions.bed
     # 0 encodeRegions.unmapped
     ssh hgwdev
     cd /cluster/data/hg17/bed/encodeRegions
     hgLoadBed hg17 encodeRegions encodeRegions.bed -noBin
 
 
 # H-INVITATIONAL GENE ANNOTATION DATABASE (WORKING 2004-07-28 kate)
     # http://www.jbirc.aist.go.jp/hinv/top.html
     # Create knownGene table to reference HINV gene ID's
     #  for link on knownGenes details page
     # Also, create an HINV gene track
 
     # download CDNA file release 1.5 -- got release # from downloads page).
     ssh kksilo
     mkdir -p /cluster/data/hinv
     cd /cluster/data/hinv
     wget http://www.jbirc.aist.go.jp/hinv/download/alldata/flatfile/FCDNA.gz
     gunzip FCDNA.gz
     mv FCDNA FCDNA.1.5
 
     # set up assembly work area
     ssh eieio
     cd /cluster/data/hg17
     mkdir -p bed/hinv
     cd bed/hinv
 
     # extract H-INV ID's and Genbank accessions of mRNAs
     awk '/CDNA_ACCESSION-NO:/ {print $2}' < /cluster/data/hinv/FCDNA.1.5 \
                                                         > accessions.txt
     awk '/CDNA_H-INVITATIONAL-ID:/ {print $2}' < /cluster/data/hinv/FCDNA.1.5 \
                                                         > ids.txt
     paste accessions.txt ids.txt > queries.txt
     wc -l ids.txt
         #   41118 ids.txt
 
     # create PSL file from alignments for these mRNA's, extracted from the 
     #       table of all aligned mRNA's
     ssh hgwdev
     cd /cluster/data/hg17/bed/hinv
     hgsql hg17 -s -e "SELECT * FROM all_mrna"  | cut -f 2- > all_mrna.tab
 
     ssh eieio
     cd /cluster/data/hg17/bed/hinv
     pslReps /dev/null stdout /dev/null | cat - all_mrna.tab > all_mrna.psl
         # using pslReps to generate the PSL file header
     ~kate/bin/i386/pslSelect -queryPairs=queries.txt all_mrna.psl hinv_mrna.psl
         # NOTE: generated with pslSelect.c v1.3 (1.4 is broken -- test is
         # setup in hg/pslSelect/tests & I requested Robert take a look)
 
     # load track of mrna alignments
     hgwdev
     cd /cluster/data/hg17/bed/hinv
     hgLoadPsl hg17 -table=HInvGeneMrna hinv_mrna.psl
     hgsql hg17 -s -e \
         "select distinct(qName) from HInvGeneMrna order by qName" > hg17.mrna
     hgsql hg16 -s -e \
         "select distinct(qName) from HInvGeneMrna order by qName" > hg16.mrna
     wc -l hg*.mrna
         # 40998 hg16.mrna
         # 41023 hg17.mrna
 
     comm -1 -3 *.mrna > hg17.aligned
     wc -l hg17.aligned
         # 29 (transcripts newly aligned in hg17)
     comm -2 -3 *.mrna > hg16.aligned
     wc -l hg16.aligned
         # 4 (transcripts no longer aligned in hg17)
     comm -2 -3 ids.txt hg17.mrna > hg17.notaligned
     wc -l hg17.notaligned
         # 95 (transcripts not aligned in hg17 -- checking on why...)
 
     # also make a table with various useful items for each transcript
     ssh hgwdev
     hgsql hg17 < ~/kent/src/hg/lib/HInv.sql
     cd /cluster/data/hg17/bed/hinv
     /cluster/data/hinv/hinvToTable.pl < /cluster/data/hinv/FCDNA.1.5 > HInv.tab
     echo 'load data local infile "HInv.tab" into table HInv' | hgsql hg17
     hgsql hg16 -s -e "select count(*) from HInv"
         # 41118
     hgsql hg17 -s -e "select count(*) from HInv"
         # 41118
 
     # create table for knownGenes detail page
     ssh hgwdev
     cd /cluster/data/hg17/bed/hinv
     hgMapToGene hg17 HInvGeneMrna knownGene knownToHInv
 
 
 # GENEID GENE PREDICTIONS (DONE 7/30/04 angie)
     ssh hgwdev
     mkdir /cluster/data/hg17/bed/geneid
     cd /cluster/data/hg17/bed/geneid
     foreach chr (`awk '{print $1;}' ../../chrom.sizes`)
       wget http://genome.imim.es/genepredictions/H.sapiens/golden_path_200405/geneid_v1.2/$chr.gtf
       wget http://genome.imim.es/genepredictions/H.sapiens/golden_path_200405/geneid_v1.2/$chr.prot
     end
     # Add ".1" suffix to each item in .prot's, to match transcript_id's in gtf
     cp /dev/null geneid.fa
     foreach f (chr*.prot)
       perl -wpe 's/^(>chr\S+)/$1.1/' $f >> geneid.fa
     end
     ldHgGene -gtf -genePredExt hg17 geneid *.gtf 
     hgPepPred hg17 generic geneidPep geneid.fa
 
 
 # MITOPRED DATA FOR HGGENE (DONE 7/30/04 angie)
     ssh hgwdev
     mkdir /cluster/data/hg17/bed/mitopred
     cd /cluster/data/hg17/bed/mitopred
     wget http://mitopred.sdsc.edu/data/hum_30.out
     perl -wpe 's/^(\S+)\s+\S+\s+(.*)/$1\t$2/' hum_30.out > mitopred.tab
     cat > mitopred.sql << '_EOF_'
 # Prediction of nuclear-encoded mito. proteins from http://mitopred.sdsc.edu/
 CREATE TABLE mitopred (
     name varchar(10) not null,      # SwissProt ID
     confidence varchar(8) not null, # Confidence level
               #Indices
     PRIMARY KEY(name(6))
 );
 '_EOF_'
     # << this line makes emacs coloring happy
     hgsql hg17 < mitopred.sql
     hgsql hg17 -e 'load data local infile "mitopred.tab" into table mitopred'
 
 
 # NUCLEAR PROTEIN DATABASE (IN PROGRESS 7/30/04 angie)
     ssh eieio
     mkdir /cluster/data/hg17/bed/npd
     cd /cluster/data/hg17/bed/npd
     wget ftp://ftp.hgu.mrc.ac.uk/pub/npd/database.zip
     unzip database.zip
     # OK, it's one big .mdb (Microsoft Access DB) file.  
     # Googling... can buy a converter for $40... free trial .exe...
 
 
 # CREATING REFFULL - DBTSS MRNA (DONE - 2004-08-02 - Hiram)
 
     ssh to eieio
     mkdir /cluster/data/hg17/bed/refFull
     cd /cluster/data/hg17/bed/refFull
     wget --timestamping "ftp://ftp.hgc.jp/pub/hgc/db/dbtss/ref-full.fa.gz" .
     wget --timestamping "ftp://ftp.hgc.jp/pub/hgc/db/dbtss/readme" .
     #	See also: http://dbtss.hgc.jp/index.html
 
     #	gunzip it and split the ref-rull.fa file into about 200 pieces
     #	(faSplit won't do this job if it is:
     #	zcat ref-full.fa.gz | faSplit sequence stdin 50 splitRefFull
     gunzip ref-full.fa.gz
     faSplit sequence ref-full.fa 50 splitRefFull
     gzip ref-full.fa
 
     #	copy to Iservers
     ssh kkr1u00
     cd /cluster/data/hg17/bed/refFull
     mkdir /iscratch/i/gs.18/build35/refFull
     cp -p split*.fa /iscratch/i/gs.18/build35/refFull
     /cluster/bin/iSync
     #	no longer need these split files here
     rm -f split*.fa
 
     #	run alignments on kluster
     ssh kk
     cd /cluster/data/hg17/bed/refFull
     ls -1S /scratch/hg/gs.18/build35/maskedContigs > genome.lst
     ls -1S /iscratch/i/gs.18/build35/refFull > refFull.lst
 
 
     #	Use BLAT to generate refFull alignments as so:
     cat << '_EOF_' > gsub
 #LOOP
 /cluster/bin/i386/blat -ooc=/scratch/hg/h/11.ooc -q=dna -t=dna {check in exists /scratch/hg/gs.18/build35/maskedContigs/$(path1)} {check in exists+ /iscratch/i/gs.18/build35/refFull/$(path2)} {check out line+ psl/$(root1)/$(root2).psl}
 #ENDLOOP
 '_EOF_'
     # << keep emacs coloring happy
 
     bash # if a csh/tcsh user
     mkdir psl
     cat genome.lst | sed -e "s/.fa//" | while read C
 	do
 		mkdir psl/${C}
 	done
     # exit bash if you are tcsh
     gensub2 genome.lst refFull.lst gsub jobList
     para create jobList
     #	18240 jobs written to batch
     para try
     para check
     para push ... etc ...
 # Completed: 18240 of 18240 jobs
 # CPU time in finished jobs:      37011s     616.85m    10.28h    0.43d  0.001 y
 # IO & Wait Time:                 62630s    1043.84m    17.40h    0.72d  0.002 y
 # Average job time:                   5s       0.09m     0.00h    0.00d
 # Longest job:                       51s       0.85m     0.01h    0.00d
 # Submission to last job:           850s      14.17m     0.24h    0.01d
 
     #	Process refFull alignments into near best in genome.
     ssh eieio
     cd /cluster/data/hg17/bed/refFull
     pslSort dirs raw.psl tmp psl/*
     pslReps -minCover=0.2 -sizeMatters -minAli=0.965 \
 	-nearTop=0.001 raw.psl contig.psl /dev/null
     liftUp -nohead all_refFull.psl ../../jkStuff/liftAll.lft warn contig.psl
     pslSortAcc nohead chrom tmp all_refFull.psl
     pslCat -dir chrom > refFullAli.psl
 
     #	Load refFull alignments into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/refFull
     hgLoadPsl hg17 -tNameIx refFullAli.psl
 
 # VAR_MULTIZ HG17/MM5/RN3/GALGAL2/FR1 (acs 2004-08-12)
 
 # This is a new, experimental version of multiz written by Minmei at
 # PSU and sent by e-mail from Webb.  This version allows for a
 # progressive alignment strategy (i.e., alignment construction in a
 # post-order traversal of the tree) using only pairwise alignments of
 # each sequence with the reference sequence and without any need for
 # "staging".  Here's a little blurb about it from the header of
 # var_multiz.v3.c.
 
 #  var_multiz.v3.c
 #
 #  Variant to multiz program. It aligns two files of
 #  alignment blocks where top row is always the reference,
 #  assuming blocks are increasing ordered based on the
 #  start position on the refernece seqence. Single-coverage
 #  on reference is required at this stage.
 #
 #  Four arguments are required: char* arg1, char* arg2,
 #  int arg3, int arg4. arg1 and arg2 are two files need
 #  to be aligned together. The alignment of reference in
 #  two files are either fixed or not, determined from
 #  argurments arg3 and arg4. arg3 and arg4 are either 1
 #  or 0, but cannot be 1 at the same time. 1 means
 #  reference is fixed. v1 and v2 cannot be both 1.
 #  ...
 
     mkdir /cluster/data/hg17/bed/var_multiz.2004-08-12
 
     # unpack source and compile
     cp /cluster/home/acs/var_multiz.tar.gz /cluster/data/hg17/bed/var_multiz.2004-08-12
     cd /cluster/data/hg17/bed/var_multiz.2004-08-12
     tar xfz var_multiz.tar.gz
     cd var_multiz_source
     make
 
      NOTE (8/14): this version of the source is already out of date!
     # Source is now checked in under hg3rdParty and updated binaries
     # are being kept under /cluster/bin/penn/var_multiz
 
     # script for creating the 5-way alignments for a given chromosome
     # (acs, 8/20/04) below revised after e-mail exchange with Minmei
     cat << '_EOF_' > doVarMultiz.csh
 #!/bin/csh -fe
 
 set chr = $1			# may include _random or _hla_hap[12]
 
 set REF = hg17.$chr
 set RAT = /cluster/bluearc/hg17/multiz8way/rn3/$chr.maf
 set MOUSE = /cluster/bluearc/hg17/multiz8way/mm5/$chr.maf
 set CHICKEN = /cluster/bluearc/hg17/multiz8way/galGal2/$chr.maf
 set FISH = /cluster/bluearc/hg17/multiz8way/fr1/$chr.maf
 set DEST = /cluster/data/hg17/bed/var_multiz.2004-08-12/maf/$chr.maf
 
 set VMZ = /cluster/bin/penn/var_multiz
 set PROJECT = /cluster/bin/penn/var_multiz.2004.08.12/maf_project
 
 mkdir -p $DEST:h
 
 if ( -s $RAT && -s $MOUSE ) then 
     echo "Aligning $RAT $MOUSE..."
     $VMZ $RAT $MOUSE 0 0 > /scratch/$chr.tmp1.maf
     echo "Projecting on $REF..."
     $PROJECT /scratch/$chr.tmp1.maf $REF > /scratch/$chr.hrm.maf
 else if ( -s $RAT ) then
     cp $RAT /scratch/$chr.hrm.maf
 else if ( -s $MOUSE ) then
     cp $MOUSE /scratch/$chr.hrm.maf
 endif
 
 if ( -s $CHICKEN && -s /scratch/$chr.hrm.maf ) then
     echo "Adding $CHICKEN..."
     $VMZ /scratch/$chr.hrm.maf $CHICKEN 1 0 > /scratch/$chr.tmp2.maf
     echo "Projecting on $REF..."
     $PROJECT /scratch/$chr.tmp2.maf $REF > /scratch/$chr.hrmc.maf
 else if ( -s $CHICKEN ) then
     cp $CHICKEN /scratch/$chr.hrmc.maf
 else if ( -s /scratch/$chr.hrm.maf ) then
     cp /scratch/$chr.hrm.maf /scratch/$chr.hrmc.maf
 endif 
 
 if ( -s $FISH && -s /scratch/$chr.hrmc.maf ) then
     echo "Adding $FISH..."
     $VMZ /scratch/$chr.hrmc.maf $FISH 1 0 > /scratch/$chr.tmp3.maf
     echo "Projecting on $REF..."
     $PROJECT /scratch/$chr.tmp3.maf $REF > $DEST
 else if ( -s $FISH ) then
     cp $FISH $DEST
 else if ( -s /scratch/$chr.hrmc.maf ) then
     cp /scratch/$chr.hrmc.maf $DEST
 endif 
 
 
 echo "Done."
 rm /scratch/$chr.tmp[123].maf /scratch/$chr.hrm.maf /scratch/$chr.hrmc.maf
 '_EOF_'
     # << keep emacs coloring happy    
     chmod 755 doVarMultiz.csh
 
     for file in `find /cluster/bluearc/hg17/multiz8way/rn3 /cluster/bluearc/hg17/multiz8way/mm5 /cluster/bluearc/hg17/multiz8way/galGal2 /cluster/bluearc/hg17/multiz8way/fr1 -name "chr*.maf"` ; do echo `basename $file .maf` ; done | sort -u > chrlist
     rm -f jobs.lst
     for chr in `cat chrlist` ; do echo "doVarMultiz.csh $chr" >> jobs.lst ; done
 
     # run cluster job
     ssh kk ; cd /cluster/data/hg17/bed/var_multiz.2004-08-12; para create jobs.lst ; para try ; para push 
     # (etc.)
 
 Completed: 46 of 46 jobs
 CPU time in finished jobs:      71302s    1188.36m    19.81h    0.83d  0.002 y
 IO & Wait Time:                  1162s      19.37m     0.32h    0.01d  0.000 y
 Average job time:                1575s      26.26m     0.44h    0.02d
 Longest job:                     6353s     105.88m     1.76h    0.07d
 Submission to last job:          6362s     106.03m     1.77h    0.07d
 
     # for now just create an ordinary maf track (conservation later)
     rm -rf /gbdb/hg17/varMultiz/maf/varMultizMm5Rn3GalGal2Fr1
     mkdir -p /gbdb/hg17/varMultiz/maf/varMultizMm5Rn3GalGal2Fr1
     ln -s /cluster/data/hg17/bed/var_multiz.2004-08-12/maf/*.maf /gbdb/hg17/varMultiz/maf/varMultizMm5Rn3GalGal2Fr1
     /cluster/bin/i386/hgLoadMaf hg17 -warn varMultizMm5Rn3GalGal2Fr1 -pathPrefix=/gbdb/hg17/varMultiz/maf/varMultizMm5Rn3GalGal2Fr1
     chmod 775 /gbdb/hg17/varMultiz /gbdb/hg17/varMultiz/maf /gbdb/hg17/varMultiz/maf/varMultizMm5Rn3GalGal2Fr1 /cluster/data/hg17/bed/var_multiz.2004-08-12 /cluster/data/hg17/bed/var_multiz.2004-08-12/maf
     chmod 664 /cluster/data/hg17/bed/var_multiz.2004-08-12/maf/*.maf
 
     # trackDb entry
 # track varMultizMm5Rn3GalGal2Fr1
 # shortLabel varMultiz5Way
 # longLabel Human/Mouse/Rat/Chicken/Fugu Var-Multiz 
 # group compGeno
 # priority 190
 # visibility hide
 # type maf
 
 
 # elephant human blastz alignment by Robert Aug 11 2004
 mkdir /cluster/bluearc/elephant
 cd /cluster/bluearc/elephant
 
 #get reads and qual scores from trace repository
 for i in `cat trace.lst`; do echo $i ; wget ftp://ftp.ncbi.nih.gov/pub/TraceDB/loxodonta_africana/fasta.loxodonta_africana.$i.gz ; done
 for i in `cat trace.lst`; do echo $i ; wget ftp://ftp.ncbi.nih.gov/pub/TraceDB/loxodonta_africana/qual.loxodonta_africana.$i.gz ; done
 for i in `cat trace.lst` ; do zcat fasta.loxodonta_africana.$i.gz > loxodonta_africana.$i.fa ; done     
 
 #trim reads
 for i in `cat trace.lst` ; do nice gunzip -c qual.loxodonta_africana.$i.gz > qual.loxodonta_africana.$i; faTrimRead loxodonta_africana.$i.fa qual.loxodonta_africana.$i tmp.$i.fa lift.$i.lft; mv -f tmp.$i.fa loxodonta_africana.$i.fa ; rm -f qual.loxodonta_africana.$i ; done
 for i in `cat trace.lst`; do faSize -detailed=on loxodonta_africana.$i.fa > mac.$i.len ; done  
 cat mac.0*.len > S2.len
 for i in `cat trace.lst`; do sed -e s/S2.len/mac.$i.len/ < DEF > DEF.$i ; done
 
 #split fa reads into 10mb chunks for blastz run and distribute to i-servers.
 ssh kkr1u00
 for i in `cat trace.lst`; do nice faSplit about loxodonta_africana.$i.fa 10000000 /iscratch/i/elephant/${i}.mac. ; done
 cd /iscratch/i/elephant
 find split -name \*.fa > /cluster/bluearc/elephant/mac.lst
 cd /cluster/bluearc/elephant
 hgsql hg17 -N < chromLen.sql > S1.len
 cd /iscratch/i/elephant
 iSync
 
 #setup cluster run to blastz reads to human genome
 ssh kk
 cd /cluster/bluearc/elephant
 BlastZ_run0.sh
 cd run.0
 para create jobList
 para push
 
 #94798 jobs in batch
 #149 jobs (including everybody's) in Parasol queue.
 #Checking finished jobs
 #Completed: 94797 of 94798 jobs
 #Crashed: 1 jobs
 #CPU time in finished jobs:   14183153s  236385.89m  3939.76h  164.16d  0.450 y
 #IO & Wait Time:                310938s    5182.30m    86.37h    3.60d  0.010 y
 #Average job time:                 153s       2.55m     0.04h    0.00d
 #Longest job:                     1770s      29.50m     0.49h    0.02d
 #Submission to last job:         52186s     869.77m    14.50h    0.60d
 
 ssh kkr9
 BlastZ_run1.sh
 cd run.1
 para create jobList
 para push
 #341 jobs in batch
 #151 jobs (including everybody's) in Parasol queue.
 #Checking finished jobs
 #Completed: 341 of 341 jobs
 #CPU time in finished jobs:     142914s    2381.91m    39.70h    1.65d  0.005 y
 #IO & Wait Time:                 14078s     234.63m     3.91h    0.16d  0.000 y
 #Average job time:                 460s       7.67m     0.13h    0.01d
 #Longest job:                      782s      13.03m     0.22h    0.01d
 #Submission to last job:          1954s      32.57m     0.54h    0.02d
 
 #generate lst and fa files for each chromosome for faster lavToAxt
 cd /cluster/bluearc/elephant
 echo "select chrom from chromInfo;" > chrom.sql
 hgsql hg17 -B -N < chrom.sql > chrom.lst
 
 for i in `cat chrom.lst` ; do grep -h '>' lav/$i/* | awk '{print $1}' | sed -e 's/"//g' | sed -e 's/>//g' > mac.$i.lst ; echo $i ; done
 
 ssh kki
 cd /cluster/bluearc/elephant
 mkdir -p splitChrom
 /bin/rm splitChrom/*
 gensub2 trace.lst chrom.lst gsub.split spec.split
 para create spec.split
 para push
 #322 jobs in batch
 #Checking finished jobs
 #Completed: 322 of 322 jobs
 #CPU time in finished jobs:        819s      13.65m     0.23h    0.01d  0.000 y
 #IO & Wait Time:                  3278s      54.63m     0.91h    0.04d  0.000 y
 #Average job time:                  13s       0.21m     0.00h    0.00d
 #Longest job:                       51s       0.85m     0.01h    0.00d
 #Submission to last job:           462s       7.70m     0.13h    0.01d
 
 
 cd /cluster/bluearc/elephant/splitChrom
 for i in `cat /cluster/bluearc/elephant/chrom.lst` ; do cat mac.*.$i.fa > mac.$i.fa ; echo $i ; done
 
 #lav to axt run
 ssh kk
 cd /cluster/bluearc/elephant
 mkdir -p run.2
 #NOTE: chr19 must be run on kolossus with 64bit executables
 #change SEQ1_DIR form /scratch to /iscratch for mini cluster
 . DEF
 echo "#LOOP" > run.2/gsub
 echo '/cluster/bin/scripts/blastz-contiglav2axt '${BASE}'/lav/$(root1) {check out line+ '${BASE}'/axtChrom/$(root1).axt} '${SEQ1_DIR}' /cluster/bluearc/elephant/splitChrom/mac.$(root1).fa' >> run.2/gsub
 echo "#ENDLOOP" >> run.2/gsub
 cd run.2
 gensub2 ../chrom.lst single gsub jobList
 para create jobList
 para push
 #chrM has no data and crashed
 #46 jobs in batch
 #Checking finished jobs
 #Completed: 45 of 46 jobs
 #Crashed: 1 jobs
 #CPU time in finished jobs:     249970s    4166.17m    69.44h    2.89d  0.008 y
 #IO & Wait Time:                  5407s      90.11m     1.50h    0.06d  0.000 y
 #Average job time:                5675s      94.58m     1.58h    0.07d
 #Longest job:                    27065s     451.08m     7.52h    0.31d
 #Submission to last job:         47744s     795.73m    13.26h    0.55d
 
 
 
 #split reads by prefix so axtBest will fit in memory
 mkdir axtByQ
 cat mac*.lst | awk -F\| '{print substr($3,1,3)}' | sort -nu >prefix.lst
 for i in `cat prefix.lst` ; do cat axtChrom/*.axt | axtFilter -qStartsWith=gnl\|ti\|$i stdin | axtSwap stdin S1.len S2.len stdout | axtSort stdin axtByQ/q$i.axt ; done
 mkdir axtByQBest
 #lots of memory needed for reciprocal best
 ssh kolossus
 cd /cluster/bluearc/elephant/axtByQ
 for i in `ls *.axt` ; do axtBest -quiet i all stdout | axtSwap stdin ../S2.len ../S1.len ../axtByQBest/$i ; echo $i done ; done
 cd ../cluster/bluearc/elephant/axtByQBest
 cat q*.axt | axtSplitByTarget stdin .
 mkdir axtRecipBest
 for i in `cat chrom.lst` ; do axtSort axtByQBest/$i.axt stdout | axtBest stdin $i axtRecipBest/$i.axt ; echo $i ;done 
 for i in `cat chrom.lst` ; do axtToMaf axtRecipBest/$i.axt S1.len S2.len maf/$i.maf -tPrefix=hg17. -qPrefix=rm1. -scoreZero ; done 
 for i in `cat chrom.lst` ; do mafFilter -minScore=1000 maf/$i.maf > mafFilter/$i.maf ; done 
 # record coverage
 cd mafFilter
 for i in `cat ../chrom.lst` ; do nice mafCoverage hg17 $i.maf -count=2 > $i.cov ; echo done $i ; done   
 
 # CHIMP DELS FROM HG16 (DONE 2004-08-17 kate)
 # NOTE: this track just for development -- it should be regenerated from the latest
 #       alignments instead of lifted.
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mkdir -p chimpDels
     cd chimpDels
     hgsql -s hg16 -e "SELECT * FROM chimpDels" | cut -f 2- > chimpDels.hg16.bed
     liftOver chimpDels.hg16.bed \
         /cluster/data/hg16/bed/liftOver/hg16ToHg17.over.chain \
                 chimpDels.bed chimpDels.unmapped
     wc -l chimpDels.bed chimpDels.unmapped
     # 27662 chimpDels.bed
     # 132 chimpDels.unmapped
     # 27794 total
     hgLoadBed hg17 chimpDels chimpDels.bed -noBin
 
 ### CREATE chimpFixedDiff -- panTro1 (Daryl, August 18, 2005)
 
     # Convert chimp quality scores from uncompressed to compressed
     # chromosome format.  This took 22 minutes on crow.
     ## previously done for hg16
     # cd /cluster/data/panTro1
     # cat */chr*.qa | qaToQac stdin chrom.qac
 
     # Make single base pair high quality differences into a bed file
     # and load into database
     cd /cluster/data/hg17/bed
     mkdir chimpFixedDiff
     cd chimpFixedDiff
     sed 's/simpleNucDiff/chimpFixedDiff/' ~/kent/src/hg/lib/simpleNucDiff.sql > chimpFixedDiffs.sql
 
     # chimpHiQualDiffs was changed to allow different 
     #    quality parameters as command line options
 
     set axtDir = /cluster/data/hg17/bed/blastz.panTro1/axtRBestNet
     # This crashed twice at the same place, but ran successfully when
     # each chromosome was run separately.
     ##  time chimpFixedDiffs /$axtDir /cluster/data/panTro1/chrom.qac chimpFixedDiffs.bed >& chimpFixedDiffs.log
     mkdir chroms; cd chroms
     ls -1 $axtDir | grep chr | grep axt | sed 's/.axt//' | xargs mkdir
     # rmdir chr*random
     touch cfd.log
     foreach f (chr*)
 	echo -n $f "  "
 	ln -s /$axtDir/$f.axt $f/$f.axt
 	time nice chimpFixedDiffs $f /cluster/data/panTro1/chrom.qac $f.chimpFixedDiffs.bed>>& cfd.log
     end
     rm ../chimpFixedDiffs.bed
     cat chr*bed > ../chimpFixedDiffs.bed
 
     ## The load (sort) ran out of memory on hgwdev, so I sorted the
     ## file first on kolossus (3 minutes) and then loaded it on hgwdev
     ssh kolossus
     hgLoadBed -strict -sqlTable=chimpFixedDiffs.sql -noLoad hg16 chimpFixedDiff chimpFixedDiffs.bed
     exit
     ## hgwdev (37 minutes)
     hgLoadBed -hasBin -noSort -sqlTable=chimpFixedDiffs.sql hg16 chimpFixedDiff bed.tab
 
     TODO: need to filter out polymorphic sites (SNPs)
 
 
 
 
 
 #  Load firstEF track (DONE 2004-08-18 braney)
     ssh hgwdev
     mkdir -p /cluster/data/hg17/bed/firstEF
     cd /cluster/data/hg17/bed/firstEF
     wget "http://bioinformatics.med.ohio-state.edu/downloads/firstEFMay04.bed.gz"
     cat << '_EOF_' > sedScript
 s/chr23/chrX/g
 s/chr24/chrY/g
 /^>/d
 /^$/d
 /^No/d
 '_EOF_'
 
     zcat firstEFMay04.bed.gz | sed -f sedScript | awk  "{OFS=\"\t\"} {\$3 +=1; print  \$0}" > firstEF.bed
     hgLoadBed hg17 firstEF firstEF.bed
     rm firstEF.tab
     gzip *.bed
 #done firstEF
 
 
 # GENE BOUNDS (RNACLUSTER) (DONE 08-18-2004 Chuck)
 # Create rnaCluster table (depends on {est,mrna}OrientInfo)
 cd ~sugnet/store1/altSplice/hg17/
 mkdir rnaCluster
 cd rnaCluster/
 mkdir chrom
 # Create a list of accessions that come from RAGE libraries and need to be excluded.
 ~/kent/src/hg/geneBounds/clusterRna/generateRageAccList.csh hg17 rage.libs
 foreach f (/cluster/data/hg17/nib/chr*.nib)
     set c = $f:t:r
     set out = chrom/$c.bed
     # Exclude accesions in the RAGE file
     echo clusterRna -mrnaExclude=hg17.rage.libs hg17 /dev/null $out -chrom=$c
     clusterRna -mrnaExclude=hg17.rage.libs hg17 /dev/null $out -chrom=$c
 end
 hgLoadBed hg17 rnaCluster chrom/*.bed
 mkdir /cluster/data/hg17/bed/rnaCluster
 cp -r chrom /cluster/data/hg17/bed/rnaCluster
 
 #  miRNA track (CORRECTION 2004-12-09 - Hiram)
     #	Received the following correction from Michel Weber:
     #	Could you please replace the two lines:
     #	chr6 72169974 72170045 hsa-mir-30a 480 - 72169977 72169999
     #	chr6 72170017 72170040 hsa-mir-30a-5p 480 - 72170017 72170040
     #	by:
     #	chr6 72169974 72170045 hsa-mir-30a 480 - 72169977 72169999
     #	chr6 72169974 72170045 hsa-mir-30a 480 - 72170017 72170040
     #	(The first line remains identical, only the second is changed. The 
     #	repetition of the hsa-mir-30a entry means that both strands of its
     #	hairpin structure are matured into microRNAs, named hsa-miR-30a-3p and 
     #	hsa-miR-30a-5p in Rfam database).
     ssh hgwdev
     cd /cluster/data/hg17/bed/miRNA
     mv miRNA_hg17_1.bed miRNA_hg17_1.bed.0
     cp miRNA_hg17_1.bed.0 miRNA_hg17_1.bed
     #	edit miRNA_hg17_1.bed to change the single line.  Then:
     mv hg17.bed hg17.bed.0
     egrep -v "^track |^browser " miRNA_hg17_1.bed | \
 	sed -e "s/miR/mir/g; s/ sa-mir/ hsa-mir/g; s/ /\t/g;" > hg17.bed
     #	Check that the edit is in place properly:
     diff hg17.bed.0 hg17.bed
     #	and load it
     hgLoadBed hg17 miRNA hg17.bed
     #	Loaded 221 elements of size 8
     #	featureBits remains the same:
     featureBits hg17 miRNA
     #	18052 bases of 2866216770 (0.001%) in intersection
 
 #  miRNA track (DONE 2004-09-03 - Hiram)(CORRECTED, see above 2004-12-09)
     #	The source data for this was received via email from Sam
     #	Griffiths-Jones to Donna 16 August 2004.  In other email Michel
     #	Weber asked to add one more data line to that file.
 
     #	data from: Sam Griffiths-Jones <sgj@sanger.ac.uk>
     #	and Michel.Weber@ibcg.biotoul.fr
     #	notify them if this assembly updates to renew this track
     cd /cluster/data/hg17/bed
     mkdir miRNA
     cd miRNA
     #	one name was missing the h in hsa-mir and one was miR instead of
     #	mir
     egrep -v "^track |^browser " miRNA_hg17_1.bed | \
 	sed -e "s/miR/mir/g; s/ sa-mir/ hsa-mir/g; s/ /\t/g;" > hg17.bed
     hgLoadBed hg17 miRNA hg17.bed
     #	compare with previous results, should be relatively similar
     #	featureBits hg16 miRNA
     #	16923 bases of 2865248791 (0.001%) in intersection
     #	featureBits hg17 miRNA
     #	18052 bases of 2866216770 (0.001%) in intersection
 
     # entry is already in trackDb/trackDb.ra
 
 
 ## blastz mRNA track for internal use - Robert 8/12/04
 mkdir /cluster/bluearc/hg17/mrnaBlastz
 cd /cluster/bluearc/hg17/mrnaBlastz
 /cluster/data/genbank/bin/i386/gbGetSeqs -gbRoot=/cluster/data/genbank genbank mrna mrna.fa -db=hg -native
 mkdir -p split
 faTrimPolyA mrna.fa trim.fa
 faSplit about trim.fa 1000000 split/mrna
 cp -ip trim.fa /panfs/ucsc.edu/home/scratch/hg17/mrnaBlastz
 faSize trim.fa -detailed=on > S2.len
 hgsql hg16 < chromInfo.sql > S1.len
 
 BlastZ_run0.sh
 cd run.0
 para push
 para time
 #113894 jobs in batch
 #207911 jobs (including everybody's) in Parasol queue.
 #Checking finished jobs
 #Completed: 113894 of 113894 jobs
 #CPU time in finished jobs:   14423845s  240397.41m  4006.62h  166.94d  0.457 y
 #IO & Wait Time:                334352s    5572.54m    92.88h    3.87d  0.011 y
 #Average job time:                 130s       2.16m     0.04h    0.00d
 #Longest job:                    38301s     638.35m    10.64h    0.44d
 #Submission to last job:         59841s     997.35m    16.62h    0.69d
 
 mkdir run.1
 ~angie/hummus/do.out2lav DEF > run.1/j
 cd run.1
 para create j
 para push
 para time
 #341 jobs in batch
 #208550 jobs (including everybody's) in Parasol queue.
 
 #Checking finished jobs
 #Completed: 341 of 341 jobs
 #CPU time in finished jobs:      28990s     483.17m     8.05h    0.34d  0.001 y
 #IO & Wait Time:                 43139s     718.98m    11.98h    0.50d  0.001 y
 #Average job time:                 212s       3.53m     0.06h    0.00d
 #Longest job:                     2015s      33.58m     0.56h    0.02d
 #Submission to last job:          2187s      36.45m     0.61h    0.03d
 
 #!/bin/tcsh
 set base="/cluster/bluearc/hg17/mrnaBlastz"
     cd $base
     mkdir -p pslRaw
     foreach c (lav/*)
       pushd $c
       set chr=$c:t
       set out=$base/pslRaw/$chr.psl
       echo "Translating $chr lav to $out"
       cat `ls -1 *.lav | sort -g` \
         | lavToPsl stdin stdout \
         | sed -e 's@scratch/hg/gs.18/build35/bothMaskedNibs//@@' | sed -e 's/\.nib:[0-9]*-[0-9]*//' > $out
       popd
     end
 
 mkdir run.2
 
 for i in `awk '{print $1}' S1.len` ; do echo doSortFilter.sh ../pslRaw/$i.psl ../pslFilter/$i.psl  >> run.2/spec.dup ; done
 cd run.2
 para create spec.dup
 para push
 #46 jobs in batch
 #3 jobs (including everybody's) in Parasol queue.
 #Checking finished jobs
 #Completed: 46 of 46 jobs
 #CPU time in finished jobs:       4409s      73.48m     1.22h    0.05d  0.000 y
 #IO & Wait Time:                  1082s      18.04m     0.30h    0.01d  0.000 y
 #Average job time:                 119s       1.99m     0.03h    0.00d
 #Longest job:                     3842s      64.03m     1.07h    0.04d
 #Submission to last job:          3842s      64.03m     1.07h    0.04d
 cd ..
 for i in `awk '{print $1}' S1.len` ; do echo axtChain -linearGap=linearGap.txt -psl pslFilter/$i.psl /scratch/hg/gs.18/build35/bothMaskedNibs/ -faQ /panfs/ucsc.edu/home/scratch/hg17/mrnaBlastz/trim.fa chain/$i.chain >> spec.chain ; done
 para create spec.chain
 para push
 
 cd run.3
 para create spec.filter
 para push
 cd ..
 ls /cluster/data/hg17/nib/*.nib > S1.lst
 #Skip chainPreNet it is not good for mrna
 #mkdir -p preNet
 #
 #cd chainFilter
 #foreach i ( *.chain)
 #chainPreNet $i ../S1.len ../S2.len ../preNet/$i
 #end
 
 mkdir run.4
 cd run.4
 for i in `awk '{print $1}' ../S1.len`; do echo "chainToPsl ../chainFilter/$i.chain ../S1.len ../S2.len ../S1.lst /panfs/ucsc.edu/home/scratch/hg17/mrnaBlastz/trim.fa ../psl/$i.psl" >> spec.chain2psl.new  ; done
 
 pslCat psl/*psl > mrnaBlastz.psl
 hgLoadPsl hg17 mrnaBlastz.psl
 
 cp trim.fa /cluster/data/hg17/bed/mrnaBlastz/hg17Mrna.fa
 ln /cluster/data/hg17/bed/mrnaBlastz/hg17Mrna.fa /gbdb/hg17/mrnaBlastz/ -s
 
 hgLoadSeq -prefix=bz hg17 /gbdb/hg17/mrnaBlastz/hg17Mrna.fa
 
 ## end of blastz Mrna track
 
 #### BUILD RETROGENE TRACK (done Robert 8/26/2004)
 #### REBUILD RETROGENE TRACK (done Robert 12/24/2004 - but no notes - kuhn)
 # diffs before push to beta:
 #  1640 hg17.pseudoGeneLink.devOnly
 #  9639 hg17.pseudoGeneLink.betaOnly
 # 15091 hg17.pseudoGeneLink.common
 
 #  RETROGENE TRACK data update - Robert - 2005-04-08
    (added by Jen 2006-01-31)
    - pushQ entry did not include psuedoMrna table. Old table is still
      present on RR. New data has since been lost on dev. 
      User impact: ~1000 sequence missing links in browser 
    - new all.joiner rule needed to link psuedoMrna to pseudoGeneLink table
    - current all.joiner rule between knownGene to pseudoGeneLink gives errors.
      the data types appear to be mismatched. pseudoGeneLink.kgName is
      a gene symbol, not the same identifier as in knownGene.name
    - data is to be regenerated soon and errors corrected at that time
 
 
 mkdir /cluster/data/hg17/bed/pseudo
 cd /cluster/data/hg17/bed/pseudo
 ls /cluster/data/hg17/nib/*.nib > S1.lst
 hgsql hg17 -N -B < allMrna.sql > allMrna.lst
 cp  /cluster/data/genbank/data/aligned/genbank.142.0/hg17/full/mrna.native.psl.gz .
 gunzip mrna.native.psl.gz
 awk  '{OFS="\t";print $1,$2,$3,$4,$5,$6,$7,$8,$9,substr($10,1,index($10,".")-1),$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23}' mrna.native.rawPsl  > mrnaBlat.psl
 
 hgsql hg17 -N -B < refGene.sql > refGene.tab
 hgsql hg17 -B -N < mgcGene.sql > mgcGene.tab
 cat ../../*/*.fa.out | awk '$5~/chr*/{OFS="\t";print $5,$6,$7}' >rmsk.bed
 
 cd /cluster/bluearc/hg17/mrnaBlastz/
 zcat /cluster/data/hg17/bed/blastz.mm5/axtChain/mouseSyn.net.gz | netToBed stdin mouseSyn.bed
 
 hgsql hg17 < mrna.sql | grep -v matches | awk '{OFS="\t"; print $2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22}' > all_mrna.psl
 ssh eieio
 pslCat -nohead -check all_mrna.psl /cluster/bluearc/hg17/mrnaBlastz/psl/*.psl |awk '{print $0, $1*3-$2}' | sort -k 10,10 -k 22nr -T /tmp | awk '{OFS="\t"; print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21}' >  blatBlastzHg17.psl 
 ssh hgwdev
 cp blatBlastzHg17.psl /scratch/
 
 tawk '$12 > 1 && $12 < 9999999{x=$11;$11=$12;$12=x;print $0}'  /cluster/data/kgDB/bed/hg17/kgBestMrna/sortedKnownGene.tab > sortedKnownGene.tab
 #copy files to iServers for cluster run
 ssh kkr1u00
 /cluster/home/baertsch/bin/i386/pslSplit nohead -chunkSize=121 /iscratch/i/gs.18/build35/pseudo blatBlastzHg17.psl
 cd /cluster/data/hg17/bed/pseudo
 cp refGene.tab /iscratch/i/gs.18/build35/pseudo
 cp /cluster/data/hg17/bed/simpleRepeat/simpleRepeat.bed /iscratch/i/gs.18/build35/pseudo
 cp mrnaHg17.fa /iscratch/i/gs.18/build35/pseudo
 cp sortedKnownGene.tab /iscratch/i/gs.18/build35/pseudo
 cp rmsk.bed /iscratch/i/gs.18/build35/pseudo
 cp all_mrna.psl /iscratch/i/gs.18/build35/pseudo
 cp mouseSyn.bed /iscratch/i/gs.18/build35/pseudo
 for i in `ls tmp*` ; do echo "doBuildkk.sh ${i%%.psl}" ; done | sed -e 's/tmp//g'  > ~/hg17/pseudo/spec.kk   
 cd /iscratch/i/hg/gs.18/build35/pseudo
 iSync
 
 para create spec.kk
 para push
 
 #post process
 
 # run from eieio
 BLUE=/cluster/bluearc/hg17/pseudo
 echo catting output
 cat $BLUE/pseudoGeneLink[0-9]*.bed | sort -k1,1 -k2,3n >pseudoGeneLinkSort.bed ; /bin/rm $BLUE/pseudoGeneLink[0-9]*.bed
 cat $BLUE/pseudo[0-9]*.psl > pseudo.psl ; /bin/rm $BLUE/pseudo[0-9]*.psl &
 echo Filtering pseudoGeneLinkSort.bed
 tawk '$5 > 10 && $15 > 10000 && $35 > 650 {OFS="\t";print $0}' pseudoGeneLinkSort.bed > pseudoGeneLinkSortFilter.bed
 echo Removing Overlaps
 doSplit
 cd /cluster/bluearc/hg17/pseudo/run.o 
 spec.overlap
 cd ~/hg17/pseudo
 cat /cluster/bluearc/hg17/pseudo/chr*pseudoNoOverlap.bed > pseudoGeneLinkNoOverlap.bed
 echo Making psl
 awk '{printf("%s\t%s\t%s\n", $5,$2,$3)}' pseudoGeneLinkNoOverlap.bed > pseudoGeneLinkSelect.tab
 ## 350 is the sacrad magic number and will probably change
 tawk '$6>=350{print $0}' pseudoGeneLinkNoOverlap.bed > pseudoGeneLinkNoOverlapFilter.bed                       
 
 pslSelect -qtStart=pseudoGeneLinkSelect.tab pseudo.psl pseudoMrna.psl
 echo Loading Bed
 hgLoadBed hg17 pseudoGeneLink pseudoGeneLinkNoOverlapFilter.bed -hasBin -sqlTable=/cluster/home/baertsch/kent/src/hg/lib/pseudoGeneLink.sql
 echo Loading Psl
 hgLoadPsl hg17 pseudoMrna.psl
 
 ## end of retroGene track
 
 
 # 3-WAY MULTIZ MULTIPLE ALIGNMENT (MM5, RN3) (DONE 2004-08-27 kate)
 #       HMR Maf's needed for regulatory potential track
     ssh eieio
     set multizDir = multiz.2004-08-27
     cd /cluster/data/hg17/bed/$multizDir
     set workingDir = /cluster/bluearc/hg17/$multizDir
     ln -s $workingDir /cluster/bluearc/hg17/multiz3way
     ln -s $multizDir multiz3way
     mkdir -p $workingDir
     mkdir -p /cluster/data/hg17/bed/$multizDir
 
 # wrapper script for multiz
     # NOTE: first arg is pairwise, 2nd arg is multiple (to add to) 
     # NOTE: next time, modify script so it only needs one arg -- saves the
     # multiple dirname in a file for use by the next run
     cat << 'EOF' > doMultiz.csh
 #!/bin/csh -fe
 mkdir -p $3:h
 /cluster/bin/penn/multiz $1 $2 - > $3
 'EOF'
 
 # << for emacs
     cat << 'EOF' > gsub
 #LOOP
 ../doMultiz.csh {check in line /cluster/bluearc/hg17/multiz.2004-08-27/$(dir1)/$(root2).maf} {check in line /cluster/bluearc/hg17/multiz.2004-08-27/$(root1)/$(root2).maf} {check out line+ /cluster/bluearc/hg17/multiz.2004-08-27/$(root1)$(dir1)/$(root2).maf}
 #ENDLOOP
 'EOF'
 # << for emacs
 
     chmod +x doMultiz.csh
 
     ssh eieio
     set workingDir = /cluster/bluearc/hg17/multiz.2004-08-27
 
     # copy mafs to bluearc -- chimp
     mkdir $workingDir/mm5
     cp /cluster/data/hg17/bed/blastz.mm5/mafNet/*.maf \
                 $workingDir/mm5
     ls $workingDir/mm5/*.maf > chrom.lst
 
     # rat
     mkdir $workingDir/rn3
     cp /cluster/data/hg17/bed/blastz.rn3/mafNet/chr*.maf $workingDir/rn3
 
     # multiz - add in rn3 rat to human/mouse
     # 
     ssh kki
     set multizDir = multiz.2004-08-27
     set workingDir = /cluster/bluearc/hg17/$multizDir
     cd /cluster/data/hg17/bed/$multizDir
     mkdir run.rn3
     cd run.rn3
     echo "rn3/mm5" > species.lst
     gensub2 species.lst ../chrom.lst ../gsub jobList
     para create jobList
         # 47 jobs
     para try, check, push, check
 
     # copy 3-way mafs to build directory
     ssh eieio
     set multizDir = multiz.2004-08-27
     set workingDir = /cluster/bluearc/hg17/$multizDir
     ln -s $workingDir/mm5rn3 $workingDir/maf
     cd /cluster/data/hg17/bed/multiz.2004-08-27
     mkdir maf
     cp $workingDir/maf/*.maf maf
 
 
 # BLASTZ TETRAODON (tetNig1) (DONE, 2004-08-26, hartera)
     # blastz requires lineage-specific repeats
     # Treat all repeats as lineage-specific.
     ssh kkr1u00
     mkdir /iscratch/i/gs.18/build35/linSpecRep.notInTetraodon
  foreach f (/iscratch/i/gs.18/build35/rmsk/chr*.fa.out)
  cp -p $f /iscratch/i/gs.18/build35/linSpecRep.notInTetraodon/$f:t:r:r.out.spec
     end
 
     mkdir /iscratch/i/tetNig1/linSpecRep.notInHuman
     foreach f (/iscratch/i/tetNig1/rmsk/chr*.fa.out)
       cp -p $f /iscratch/i/tetNig1/linSpecRep.notInHuman/$f:t:r:r.out.spec
     end
     iSync
 
     ssh kk
     mkdir -p /cluster/data/hg17/bed/blastz.tetNig1.2004-08-20
     ln -s /cluster/data/hg17/bed/blastz.tetNig1.2004-08-20 \
             /cluster/data/hg17/bed/blastz.tetNig1
     cd /cluster/data/hg17/bed/blastz.tetNig1
  
     # abridge repeats.
     # Treat all repeats as lineage-specific.
     cat << '_EOF_' > DEF
 # human vs. Tetraodon
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin
 
 # use same parameters as for danRer1-fr1
 ALIGN=blastz-run
 BLASTZ=blastz
 BLASTZ_H=2000
 BLASTZ_ABRIDGE_REPEATS=1
 
 # TARGET: Human
 SEQ1_DIR=/iscratch/i/hg17/bothMaskedNibs
 SEQ1_RMSK=
 SEQ1_FLAG=
 SEQ1_SMSK=/iscratch/i/hg17/linSpecRep.notInTetraodon
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY: Tetraodon
 SEQ2_DIR=/iscratch/i/tetNig1/nib
 SEQ2_RMSK=
 SEQ2_FLAG=
 SEQ2_SMSK=/iscratch/i/tetNig1/linSpecRep.notInHuman
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=10000000
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/blastz.tetNig1
 
 DEF=$BASE/DEF
 RAW=$BASE/raw
 CDBDIR=$BASE
 SEQ1_LEN=$BASE/S1.len
 SEQ2_LEN=$BASE/S2.len
 
 #DEBUG=1
 '_EOF_'
     # << this line keeps emacs coloring happy
     # save the DEF file in the current standard place
     chmod +x DEF
     cp DEF ~angie/hummus/DEF.hg17-tetNig1.2004-08-20
  
     # make sure BlastZ_run0.sh, BlastZ_run1.sh and BlastZ_run2.sh scripts
     # are in /cluster/data/hg17/jkStuff
     # edit BlastZ_run0.sh so directory for blastz is /cluster/bin/penn
 
     bash # if a csh/tcsh user
     . ./DEF
     /cluster/data/hg17/jkStuff/BlastZ_run0.sh
     cd run.0
     # check batch looks ok then
     para try, check, push, check, ....
 # para time
 # Completed: 19437 of 19437 jobs
 # CPU time in finished jobs:    3225816s   53763.60m   896.06h   37.34d  0.102 y
 # IO & Wait Time:                174096s    2901.60m    48.36h    2.01d  0.006 y
 # Average job time:                 175s       2.92m     0.05h    0.00d
 # Longest job:                      709s      11.82m     0.20h    0.01d
 # Submission to last job:          5324s      88.73m     1.48h    0.06d
 
     # second cluster run: lift raw alignments -> lav dir
     ssh kki
     cd /cluster/data/hg17/bed/blastz.tetNig1
     bash # if a csh/tcsh user
     . ./DEF
     /cluster/data/hg17/jkStuff/BlastZ_run1.sh
     cd run.1
     para try, check, push, check etc.
 # para time
 # Completed: 341 of 341 jobs
 # CPU time in finished jobs:        280s       4.66m     0.08h    0.00d  0.000 y
 # IO & Wait Time:                  2183s      36.39m     0.61h    0.03d  0.000 y
 # Average job time:                   7s       0.12m     0.00h    0.00d
 # Longest job:                       41s       0.68m     0.01h    0.00d
 # Submission to last job:           469s       7.82m     0.13h    0.01d
 
     # third run: lav -> axt
     ssh kki
     cd /cluster/data/hg17/bed/blastz.tetNig1
     mkdir axtChrom run.2
     cd run.2
     cat << '_EOF_' > do.csh
 #!/bin/csh -ef
 cd $1
 cat `ls -1 *.lav | sort -g` \
 | lavToAxt stdin /iscratch/i/gs.18/build35/bothMaskedNibs \
 /iscratch/i/tetNig1/nib stdout \
 | axtSort stdin $2
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod a+x do.csh
     cat << '_EOF_' > gsub
 #LOOP
 ./do.csh {check in exists $(path1)} {check out line+ /cluster/data/hg17/bed/blastz.tetNig1/axtChrom/$(root1).axt}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
     \ls -1Sd ../lav/chr* > chrom.list
     gensub2 chrom.list single gsub jobList
     wc -l jobList
     head jobList
     para create jobList
     para try, check, push, check,...
 # para time
 # Completed: 45 of 46 jobs
 # Crashed: 1 jobs
 # CPU time in finished jobs:         52s       0.87m     0.01h    0.00d  0.000 y
 # IO & Wait Time:                   256s       4.27m     0.07h    0.00d  0.000 y
 # Average job time:                   7s       0.11m     0.00h    0.00d
 # Longest job:                       36s       0.60m     0.01h    0.00d
 # Submission to last job:           275s       4.58m     0.08h    0.00d
 
 # one job crashed because chr6_hla_hap1.axt is empty. Checked by running this
 # again and then looked at the lav file which has no alignments in it.
                                                                                 
    # translate sorted axt files into psl
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.tetNig1
     mkdir -p pslChrom
     set tbl = "blastzTetNig1"
     foreach f (axtChrom/chr*.axt)
       set c=$f:t:r
       echo "Processing chr $c"
       /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
     end
 
     # Load database tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.tetNig1/pslChrom
     foreach f (./*.psl)
       /cluster/bin/i386/hgLoadPsl -noTNameIx hg17 $f
       echo "$f Done"
     end
 
 # original blastzTetNig1: 
 # BLASTZ_H=2000
 # BLASTZ_Y=3400
 # BLASTZ_L=6000
 # BLASTZ_K=2200
 # BLASTZ_Q=/cluster/data/blastz/HoxD55.q
 # BLASTZ_ABRIDGE_REPEATS=1
 # featureBits -chrom=chr1 hg17 blastzTetNig1
 # 6378680 bases of 222827847 (2.863%) in intersection
 # featureBits -chrom=chr1 hg17 refGene:cds blastzTetNig1 -enrichment
 # refGene:cds 1.246%, blastzTetNig1 2.863%, both 0.856%, cover 68.70%, 
 # enrich 24.00x
 # featureBits -chrom=chr1 hg17 refGene:cds blastzDanRer1 -enrichment
 # refGene:cds 1.246%, blastzDanRer1 3.934%, both 0.831%, cover 66.72%, 
 # enrich 16.96x
 # comparable to zebrafish so good
 # try same parameters with L=8000
 # featureBits -chrom=chr1 hg17 refGene:cds blastzTetNig1L8k -enrichment
 # refGene:cds 1.246%, blastzTetNig1L8k 2.095%, both 0.753%, cover 60.47%, 
 # enrich 28.87x
 # load chr1 with blastz using just H=2000 and default parameters
 # featureBits -chrom=chr1 hg17 refGene:cds blastzTetNig1Default -enrichment
 # refGene:cds 1.246%, blastzTetNig1Default 1.630%, both 0.808%, cover 64.87%, 
 # enrich 39.80x
 # rows in chr1_blastzTetNig1 tables 
 # blastzTetNig1	95156
 # blastzTetNig1L8k 58015
 # blastzTetNig1Default 71342
 # The default values also used for danRer1 vs fugu give good coverage and
 # higher enrichment than blastzTetNig1 with less alignments so this will be
 # used for the blastz track - now called blastzTetNig1.
 
 # CHAIN TETRAODON (tetNig1) BLASTZ (DONE, 2004-08-26, hartera)
 # Make chains with rescored blastz
     # Run axtChain on little cluster
     ssh kki
     cd /cluster/data/hg17/bed/blastz.tetNig1
     mkdir -p axtChain/run1
     cd axtChain/run1
     mkdir out chain
     ls -1S /cluster/data/hg17/bed/blastz.tetNig1/axtChrom/*.axt \
         > input.lst
 cat << '_EOF_' > gsub
 #LOOP
 doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 
     # Make our own linear gap file with reduced gap penalties, 
     # in hopes of getting longer chains - works well for species at 
     # chicken-human distance or greater
     cat << '_EOF_' > ../../chickenHumanTuned.gap
 tablesize	11
 smallSize	111
 position	1	2	3	11	111	2111	12111	32111	72111	152111	252111
 qGap	325	360	400	450	600	1100	3600	7600	15600	31600	56600
 tGap	325	360	400	450	600	1100	3600	7600	15600	31600	56600
 bothGap	625	660	700	750	900	1400	4000	8000	16000	32000	57000
 '_EOF_'
     # << this line makes emacs coloring happy
 
 cat << '_EOF_' > doChain
 #!/bin/csh
 axtChain -linearGap=../../chickenHumanTuned.gap $1 \
     /iscratch/i/gs.18/build35/bothMaskedNibs \
     /iscratch/i/tetNig1/nib $2 >& $3
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod a+x doChain
     gensub2 input.lst single gsub jobList
     para create jobList
     para try, check, push, check...
 # para time
 # Completed: 45 of 46 jobs
 # Crashed: 1 jobs
 # CPU time in finished jobs:        553s       9.22m     0.15h    0.01d  0.000 y
 # IO & Wait Time:                   102s       1.69m     0.03h    0.00d  0.000 y
 # Average job time:                  15s       0.24m     0.00h    0.00d
 # Longest job:                       56s       0.93m     0.02h    0.00d
 # Submission to last job:           985s      16.42m     0.27h    0.01d
 
 # one job crashed since chr6_hla_hap1.axt is empty - no alignments
 
    # now on the cluster server, sort chains
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
     chainMergeSort run1/chain/*.chain > all.chain
     chainSplit chain all.chain
    
     # take a look at score distr's
     foreach f (chain/*.chain)
       grep chain $f | awk '{print $2;}' | sort -nr > /tmp/score.$f:t:r
       echo $f:t:r >> hist5000.out
       textHistogram -binSize=5000 /tmp/score.$f:t:r >> hist5000.out
       echo ""
     end
 
     # only chr19 has a very large number of chains with score < 5000
     # load chr 1 into table
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain/chain
     hgLoadChain hg17 chr1_chainTetnig1 chr1.chain
 
 # featureBits -chrom=chr1 hg17 refGene:cds chainTetnig1Link -enrichment
 # refGene:cds 1.246%, chainTetnig1Link 1.582%, both 0.805%, cover 64.59%, 
 # enrich 40.83x
 
     # try filtering with minScore of 5000
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
     mv all.chain all.chain.unfiltered
     chainFilter -minScore=5000 all.chain.unfiltered > all.chain
     rm -r chain
     chainSplit chain all.chain
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain/chain
     hgLoadChain hg17 chr1_chainTetNig1Filt5k chr1.chain
 # featureBits -chrom=chr1 hg17 refGene:cds chainTetNig1Filt5kLink -enrichment
 # refGene:cds 1.246%, chainTetNig1Filt5kLink 1.487%, both 0.789%, cover 63.33%,
 # enrich 42.58x
 # this cleans it up a lot with little reduction in coverage.
 # check in browser - filtered version looks good.
 
 # add all chains for minScore=5000 filtered chains
 # remove test chain tables for chr1
     ssh hgwdev
     hgsql -e "drop table chr1_chainTetnig1;" hg17
     hgsql -e "drop table chr1_chainTetnig1Link;" hg17
     hgsql -e "drop table chr1_chainTetNig1Filt5k;" hg17
     hgsql -e "drop table chr1_chainTetNig1Filt5kLink;" hg17
 
     cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain/chain
     foreach i (*.chain)
         set c = $i:r
         hgLoadChain hg17 ${c}_chainTetNig1 $i
         echo done $c
     end
 
 # NET TETRAODON (tetNig1) BLASTZ (DONE, 2004-08-26, hartera)
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
     mkdir preNet
     cd chain
     foreach i (*.chain)
        echo preNetting $i
        /cluster/bin/i386/chainPreNet $i ../../S1.len ../../S2.len \
                                      ../preNet/$i
     end
     cd ..
     mkdir n1
     cd preNet
     foreach i (*.chain)
       set n = $i:r.net
       echo primary netting $i
       /cluster/bin/i386/chainNet $i -minSpace=1 ../../S1.len ../../S2.len \
                                  ../n1/$n /dev/null
     end
     cd ..
     cat n1/*.net | /cluster/bin/i386/netSyntenic stdin noClass.net
     # memory usage 55373824, utime 415 s/100, stime 45
     # Add classification info using db tables:
     cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
     # netClass looks for ancient repeats in one of the databases
     # hg17 has this table - hand-curated by Arian but this is for
     # human-rodent comparisons so do not use here, use -noAr option
     mkdir -p /cluster/bluearc/hg17/linSpecRep.notInTetraodon
     mkdir -p /cluster/bluearc/tetNig1/linSpecRep.notInHuman
     cp /iscratch/i/hg17/linSpecRep.notInTetraodon/* \
        /cluster/bluearc/hg17/linSpecRep.notInTetraodon
     cp /iscratch/i/tetNig1/linSpecRep.notInHuman/* \
        /cluster/bluearc/tetNig1/linSpecRep.notInHuman
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
     time netClass noClass.net hg17 tetNig1 tetNig1.net \
           -tNewR=/cluster/bluearc/hg17/linSpecRep.notInTetraodon \
           -qNewR=/cluster/bluearc/tetNig1/linSpecRep.notInHuman -noAr
     # 54.100u 31.890s 2:20.01 61.4%   0+0k 0+0io 197pf+0w
     netFilter -minGap=10 tetNig1.net |  hgLoadNet hg17 netTetNig1 stdin
     # featureBits hg17 refGene:cds netTetNig1 -enrichment
     # refGene:cds 0.978%, netTetNig1 25.095%, both 0.778%, cover 79.53%, 
     # enrich 3.17x
 
 # TWINSCAN 1.3 GENE PREDICTIONS (Done, 2004-Aug-26, heather)
 
     cd /cluster/data/hg17/bed
     mkdir twinscan
     tarFile=hg17_TS13_pseudomasked.tar.gz
     wget http://genes.cs.wustl.edu/predictions/human/NCBI35/hg17_TS13_pseudomasked.tar.gz
     wget http://genes.cs.wustl.edu/predictions/human/NCBI35/md5sum.txt
 
     # check file transferred correctly
     grep gz md5sum.txt > gz.sum
     md5sum $tarFile | diff - gz.sum
 
     # extract
     tar xvfz $tarFile
     unset tarFile 
     
     # check that files unzipped and untarred correctly
     # expect no differences
     cd chr_gtf
     grep gtf ../md5sum.txt > md5sum.txt
     cd ../chr_ptx
     grep ptx ../md5sum.txt > md5sum.txt
     cd ../chr_tx
     grep tx ../md5sum.txt > md5sum.txt
     cd ..
     
     md5sum chr_gtf/* > gtf.sum
     diff gtf.sum chr_gtf/md5sum.txt
 
     md5sum chr_ptx/* > ptx.sum
     diff ptx.sum chr_ptx/md5sum.txt
 
     md5sum chr_tx/* > tx.sum
     diff tx.sum chr_tx/md5sum.txt
 
     # pare down protein FASTA header to id and add missing .a:
     foreach c (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y)
       echo chr$c
       perl -wpe 's/^(\>\S+)\s.*$/$1.a/' < chr_ptx/chr$c.ptx > chr_ptx/chr$c-fixed.fa
     end
     ldHgGene hg17 twinscan chr_gtf/chr*.gtf -gtf -genePredExt
     hgPepPred hg17 generic twinscanPep chr_ptx/chr*-fixed.fa
 
 # MAKE VSTETNIG1 DOWNLOADABLES (DONE, 2004-09-08, hartera)
 #    Replace with gzipped versions (DONE 2004-09-14 kate)
     ssh kksilo
     # zip chains and nets
     cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
     cp all.chain tetNig1.chain
     zip -j /cluster/data/hg17/zip/tetNig1.chain.zip tetNig1.chain
     rm tetNig1.chain
     zip -j /cluster/data/hg17/zip/tetNig1.net.zip tetNig1.net
    
     ssh hgwdev
     # copy chains and nets to downloads area
     set gp = /usr/local/apache/htdocs/goldenPath/hg17
     mkdir -p $gp/vsTetNig1
     cd $gp/vsTetNig1
     mv /cluster/data/hg17/zip/tetNig1*.zip .
     md5sum *.zip > md5sum.txt
 
     # move axt files to downloads area and zip
     cd /cluster/data/hg17/bed/blastz.tetNig1/axtChrom
     mkdir -p $gp/vsTetNig1/axtChrom
     cp -p *.axt $gp/vsTetNig1/axtChrom
     cd $gp/vsTetNig1/axtChrom
     gzip *.axt
     md5sum *.gz > md5sum.txt
 
     # Copy over & edit README.txt w/pointers to chain, net formats.
 
 # BLASTZ TETRAODON (tetNig1) CLEANUP (DONE, 2004-09-10, hartera)
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.tetNig1
     nice rm -rf raw &
     nice rm -rf lav &
     nice rm axtChain/run1/chain/* &
     nice gzip {axt,psl}Chrom/* axtChain/{all.chain,*.net} &
 
 
 # regulatory potential 2X track (WORKING - 2004-09-14 - Hiram)
     ssh eieio
     mkdir /cluster/store3/gs.18/build35/bed/regPotential2X
     mkdir /cluster/store3/gs.18/build35/bed/regPotential3X
     cd /cluster/data/hg17/bed
     ln -s /cluster/store3/gs.18/build35/bed/regPotential2X .
     ln -s /cluster/store3/gs.18/build35/bed/regPotential3X .
     cd regPotential2X
     wget --timestamping 'http://www.bx.psu.edu/~james/stuff/rp_kit.tgz' .
     tar xvzf rp_kit.tgz
     # fixup the hmr_rp_score.sh and hm_rp_score.sh to set
     #	RP=. to read: RP=/cluster/data/hg17/bed/regPotential2X/rp_kit
     #	And fix the usage of SHIFT and WINDOW, the following diff shows
     #	the changes:
     #	5c5
     #	< RP_DIR=/cluster/data/hg17/bed/regPotential2X/rp_kit
     #	---
     #	> RP_DIR=.
     #	8,9c8,9
     #	< MAPPING=rp_kit/hm_5a_mapping.txt
     #	< MATRIX=rp_kit/hm_5a+3_scoreMatrix.dat
     #	---
     #	> MAPPING=hm_5a_mapping.txt
     #	> MATRIX=hm_5a+3_scoreMatrix.dat
     #	12c12
     #	< SHIFT=1
     #	---
     #	> SHIFT=5
     #	24,25c24,25
     #	<     --shiftAmount $SHIFT \
     #	<     --windowSize $WINDOW \
     #	---
     #	>     --shiftAmount 5 \
     #	>     --windowSize 100 \
 
     mkdir maf
     for A in  `(cd /cluster/data/hg17/bed/blastz.mm5/axtNet; ls chr*.axt)`
     do
 	C=${A/.axt}
 	echo "/cluster/data/hg17/bed/blastz.mm5/axtNet/${A} -> maf/${C}.maf.gz"
 	axtToMaf /cluster/data/hg17/bed/blastz.mm5/axtNet/${A} \
 		/cluster/data/hg17/chrom.sizes /cluster/data/mm5/chrom.sizes \
 		stdout | gzip > maf/${C}.maf.gz
     done
     # Replace bad chr5 axtNet and mafNet (2006-01-05 kate) 
 
     #	a valid java runtime is only on hgwdev.  This is a java procedure
     ssh hgwdev
     cd /cluster/data/hg17/bed/regPotential2X
     mkdir rp_scores
     #  WARNING - the following loop takes almost 12 hours !
     for M in maf/chr*.maf.gz
     do
 	C=${M/.maf.gz}
 	C=${C#maf/}
 	echo "$M -> rp_scores/$C.score.gz"
 	(zcat ${M} | ./rp_kit/hm_rp_score.sh /dev/stdin /dev/stderr 2>&1 >/dev/null) | sort -n | \
 	    gzip > rp_scores/${C}.score.gz
     done
     #	real    709m55.805s
     #	user    754m51.030s
     #	sys     20m11.000s
 
     #	Back to the file server to create the wiggle data
 
     ssh eieio
     cd /cluster/data/hg17/bed/regPotential2X
     mkdir wigData dataLimits
     for S in rp_scores/chr*.score.gz
     do
 	C=${S/.score.gz}
 	C=${C#rp_scores/}
 	echo "$S -> wigData/$C.wig"
 	zcat $S | sort -n | \
 	    wigAsciiToBinary -chrom=$C -dataSpan=1 \
 	    -wibFile=wigData/$C stdin 2> dataLimits/$C.limits
     done
     #	real    313m0.567s
     #	user    285m37.319s
     #	sys     23m8.301s
 
     #	Loading the table on hgwdev
     ssh hgwdev
     cd /cluster/data/hg17/bed/regPotential2X/wigData
     mkdir /gbdb/hg17/wib/regPotential2X
     ln -s `pwd`/*.wib /gbdb/hg17/wib/regPotential2X
     time hgLoadWiggle -pathPrefix=/gbdb/hg17/wib/regPotential2X \
 	hg17 regPotential2X chr*.wig
     #	real    2m29.668s
     #	user    0m33.380s
     #	sys     0m8.200s
 
 # regulatory potential 3X track (WORKING - 2004-09-14 - Hiram)
     #	Expects groundwork done above in the 2X track
     #	a valid java runtime is only on hgwdev.  This is a java procedure
     ssh hgwdev
     cd /cluster/data/hg17/bed/regPotential3X
     ln -s ../regPotential2X/rp_kit/hmr_rp_score.sh .
     mkdir rp_scores
     #  WARNING - the following loop takes almost 12 hours !
     for M in maf/chr*.maf.gz
     do
 	C=${M/.maf.gz}
 	C=${C#maf/}
 	echo "$M -> rp_scores/$C.score.gz"
 	(zcat ${M} | ./rp_kit/hm_rp_score.sh /dev/stdin /dev/stderr 2>&1 >/dev/null) | sort -n | \
 	    gzip > rp_scores/${C}.score.gz
     done
     #	real    613m8.230s
     #	user    623m7.110s
     #	sys     20m24.550s
 
     #	Back to the file server to create the wiggle data
 
     ssh eieio
     cd /cluster/data/hg17/bed/regPotential3X
     mkdir wigData dataLimits
 
     for S in rp_scores/chr*.score.gz
     do
 	C=${S/.score.gz}
 	C=${C#rp_scores/}
 	echo "$S -> wigData/$C.wig"
 	zcat $S | sort -n | \
 	    wigAsciiToBinary -chrom=$C -dataSpan=1 \
 		-wibFile=wigData/$C stdin 2> dataLimits/$C.limits
     done
 
     #	Loading the table on hgwdev
     ssh hgwdev
     cd /cluster/data/hg17/bed/regPotential3X/wigData
     mkdir /gbdb/hg17/wib/regPotential3X
     ln -s `pwd`/*.wib /gbdb/hg17/wib/regPotential3X
     time hgLoadWiggle -pathPrefix=/gbdb/hg17/wib/regPotential3X \
 	hg17 regPotential3X chr*.wig
     #	real    1m45.568s
     #	user    0m32.740s
     #	sys     0m6.140s
 
 # regulatory potential 5X track (DONE - 2005-09-19 - Daryl)
     ssh kkstore02
     mkdir -p /cluster/data/hg17/bed/regPotential5X/rp_scores
     cd /cluster/data/hg17/bed/regPotential5X/rp_scores
     wget -r -l 1 -nH http://www.bx.psu.edu/~james/rp/hg17panTro1mm5rn3canFam1/all_truncate.tar
     tar xvf all_truncate.tar
     cd /cluster/data/hg17/bed/regPotential5X
     mkdir -p wigData dataLimits 
     cd wigData
     ## 8 minutes
     for S in ../rp_scores/chr*.scores.truncated.gz
     do
 	C=${S/.scores.truncated.gz}
 	C=${C#../rp_scores/}
 	echo "$S -> wigData/$C.wig"
 	zcat $S | wigEncode stdin $C.wig $C.wib 2> ../dataLimits/$C.limits
     done
 
     #	Loading the table on hgwdev
     ssh hgwdev
     cd /cluster/data/hg17/bed/regPotential5X/wigData
     mkdir -p /gbdb/hg17/wib/regPotential5X
     chmod o+rx /gbdb/hg17/wib/regPotential5X
     ln -s /cluster/data/hg17/bed/regPotential5X/wigData/*.wib /gbdb/hg17/wib/regPotential5X
     chmod o+r /gbdb/hg17/wib/regPotential5X/ch*wib
     time hgLoadWiggle -pathPrefix=/gbdb/hg17/wib/regPotential5X hg17 regPotential5X chr*.wig
     # 57.720u 9.960s 2:26.05 46.3%    0+0k 0+0io 213pf+0w
 
 # SGP GENES (DONE 9/17/04 angie)
     ssh eieio
     mkdir /cluster/data/hg17/bed/sgp
     cd /cluster/data/hg17/bed/sgp
     foreach chr (`awk '{print $1;}' ../../chrom.sizes`)
       wget http://genome.imim.es/genepredictions/H.sapiens/golden_path_200405/SGP/$chr.gtf
       wget http://genome.imim.es/genepredictions/H.sapiens/golden_path_200405/SGP/$chr.prot
     end
     # Add ".1" suffix to each item in .prot's, to match transcript_id's in gtf
     cp /dev/null sgpPep.fa
     foreach f (chr*.prot)
       perl -wpe 's/^(>chr\S+)/$1.1/' $f >> sgpPep.fa
     end
     ssh hgwdev
     cd /cluster/data/hg17/bed/sgp
     ldHgGene -gtf -genePredExt hg17 sgpGene chr*.gtf
     hgPepPred hg17 generic sgpPep sgpPep.fa
 
 # SGP GENES (UPDATE 1/18/2006)
     sgpPep table dropped, replaced by hgc generated protein seq in browser
 
 # LIFTOVER RNAGENE FROM HG16 (09/29/04, acs)
     cd /cluster/data/hg17/bed
     mkdir rnaGene
     cd rnaGene
     liftOver -gff /cluster/data/hg16/bed/rnaGene/all.gff \
 	/cluster/data/hg16/bed/liftOver/hg16ToHg17.over.chain \
 	rnaGeneLift.gff rnaGeneMiss.gff
     # 7204 records passed, 16 failed
     hgsql hg17 < ~/kent/src/hg/lib/rnaGene.sql
     hgRnaGenes hg17 rnaGeneLift.gff
 
     
 # BUILD BioCyc TABLES (DONE 10/1/04 Fan)
 
   - Create bioCycMapDesc table.
 
 	CREATE TABLE bioCycMapDesc (
   	mapID varchar(40) NOT NULL default '',
   	description varchar(255) NOT NULL default '',
   	KEY mapID (mapID)
 	) TYPE=MyISAM;
 
   - Crate bioCycPathway table.
 
 	CREATE TABLE bioCycPathway (
   	kgID varchar(40) NOT NULL default '',
   	geneID varchar(40) NOT NULL default '',
   	mapID varchar(40) NOT NULL default '',
   	KEY kgID (kgID),
  	KEY geneID (geneID),
   	KEY mapID (mapID)
 	) TYPE=MyISAM;
 
   Using data files sent by Peter Carp from SRI,
   per Peter's email of 10/1/04, they don't have recent update,
   so data files received last year are used.
   
   Save the BioCyc Pathway name and description table as names.txt.
   Save the pathway data file as gene-pathway.dat.
   Make sure there is no extra ^M at end of the lines.
 
   hgsql hg17 -e	'LOAD DATA local INFILE 'names.txt' into table bioCycMapDesc'
 
   Run hgBioCyc program to generate the file bioCycPathway.tab.
 
 	 hgBioCyc gene-pathway.dat hg17
 
   Load into hg17.
  
 	 hgsql hg17 -e 'LOAD DATA local INFILE "bioCycPathway.tab" into table bioCycPathway'
 	
 # MAKING FOLDUTR TABLES (DONE - 2004-10-4 Fan)
 # First set up directory structure and extract UTR sequence on hgwdev
     ssh hgwdev
     mkdir -p /cluster/data/hg17/bed/rnaStruct
     cd /cluster/data/hg17/bed/rnaStruct
     mkdir -p utr3/split utr5/split utr3/fold utr5/fold
     utrFa hg17 knownGene utr3 utr3/utr.fa
     utrFa hg17 knownGene utr5 utr5/utr.fa
 
 # Split up files and make files that define job.
     ssh kk
     cd /cluster/data/hg17/bed/rnaStruct
     faSplit sequence utr3/utr.fa 50000 utr3/split/s
     faSplit sequence utr5/utr.fa 50000 utr5/split/s
     ls -1 utr3/split > utr3/in.lst
     ls -1 utr5/split > utr5/in.lst
     cd utr3
     cat > gsub <<end
 #LOOP
 rnaFoldBig split/\$(path1) fold
 #ENDLOOP
 end
     cp gsub ../utr5
 
 # Do cluster run for 3' UTRs
     gensub2 in.lst single gsub spec
     para create spec
     para try
     para push
 # Completed: 37244 of 37244 jobs
 # CPU time in finished jobs:    1036479s   17274.64m   287.91h   12.00d  0.033 y
 # IO & Wait Time:                112286s    1871.44m    31.19h    1.30d  0.004 y
 # Average job time:                  31s       0.51m     0.01h    0.00d
 # Longest job:                     3370s      56.17m     0.94h    0.04d
 # Submission to last job:          4355s      72.58m     1.21h    0.05d
 
 # Do cluster run for 5' UTRs 
     cd ../utr5
     gensub2 in.lst single gsub spec
     para create spec
     para try
     para push
 # Completed: 29817 of 29817 jobs
 # CPU time in finished jobs:      98143s    1635.72m    27.26h    1.14d  0.003 y
 # IO & Wait Time:                105763s    1762.71m    29.38h    1.22d  0.003 y
 # Average job time:                   7s       0.11m     0.00h    0.00d
 # Longest job:                     2133s      35.55m     0.59h    0.02d
 # Submission to last job:          2465s      41.08m     0.68h    0.03d
 
 # Load database
     ssh hgwdev
     cd /cluster/data/hg17/bed/rnaStruct/utr5
     hgLoadRnaFold hg17 foldUtr5 fold
     cd ../utr3
     hgLoadRnaFold hg17 foldUtr3 fold
 
 # Clean up
     rm -r split fold err batch.bak
     cd ../utr5
     rm -r split fold err batch.bak
 
 ####### BUILD RGD HUMAN QTL TRACKS (DONE 10/7/04 Fan) ##############
 
 mkdir -p /cluster/store8/rgd/human041007
 ln -s /cluster/store8/rgd/human041007 /cluster/data/hg17/bed/rgdQtl
 cd /cluster/data/hg17/bed/rgdQtl
 
 # download data files from RGD
 
 wget --timestamp ftp://rgd.mcw.edu/pub/RGD_genome_annotations/human/human_QTL.gff
 
 # remove extra line feed character at the end of lines
 
 rmLf human_QTL.gff > rgdQtl.gff
 
 # create rgdQtl.tab
 awk '{print $1"\t"$4"\t"$5"\t"$10}'  rgdQtl.gff |sed -e 's/Chr/chr/g'| \
 sed -e 's/"//g' |sed -e 's/RGD://g' | sed -e 's/;//g' > rgdQtl.tab
 
 # create rgdQtlLink.tab
 
 awk '{printf "%s\t%s\t", $12, $10; for (i = 14;i <= NF; ++i ) {printf "%s ", $i} printf "\n"} ' rgdQtl.gff | \
 sed -e 's/"//g'| sed -e 's/RGD://g' | sed -e 's/;//g'| sed -e 's/Note//g' > rgdQtlLink.tab
 
 # load rgdQtl table
 hgLoadBed hg17 rgdQtl rgdQtl.tab
 
 # check rgdQtl table
 checkTableCoords hg17 rgdQtl
 
 # load rgdQtlLink table
 hgsql hg17 -e "drop table hg17.rgdQtlLink;"
 hgsql hg17 <~/kent/src/hg/lib/rgdQtlLink.sql
 hgsql hg17 -e 'load data local infile "rgdQtlLink.tab" into table hg17.rgdQtlLink;'
 
 # updated trackDb.ra under /kent/src/hg/makeDb/trackDb/human/hg17 and
 # added rgdQtl.html.
 
 #### AFFYMETRIX HG-U133 PLUS TRACK (DONE, 2004-10-11, hartera)
      ssh hgwdev
      mkdir -p /projects/compbio/data/microarray/affyHuman/HG-U133Plus2
      # Go to 
 #http://www.affymetrix.com/support/technical/byproduct.affx?product=hg-u133-plus 
      # and download the consensus and exemplar sequences to this directory
      cd /projects/compbio/data/microarray/affyHuman/HG-U133Plus2
      unzip HG-U133_Plus_2_consensus.zip
      unzip HG-U133_Plus_2_exemplar.zip
      cat HG-U133_Plus_2_consensus HG-U133_Plus_2_exemplar >> U133Plus2_all.fa
      perl -pi.bak -e "s/(consensus|exemplar):HG-U133_Plus_2:/U133+2:/" \
                      U133Plus2_all.fa
      # remove ";" from probe set names
      perl -pi.bak -e "s/;//" U133Plus2_all.fa
      # clean up
      rm *.zip *.bak
      mkdir -p /cluster/data/hg17/bed/affyU133Plus2.2004-10-11
      cp U133Plus2_all.fa /cluster/data/hg17/bed/affyU133Plus2.2004-10-11
      # Set up cluster job to align consensus/exemplars to hg16
      ssh kkr1u00
      mkdir -p /iscratch/i/affy
      mv /cluster/data/hg17/bed/affyU133Plus2.2004-10-11/U133Plus2_all.fa \
         /iscratch/i/affy
      iSync
 
      ssh kk
      cd /cluster/data/hg17/bed/affyU133Plus2.2004-10-11
      ls -1 /iscratch/i/affy/U133Plus2_all.fa > affy.lst
      ls -1 /iscratch/i/gs.18/build35/maskedContigs/* > allctg.lst
 
     cat << '_EOF_' > template.sub
 #LOOP
 /cluster/bin/i386/blat -fine -mask=lower -minIdentity=95 -ooc=/iscratch/i/gs.18/build35/hg17.11.ooc $(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
 #ENDLOOP
 '_EOF_'
     # << for emacs
     gensub2 allctg.lst affy.lst template.sub para.spec
     mkdir psl
     para create para.spec
     para try, para check, para push .....
 # para time
 # Completed: 380 of 380 jobs
 # CPU time in finished jobs:      24533s     408.88m     6.81h    0.28d  0.001 y
 # IO & Wait Time:                  2180s      36.34m     0.61h    0.03d  0.000 y
 # Average job time:                  70s       1.17m     0.02h    0.00d
 # Longest job:                      751s      12.52m     0.21h    0.01d
 # Submission to last job:          2425s      40.42m     0.67h    0.03d
 
     # Do sort, best in genome filter, and convert to chromosome coordinates
     # to create affyU133Plus2.psl
     pslSort dirs raw.psl tmp psl
                                                                                 
     # use filter parameters for these sequences. only use alignments that
     # cover 30% of sequence and have at least 95% identity in aligned region.
     # minAli = 0.97 too high. low minCover as a lot of n's in these sequences
   pslReps -minCover=0.3 -minAli=0.95 -nearTop=0.005 raw.psl contig.psl /dev/null
     liftUp affyU133Plus2.psl ../../jkStuff/liftAll.lft warn contig.psl
     perl -pi.bak -e "s/U133\+2://" affyU133Plus2.psl
     # load into the database
     ssh hgwdev
     cd /cluster/data/hg17/bed/affyU133Plus2.2004-10-11
     hgLoadPsl hg17 affyU133Plus2.psl
     # Add sequence data to database
         # Copy probe sequence to /gbdb if it isn't already
     mkdir -p /gbdb/hgFixed/affyProbes
     cd /gbdb/hgFixed/affyProbes
     ln -s /projects/compbio/data/microarray/affyHuman/HG-U133Plus2/U133Plus2_all.fa .
     cd /cluster/data/hg17/bed/affyU133Plus2.2004-10-11
     hgLoadSeq -abbr=U133+2: hg17 /gbdb/hgFixed/affyProbes/U133Plus2_all.fa
     
     # clean up
     rm -r psl tmp err contig.psl raw.psl *.bak psl.tab seq.tab
 # Added knownToU133Plus2 track (2004-10-14) - see GeneSorter section
 
 
 #### MAF COVERAGE FIGURES FOR ADAM (DONE 10/18/04 angie)
     # First, get ranges of target coverage:
     ssh eieio
     mkdir /cluster/data/hg17/bed/var_multiz.2004-08-12/coverage
     cd /cluster/data/hg17/bed/var_multiz.2004-08-12/coverage/
     cat /cluster/data/hg17/bed/var_multiz.2004-08-12/maf.09-12-04/*.maf \
     | nice mafRanges -notAllOGap stdin hg17 hg17.mafRanges.bed
     # Get pairwise coverage as well.
     ssh kolossus
     cd /cluster/data/hg17/bed/var_multiz.2004-08-12/coverage/
     cat /cluster/bluearc/hg17/multiz8way/rn3/*.maf \
     | nice mafRanges -notAllOGap stdin hg17 hg17.rn3.mafRanges.bed
     cat /cluster/bluearc/hg17/multiz8way/mm5/*.maf \
     | nice mafRanges -notAllOGap stdin hg17 hg17.mm5.mafRanges.bed
     cat /cluster/bluearc/hg17/multiz8way/galGal2/*.maf \
     | nice mafRanges -notAllOGap stdin hg17 hg17.galGal2.mafRanges.bed
     cat /cluster/bluearc/hg17/multiz8way/fr1/*.maf \
     | nice mafRanges -notAllOGap stdin hg17 hg17.fr1.mafRanges.bed
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/var_multiz.2004-08-12/coverage
     # To make subsequent intersections a bit quicker, output a bed with 
     # duplicate/overlapping ranges collapsed:
     nice featureBits hg17 hg17.mafRanges.bed \
       -bed=hg17.mafRangesCollapsed.bed
 #1147548420 bases of 2866216770 (40.037%) in intersection
     foreach other (mm5 rn3 galGal2 fr1)
       nice featureBits hg17 hg17.$other.mafRanges.bed \
       -bed=hg17.${other}.mafRangesCollapsed.bed
     end
 #1013348528 bases of 2866216770 (35.355%) in intersection
 #975533772 bases of 2866216770 (34.036%) in intersection
 #101623034 bases of 2866216770 (3.546%) in intersection
 #46737824 bases of 2866216770 (1.631%) in intersection
 
     # mafCoverage barfs currently, so pass on this for now:
     #cat ../maf.09-12-04/*.maf \
     #| nice mafCoverage -count=2 hg17 stdin > hg17.mafCoverage
 
     # Intersect maf target coverage with gene regions -- 
     # use Adam's knownGene region files:
     nice featureBits hg17 -enrichment \
       ../phastCons/stats2/knownGenesCds.bed \
       hg17.mafRangesCollapsed.bed \
       -bed=hg17.mafCds.bed
 #knownGenesCds.bed 1.166%, hg17.mafRangesCollapsed.bed 40.037%, both 1.111%, cover 95.36%, enrich 2.38x
 
     nice featureBits hg17 -enrichment \
       ../phastCons/stats2/knownGenesUtr3.bed \
       hg17.mafRangesCollapsed.bed \
       -bed=hg17.mafUtr3.bed
 #knownGenesUtr3.bed 0.918%, hg17.mafRangesCollapsed.bed 40.037%, both 0.662%, cover 72.18%, enrich 1.80x
     nice featureBits hg17 -enrichment \
       ../phastCons/stats2/knownGenesUtr5.bed \
       hg17.mafRangesCollapsed.bed \
       -bed=hg17.mafUtr5.bed
 #knownGenesUtr5.bed 0.266%, hg17.mafRangesCollapsed.bed 40.037%, both 0.198%, cover 74.42%, enrich 1.86x
 
     # Intersect pairwise target coverages with gene regions:
     foreach other (mm5 rn3 galGal2 fr1)
       nice featureBits hg17 -enrichment \
         ../phastCons/stats2/knownGenesCds.bed \
         hg17.$other.mafRangesCollapsed.bed -bed=hg17.${other}Cds.bed
       nice featureBits hg17 -enrichment \
         ../phastCons/stats2/knownGenesUtr3.bed \
         hg17.$other.mafRangesCollapsed.bed -bed=hg17.${other}Utr3.bed
       nice featureBits hg17 -enrichment \
         ../phastCons/stats2/knownGenesUtr5.bed \
         hg17.$other.mafRangesCollapsed.bed -bed=hg17.${other}Utr5.bed
     end
 #knownGenesCds.bed 1.166%, hg17.mm5.mafRangesCollapsed.bed 35.355%, both 1.093%, cover 93.74%, enrich 2.65x
 #knownGenesUtr3.bed 0.918%, hg17.mm5.mafRangesCollapsed.bed 35.355%, both 0.618%, cover 67.37%, enrich 1.91x
 #knownGenesUtr5.bed 0.266%, hg17.mm5.mafRangesCollapsed.bed 35.355%, both 0.186%, cover 69.81%, enrich 1.97x
 #knownGenesCds.bed 1.166%, hg17.rn3.mafRangesCollapsed.bed 34.036%, both 1.071%, cover 91.85%, enrich 2.70x
 #knownGenesUtr3.bed 0.918%, hg17.rn3.mafRangesCollapsed.bed 34.036%, both 0.597%, cover 65.09%, enrich 1.91x
 #knownGenesUtr5.bed 0.266%, hg17.rn3.mafRangesCollapsed.bed 34.036%, both 0.179%, cover 67.33%, enrich 1.98x
 #knownGenesCds.bed 1.166%, hg17.galGal2.mafRangesCollapsed.bed 3.546%, both 0.779%, cover 66.84%, enrich 18.85x
 #knownGenesUtr3.bed 0.918%, hg17.galGal2.mafRangesCollapsed.bed 3.546%, both 0.194%, cover 21.12%, enrich 5.96x
 #knownGenesUtr5.bed 0.266%, hg17.galGal2.mafRangesCollapsed.bed 3.546%, both 0.056%, cover 21.03%, enrich 5.93x
 #knownGenesCds.bed 1.166%, hg17.fr1.mafRangesCollapsed.bed 1.631%, both 0.714%, cover 61.26%, enrich 37.57x
 #knownGenesUtr3.bed 0.918%, hg17.fr1.mafRangesCollapsed.bed 1.631%, both 0.073%, cover 7.92%, enrich 4.86x
 #knownGenesUtr5.bed 0.266%, hg17.fr1.mafRangesCollapsed.bed 1.631%, both 0.039%, cover 14.82%, enrich 9.09x
 
 
 # ALTERNATIVE CPG ISLANDS (DONE 10/14/04 angie)
     ssh eieio
     nice tcsh
     mkdir /cluster/data/hg17/bed/cpgIslandAlt
     cd /cluster/data/hg17/bed/cpgIslandAlt
     # Try cpg_ash (WUSTL program modified to not chop islands in half before 
     # scoring) with default params:
     cp /dev/null cpg_ash.default.cpg
     foreach f (../../?{,?}/chr*.fa.masked)
       echo running on $f:t:r:r
       ~angie/cb/hg3rdParty/cpgIslands/cpg_ash.exe $f >> cpg_ash.default.cpg
     end
     awk -f ../cpgIsland/filter.awk cpg_ash.default.cpg > cpgIslandAlt.bed
     # Run Andy Law's script on masked seq:
     cp /dev/null cpgIslandGgfAndyMasked.bed
     foreach f (../../?{,?}/chr*.fa.masked)
       set chr = $f:t:r:r
       echo running on $chr
       /cluster/home/angie/bin/$MACHTYPE/preProcGgfAndy $f \
       | /cluster/home/angie/ggf-andy-cpg-island.pl \
       | perl -wpe 'chomp; ($s,$e,$cpg,$n,$c,$g,$oE) = split("\t"); $s--; \
                    $gc = $c + $g;  $pCpG = (100.0 * 2 * $cpg / $n); \
                    $pGc = (100.0 * $gc / $n); \
                    $_ = "'$chr'\t$s\t$e\tCpG: $cpg\t$n\t$cpg\t$gc\t" . \
                         "$pCpG\t$pGc\t$oE\n";' \
       >> cpgIslandGgfAndyMasked.bed
     end
     # Compare enrichment for knownGene upstream -- an uphill battle for 
     # programs closer to meeting the stated length, GC, O/E params!
     ssh hgwdev
     nice featureBits hg17 -enrichment knownGene:upstream:1000 \
       /cluster/data/hg17/bed/cpgIsland/cpgIsland.bed
 #knownGene:upstream:1000 0.857%, cpgIsland.bed 0.741%, both 0.166%, cover 19.37%, enrich 26.13x
     nice featureBits hg17 -enrichment knownGene:upstream:1000 \
       /cluster/data/hg17/bed/cpgIslandAlt/cpgIslandAlt.bed
 #knownGene:upstream:1000 0.857%, cpgIslandAlt.bed 1.075%, both 0.200%, cover 23.38%, enrich 21.76x
     nice featureBits hg17 -enrichment knownGene:upstream:1000 \
       /cluster/data/hg17/bed/cpgIslandAlt/cpgIslandGgfAndyMasked.bed
 #knownGene:upstream:1000 0.857%, cpgIslandGgfAndyMasked.bed 1.964%, both 0.292%, cover 34.06%, enrich 17.34x
     cd /cluster/data/hg17/bed/cpgIslandAlt
     sed -e 's/cpgIslandExt/cpgIslandAlt/g' \
       ~/kent/src/hg/lib/cpgIslandExt.sql > cpgIslandAlt.sql
     hgLoadBed -noBin -tab -sqlTable=cpgIslandAlt.sql \
       hg17 cpgIslandAlt cpgIslandAlt.bed
 #Loaded 29998 elements of size 10
     sed -e 's/cpgIslandExt/cpgIslandGgfAndyMasked/g' \
       ~/kent/src/hg/lib/cpgIslandExt.sql > cpgIslandGgfAndyMasked.sql
     hgLoadBed -noBin -tab -sqlTable=cpgIslandGgfAndyMasked.sql \
       hg17 cpgIslandGgfAndyMasked cpgIslandGgfAndyMasked.bed
 #Loaded 80555 elements of size 10
     # Quick length stats:
     hgsql hg17 -e 'select min(length), avg(length), max(length) from cpgIslandExt'
 #|         201 |    764.1913 |       40058 |
     hgsql hg17 -e 'select min(length), avg(length), max(length) from cpgIslandAlt'
 #|         200 |   1026.9194 |       32440 |
     hgsql hg17 -e 'select min(length), avg(length), max(length) from cpgIslandGgfAndyMasked'
 #|         200 |    698.8257 |      100308 |
     # 1/26/05: Make better island names in cpgIslandGgfAndyMasked,
     # for Dave Burt's cross-species island comparisons.
     ssh eieio
     cd /cluster/data/hg17/bed/cpgIslandAlt
     mv cpgIslandGgfAndyMasked.bed cpgIslandGgfAndyMasked.bed.orig
     perl -wpe '@w=split("\t"); $w[3] = "hg17.$w[0]." . ($w[1]+1) . ".$w[2]"; \
                $_ = join("\t", @w);' \
       cpgIslandGgfAndyMasked.bed.orig \
     > cpgIslandGgfAndyMasked.bed
     # Now liftOver islands from mm5, rn3, galGal2:
     ssh kolossus
     cd /cluster/data/hg17/bed/cpgIslandAlt
     foreach match (50 95)
       liftOver /cluster/data/mm5/bed/cpgIslandGgfAndy/cpgIslandGgfAndyMasked.bed \
         /cluster/data/mm5/bed/bedOver/mm5Tohg17.chain -minMatch=0.$match \
         cpgIslandGAMFromMm5_$match.bed cpgIslandGAMFromMm5_$match.unmapped
       liftOver /cluster/data/rn3/bed/cpgIslandGgfAndy/cpgIslandGgfAndyMasked.bed \
         /cluster/data/rn3/bed/bedOver/rn3ToHg17.over.chain -minMatch=0.$match \
         cpgIslandGAMFromRn3_$match.bed cpgIslandGAMFromRn3_$match.unmapped
       liftOver /cluster/data/galGal2/bed/cpgIslandGgfAndy/cpgIslandGgfAndyMasked.bed \
         /cluster/data/galGal2/bed/bedOver/galGal2ToHg17.over.chain -minMatch=0.$match \
         cpgIslandGAMFromGalGal2_$match.bed cpgIslandGAMFromGalGal2_$match.unmapped
     end
     # Load up the renamed islands as well as
     ssh hgwdev
     cd /cluster/data/hg17/bed/cpgIslandAlt
     hgLoadBed -noBin -tab -sqlTable=cpgIslandGgfAndyMasked.sql \
       hg17 cpgIslandGgfAndyMasked cpgIslandGgfAndyMasked.bed
 
 
 # MAKE UNIGENE/SAGE TRACK (DONE - 2004-10-15 Fan)
 
 # First get SAGE data and determine which version of UniGene to use first
     ssh hgwdev
     cd ~/kent/src/hg/sage
     make
     # XXX = uniGene build for which SAGE was built -- not necessarily current!
     # Figure out the build number by peeking at this file:
     wget -O - ftp://ftp.ncbi.nih.gov/pub/sage/map/info.txt 2> /dev/null
 
 # UniGene Build #44  Arabidopsis thaliana
 # UniGene Build #61  Bos taurus
 # UniGene Build #16  Caenorhabditis elegans
 # UniGene Build #171  Homo sapiens
 # UniGene Build #19  Medicago truncatula
 # UniGene Build #138  Mus musculus
 # UniGene Build #52  Oryza sativa
 # UniGene Build #14  Pinus taeda
 # UniGene Build #132  Rattus norvegicus
 # UniGene Build #27  Sus scrofa
 # UniGene Build #38  Triticum aestivum
 # UniGene Build #11  Vitis vinifera
 # UniGene Build #41  Zea mays
 
 # From above info, set Version 171 for hg17
     ls /projects/cc/hg/sugnet/uniGene
 #   set Version = XXX
     set Version=171
     mkdir /projects/cc/hg/sugnet/sage/sage.$Version
     cd /projects/cc/hg/sugnet/sage/sage.$Version
 
     wget -r -nH --cut-dirs=2 --timestamp ftp://ftp.ncbi.nih.gov/pub/sage/map/Hs
     wget -r -nH --cut-dirs=2 --timestamp ftp://ftp.ncbi.nih.gov/pub/sage/map/readme.txt
     wget -r -nH --cut-dirs=2 --timestamp ftp://ftp.ncbi.nih.gov/pub/sage/map/info.txt
     wget -r -nH --cut-dirs=2 --timestamp ftp://ftp.ncbi.nih.gov/pub/sage/extr
     wget -r -nH --cut-dirs=2 --timestamp ftp://ftp.ncbi.nih.gov/pub/sage/info
 
 #  That downloaded about 1 GB of data
 
     cd map/Hs/NlaIII
     unzip -j SAGEmap_tag_ug-rel.zip
     cd ../../../extr/
     ../../scripts/summarizeCounts.pl expCounts.tab ./SAGE_*
     ../../scripts/countGenesPerTag.pl expCounts.tab allTags.count.tab
     ../../scripts/createArraysForTags.pl allTags.count.tab tagExpArrays.tab \
       ./SAGE_*
     ../../scripts/countsPerExp.pl expCounts.tab expList.tab
 
     cd ../map/Hs/NlaIII/ 
     cat << '_EOF_' > /tmp/t.pl
 #!/usr/local/bin/perl
 
 while (<>) {
  chomp($_);
  @p = split(/\t/, $_);
  print "$p[2]\t$p[3]\t$p[0]\n";
 }
 '_EOF_'
     chmod +x /tmp/t.pl
 
     cat SAGEmap_tag_ug-rel | /tmp/t.pl | sort | sed -e 's/ /_/g' \
       > SAGEmap_ug_tag-rel_Hs
     cd ../../../extr
     createSageSummary ../map/Hs/NlaIII/SAGEmap_ug_tag-rel_Hs \
       tagExpArrays.tab sageSummary.sage
 
 # Create the uniGene alignments 
 # /cluster/data/hg17/uniGene/hg17.uniGene.lifted.pslReps.psl
 
     # Download of the latest UniGene version is now automated by a 
     # cron job -- see /cluster/home/angie/crontab , 
     # /cluster/home/angie/unigeneVers/unigene.csh .  
     # If hgwdev gets rebooted, that needs to be restarted... maybe there's 
     # a more stable place to set up that cron job.  
 
     # substitute XXX -> the uniGene version used by SAGE.
     # set Version = XXX
     set Version = 171			(bash: export Version=171)
     cd /projects/cc/hg/sugnet/uniGene/uniGene.$Version
     gunzip Hs.seq.uniq.gz Hs.data.gz
     ../countSeqsInCluster.pl Hs.data counts.tab
     ../parseUnigene.pl Hs.seq.uniq Hs.seq.uniq.simpleHeader.fa leftoverData.tab
     # Distribute UniGene sequence to /iscratch/i/ (kkstore can see /projects)
     ssh kkstore
     set Version = 171 # same as above
     mkdir -p /iscratch/i/uniGene.$Version
     cp -p \
   /projects/cc/hg/sugnet/uniGene/uniGene.$Version/Hs.seq.uniq.simpleHeader.fa \
       /iscratch/i/uniGene.$Version
     ssh kkr1u00
     iSync
     ssh kk
     set Version = 171 # same as above
     mkdir -p /cluster/data/hg17/bed/uniGene.$Version
     cd /cluster/data/hg17/bed/uniGene.$Version
     ls -1S /scratch/hg/gs.18/build35/maskedContigs/*.fa > allctg.lst
     ls -1S /iscratch/i/uniGene.$Version/Hs.seq.uniq.simpleHeader.fa \
       > uniGene.lst
     cat << '_EOF_' > template.sub
 #LOOP
 /cluster/bin/i386/blat -mask=lower -minIdentity=95 -ooc=/scratch/hg/h/11.ooc $(path1) $(path2)  {check out 
 line+ psl/$(root1)_$(root2).psl}
 #ENDLOOP
 '_EOF_'
 
     gensub2 allctg.lst uniGene.lst template.sub para.spec
     para create para.spec
     mkdir psl
     para try
     para check
     para push
 # Completed: 380 of 380 jobs
 # CPU time in finished jobs:      35994s     599.91m    10.00h    0.42d  0.001 y
 # IO & Wait Time:                  1812s      30.19m     0.50h    0.02d  0.000 y
 # Average job time:                  99s       1.66m     0.03h    0.00d
 # Longest job:                     1497s      24.95m     0.42h    0.02d
 # Submission to last job:          1551s      25.85m     0.43h    0.02d
 
     ssh eieio
     set Version = 171 # same as above
     cd /cluster/data/hg17/bed/uniGene.$Version
     pslSort dirs raw.psl tmp psl >& pslSort.log
     liftUp -type=.psl stdout ../../jkStuff/liftAll.lft warn raw.psl \
     | pslReps -minCover=0.2 -sizeMatters -minAli=0.965 -nearTop=0.002 \
       stdin hg17.uniGene.lifted.pslReps.psl /dev/null
 # Processed 141416 alignments
 
 # use hg17.uniGene.lifted.pslReps.psl for building UNIGENE/SAGE track.
 
     ssh hgwdev
     set Version = 171
     cd /projects/cc/hg/sugnet/sage/sage.$Version/extr
     addAveMedScoreToPsls \
       /cluster/data/hg17/bed/uniGene.$Version/hg17.uniGene.lifted.pslReps.psl \
       sageSummary.sage  uniGene.wscores.bed
     hgLoadBed hg17 uniGene_2 uniGene.wscores.bed
     hgsql hg17 < ~kent/src/hg/lib/sage.sql 
     echo "load data local infile 'sageSummary.sage' into table sage" \
         | hgsql hg17
     cd ../info
     ../../scripts/parseRecords.pl ../extr/expList.tab  > sageExp.tab
     hgsql hg17 < ~/kent/src/hg/lib/sageExp.sql 
     echo "load data local infile 'sageExp.tab' into table sageExp" | hgsql hg17
     # update ~/kent/src/hg/makeDb/trackDb/human/hg17/uniGene_2.html 
     # with current uniGene date.
 
 
 # CREATE kgSpAlias TABLE FOR PB (Done 10/20/04)
 
     hgsql hg17 -e \
     'select kgXref.kgID, spID, alias from kgXref, kgAlias where kgXref.kgID=kgAlias.kgID' >j.tmp
     hgsql hg17 -e \
     'select kgXref.kgID, spID, alias from kgXref, kgProtAlias where kgXref.kgID=kgProtAlias.kgID'\
     >>j.tmp
     cat j.tmp|sort -u |grep -v 'kgID' >hg17.kgSpAlias.tab
     rm j.tmp
 
     hgsql hg17 -e 'drop table kgSpAlias';
     hgsql hg17 < ~/src/hg/lib/kgSpAlias.sql
     hgsql hg17 -e 'load data local infile "hg17.kgSpAlias.tab" into table kgSpAlias'
 
 
 # SEGMENTAL DUPLICATIONS (DONE 10/21/04 angie)
     ssh hgwdev
     mkdir /cluster/data/hg17/bed/genomicSuperDups
     cd /cluster/data/hg17/bed/genomicSuperDups
     # A tar file containing files for both hg16 and hg17 was downloaded into 
     # /cluster/data/hg16/bed/genomicSuperDups;  move over the hg17 part.
     mv /cluster/data/hg16/bed/genomicSuperDups/bd35 .
     cd bd35
     # A note from Xinwei She about the contents:
 #Build35 contains only 2 tables: genomicSuperDups and celeraDupPositive.
     # use tail +2 to skip past the header line:
     # actually, celeraDupPositive.tab.gz has one extra bogus line so +3 for it:
     zcat celeraDupPositive.tab.gz | tail +3 \
     | hgLoadBed -tab -sqlTable=$HOME/kent/src/hg/lib/celeraDupPositive.sql \
         hg17 celeraDupPositive stdin
     zcat genomicSuperDups.tab.gz | tail +2 \
     | hgLoadBed -tab -sqlTable=$HOME/kent/src/hg/lib/genomicSuperDups.sql \
         hg17 genomicSuperDups stdin
     # clean up
     rm bed.tab
 
 # ECGENE TRACK (DONE, 2004-10-29, hartera)
     ssh eieio
     mkdir -p /cluster/data/hg17/bed/ECgene.2004-10-27
     ln -s /cluster/data/hg17/bed/ECgene.2004-10-27 \
           /cluster/data/hg17/bed/ECgene
     cd  /cluster/data/hg17/bed/ECgene
     wget \
 "http://genome.ewha.ac.kr/ECgene/download/v1.2_ECgene/v1.2_hg17_low_gene.txt.gz"
     wget \
 "http://genome.ewha.ac.kr/ECgene/download/v1.2_ECgene/v1.2_hg17_low_pep.txt.gz"
     gunzip *.gz
 
     # load database
     ssh hgwdev
     cd /cluster/data/hg17/bed/ECgene
     ldHgGene -predTab hg17 ECgene v1.2_hg17_low_gene.txt
     # 646778 gene predictions
     hgPepPred hg17 tab ECgenePep v1.2_hg17_low_pep.txt
     rm *.tab
     nice gzip *.txt
 
 # LOAD ENSEMBL GENES (DONE, 2004-11-19, hartera)
 
     mkdir /cluster/data/hg17/bed/ensembl
     cd /cluster/data/hg17/bed/ensembl
     # Get the ensembl protein data from 
     # http://www.ensembl.org/Homo_sapiens/martview
     # Follow this sequence through the pages:
     # Page 1) Make sure that the Homo_sapiens choice is selected. Hit next.
     # Page 2) Uncheck the "Limit to" box in the region choice. Then hit next.
     # Page 3) Choose the "Structures" box. 
     # Page 4) Choose GTF as the ouput.  choose gzip compression.  hit export.
     # Save as ensemblGene.gtf.gz
 
     # Ensembl handles random chromosomes differently than us, so we
     # strip this data.  Fortunately it just loses a couple of genes.
     # Add "chr" to front of each line in the gene data gtf file to make 
     # it compatible with our software.
     # Finally, get rid of the ".1" or ".2" after the name
     gunzip -c ensemblGene.gtf.gz \
     | grep -v ^6_DR51 \
     | grep -v ^DR51 \
     | grep -v ^DR52 \
     | grep -v ^DR53 \
     | grep -v _NT_ \
     | perl -wpe 's/^([0-9]|X|Y|Un|MT)/chr$1/ \
                  || die "Line $. doesnt start with human chrom:\n$_"' \
     | sed -e 's/\..\"/\"/g' \
     | sed -e 's/chrMT_NC_001807/chrM/' \
     > ensGene.gtf
 
     ssh hgwdev
     /cluster/data/hg17/bed/ensembl
     /cluster/bin/i386/ldHgGene hg17 ensGene ensGene.gtf
 
     # Read 33666 transcripts in 696579 lines in 1 files
     # 33666 groups 25 seqs 1 sources 4 feature types
     # 33666 gene predictions
 
     # ensGtp associates geneId/transcriptId/proteinId for hgPepPred and 
     # hgKnownToSuper.  Use ensMart to create it as above, except:
     # Page 3) Choose the "Features" box. In "Ensembl Attributes", check 
     # Ensembl Gene ID, Ensembl Transcript ID, Ensembl Peptide ID.  
     # Choose Text, tab-separated as the output format.  Result name ensGtp.
     # Save file as ensGtp.txt.gz
     gunzip ensGtp.txt.gz
     hgsql hg17 < ~/kent/src/hg/lib/ensGtp.sql
     # remove header line from ensGtp.txt
     echo "load data local infile 'ensGtp.txt' into table ensGtp" | hgsql -N hg17
 
     # Load Ensembl peptides:
     # Get them from ensembl as above in the gene section except for
     # Page 3) Choose the "Sequences" box. 
     # Page 4) Transcripts/Proteins.  Peptide.  Format = FASTA.
     # Save file as ensemblPep.fa.gz
     gunzip ensemblPep.fa.gz
     hgPepPred hg17 ensembl ensemblPep.fa
 
 # UPDATE KNOWN GENES TABLES (DONE 11/22/04 Fan)
 
 # Make sure the protein databases (sp041115 and proteins041115) were built first.
   
   hgsql hg17 -e "create database kgHg17B"
   
   mkdir -p /cluster/store8/kg/kgHg17B
   cd /cluster/store6/kgDB/bed
   ln -s /cluster/store8/kg/kgHg17B kgHg17B
   cd kgHg17B
 
     ~/src/hg/protein/KGprocess.sh kgHg17B hg17 041115
 
 # Found gbGetSeqs changed the format of mrna.fa output file 
 # (extra version number).  Updated Kgprocess.sh and manually
 # re-ran the following manually:
 
     grep "^>" mrna.fa |awk '{print $1}' > mrna.lis
 
     kgGetPep 041115 > mrnaPep.fa
 
     hgKgMrna kgH17BTemp mrna.fa mrna.ra tight_mrna.psl ll/loc2ref \
         mrnaPep.fa ll/mim2loc ${PDB} > kgHg17BKgMrna.out 2> kgHg17BKgMrna.err
 
 # then run Kgprocess.sh again to continue processing.
 
     ~/src/hg/protein/KGprocess.sh kgHg17B hg17 041115
 
 hgsql hg17 -e "select * from chromInfo" > chromInfo.tab
 getDbTabDef hg17 chromInfo >chromInfo.sql
 hgsql kgHg17B <chromInfo.sql
 hgsql kgHg17B -e 'load data local infile "chromInfo.tab" into table chromInfo ignore 1 lines'
 
 # Build kgProtMap table.  This table is needed by the Proteome Browser and 
 # it should be built before all the KG tables be moved from kgHg17B to hg17.
 
     ~/src/hg/protein/kgProtMap.sh kgHg17B hg17 041115
 
 # Completed: 7923 of 7923 jobs
 # CPU time in finished jobs:    2502923s   41715.39m   695.26h   28.97d  0.079 y
 # IO & Wait Time:                175358s    2922.63m    48.71h    2.03d  0.006 y
 # Average job time:                 338s       5.63m     0.09h    0.00d
 # Longest job:                     2403s      40.05m     0.67h    0.03d
 # Submission to last job:          7164s     119.40m     1.99h    0.08d
   
 # The script ran successfully with the last message:
 # Mon Nov 22 17:11:59 PST 2004 DONE =========================
 
 # Create database hg17Kg1 to store the old KG tables, just in case.
 
   hgsql hg17
 
   create database hg17Kg1;
 
   alter table cgapAlias rename as hg17Kg1.cgapAlias;
   alter table cgapBiocDesc rename as hg17Kg1.cgapBiocDesc;
   alter table cgapBiocPathway rename as hg17Kg1.cgapBiocPathway;
   alter table dupSpMrna rename as hg17Kg1.dupSpMrna;
   alter table keggMapDesc rename as hg17Kg1.keggMapDesc;
   alter table keggPathway rename as hg17Kg1.keggPathway;
   alter table kgAlias rename as hg17Kg1.kgAlias;
   alter table kgProtAlias rename as hg17Kg1.kgProtAlias;
   alter table kgXref rename as hg17Kg1.kgXref;
   alter table knownGene rename as hg17Kg1.knownGene;
   alter table knownGeneLink rename as hg17Kg1.knownGeneLink;
   alter table knownGeneMrna rename as hg17Kg1.knownGeneMrna;
   alter table knownGenePep rename as hg17Kg1.knownGenePep;
   alter table mrnaRefseq rename as hg17Kg1.mrnaRefseq;
   alter table spMrna rename as hg17Kg1.spMrna;
   alter table kgProtMap rename as hg17Kg1.kgProtMap;
 
 # After initial inspection of tables in kgHg17B, do the following
 # from mySql prompt:
 
   alter table kgHg17B.cgapAlias rename as hg17.cgapAlias;
   alter table kgHg17B.cgapBiocDesc rename as hg17.cgapBiocDesc;
   alter table kgHg17B.cgapBiocPathway rename as hg17.cgapBiocPathway;
   alter table kgHg17B.dupSpMrna rename as hg17.dupSpMrna;
   alter table kgHg17B.keggMapDesc rename as hg17.keggMapDesc;
   alter table kgHg17B.keggPathway rename as hg17.keggPathway;
   alter table kgHg17B.kgAlias rename as hg17.kgAlias;
   alter table kgHg17B.kgProtAlias rename as hg17.kgProtAlias;
   alter table kgHg17B.kgXref rename as hg17.kgXref;
   alter table kgHg17B.knownGene rename as hg17.knownGene;
   alter table kgHg17B.knownGeneLink rename as hg17.knownGeneLink;
   alter table kgHg17B.knownGeneMrna rename as hg17.knownGeneMrna;
   alter table kgHg17B.knownGenePep rename as hg17.knownGenePep;
   alter table kgHg17B.mrnaRefseq rename as hg17.mrnaRefseq;
   alter table kgHg17B.spMrna rename as hg17.spMrna;
   alter table kgHg17B.kgProtMap rename as hg17.kgProtMap;
 
 # Old hg17.knownGene has 43,401 entries and the new one has 44,338 entries.
 # Previously:
   
     featureBits hg17 knownGene
 #      65728598 bases of 2866216770 (2.293%) in intersection
 # Previously, was:
 #    	 63983072 bases of 2866216770 (2.232%) in intersection
   
 # Connect to genome-testdb and use hgcentraltest DB.
 # Update the entry in gdbPdb table:
 
     delete from gdbPdb where genomeDb='hg17';
     insert into gdbPdb values('hg17', 'proteins041115');
 
 # UPDATE KGSPALIAS TABLE TO BE USED BY PB (Done 12/20/04)
 
     cd /cluster/data/hg17/bed/pb
     mkdir new
     cd new
     hgsql hg17 -e \
     'select kgXref.kgID, spID, alias from kgXref, kgAlias where kgXref.kgID=kgAlias.kgID' >j.tmp
     hgsql hg17 -e \
     'select kgXref.kgID, spID, alias from kgXref, kgProtAlias where kgXref.kgID=kgProtAlias.kgID'\
     >>j.tmp
     cat j.tmp|sort -u |grep -v 'kgID' >hg17.kgSpAlias.tab
     rm j.tmp
 
     hgsql hg17 -e 'drop table kgSpAlias';
     hgsql hg17 < ~/src/hg/lib/kgSpAlias.sql
     hgsql hg17 -e 'load data local infile "hg17.kgSpAlias.tab" into table kgSpAlias'
     gzip kgSpAlias.tab
 
 # Create hg17GeneList.html (to be used by Google).
 # This step was done 12/08/04.
 
     cd /cluster/data/hg17/bed
     mkdir geneList
     cd geneList
     wget -O hg17GeneList.html "http://hgwdev-fanhsu.cse.ucsc.edu/cgi-bin/hgGeneList?db=hg17"
     cp -p hg17GeneList.html /usr/local/apache/htdocs/goldenPath
 # Check this html file into CVS.
 
 # CREATE FULL TEXT INDEX FOR KNOWN GENES (DONE 1/19/2006 JK)
 # This depends on the go and uniProt databases as well as 
 # the kgAlias and kgProAlias tables.  The hgKgGetText takes
 # about 5 minutes when the database is not too busy.  The rest
 # is real quick.
      ssh hgwdev
      cd /cluster/data/hg17/bed/kgHg17F
      mkdir index
      cd index
      hgKgGetText hg17 knownGene.text
      ixIxx knownGene.text knownGene.ix knownGene.ixx
      ln -s /cluster/data/hg17/bed/kgHg17F/index/knownGene.ix /gbdb/hg17/knownGene.ix
      ln -s /cluster/data/hg17/bed/kgHg17F/index/knownGene.ixx /gbdb/hg17/knownGene.ixx
 
 
 
 # UPDATE TABLES NEEDED BY hgGene (DONE 11/30/04 Fan)
 
 # UPDATE BioCyc TABLES
     hgsql hg17 -e 'delete from bioCycPathway'
     hgsql hg17 -e 'delete from bioCycMapDesc'
 
 #  Using data files sent by Peter Carp from SRI,
 #  per Peter's email of 10/1/04, they don't have recent update,
 #  so data files received last year are used.
   
 #  Save the BioCyc Pathway name and description table as gene-pathway.dat.
 #  Save the pathway data file as gene-pathway.dat.
 #  Make sure there is no extra ^M at end of the lines.
 
 # Run hgBioCyc program to generate the file bioCycPathway.tab.
 
 	 hgBioCyc gene-pathway.dat hg17
 
 # Load results into hg17.
 
     LOAD DATA local INFILE 'pathway-names.dat' into table bioCycMapDesc;
     LOAD DATA local INFILE 'bioCycPathway.tab' into table bioCycPathway;
 
 # REBUID FOLDUTR TABLES (DONE - 2004-11-30 Fan)
 # First set up directory structure and extract UTR sequence on hgwdev
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mv rnaStruct rnaStruct.2004-10-04
     mkdir -p /cluster/data/hg17/bed/rnaStruct.2004-11-30
     ln -s rnaStruct.2004-11-30 rnaStruct
     cd /cluster/data/hg17/bed/rnaStruct
     mkdir -p utr3/split utr5/split utr3/fold utr5/fold
     utrFa hg17 knownGene utr3 utr3/utr.fa
     utrFa hg17 knownGene utr5 utr5/utr.fa
 
 # Split up files and make files that define job.
     ssh kk
     cd /cluster/data/hg17/bed/rnaStruct
     faSplit sequence utr3/utr.fa 50000 utr3/split/s
     faSplit sequence utr5/utr.fa 50000 utr5/split/s
     ls -1 utr3/split > utr3/in.lst
     ls -1 utr5/split > utr5/in.lst
     cd utr3
     cat > gsub <<end
 #LOOP
 rnaFoldBig split/\$(path1) fold
 #ENDLOOP
 end
     cp gsub ../utr5
 
 # Do cluster run for 3' UTRs
     gensub2 in.lst single gsub spec
     para create spec
     para try
     para push
 # Completed: 38115 of 38115 jobs
 # CPU time in finished jobs:    1101680s   18361.33m   306.02h   12.75d  0.035 y
 # IO & Wait Time:                100275s    1671.25m    27.85h    1.16d  0.003 y
 # Average job time:                  32s       0.53m     0.01h    0.00d
 # Longest job:                     3645s      60.75m     1.01h    0.04d
 # Submission to last job:          7007s     116.78m     1.95h    0.08d
 
 # Do cluster run for 5' UTRs 
     cd ../utr5
     gensub2 in.lst single gsub spec
     para create spec
     para try
     para push
 # Completed: 30524 of 30524 jobs
 # CPU time in finished jobs:     116647s    1944.12m    32.40h    1.35d  0.004 y
 # IO & Wait Time:                 80477s    1341.28m    22.35h    0.93d  0.003 y
 # Average job time:                   6s       0.11m     0.00h    0.00d
 # Longest job:                     2449s      40.82m     0.68h    0.03d
 # Submission to last job:          3386s      56.43m     0.94h    0.04d
 
 # Load database
     ssh hgwdev
     cd /cluster/data/hg17/bed/rnaStruct/utr5
     hgLoadRnaFold hg17 foldUtr5 fold
 # Parsed 30525 files
 # Warning: load of foldUtr5 did not go as planned: 30525 record(s), 2 row(s) skipped, 0 warning(s) loading 
 ./foldUtr5.tab
 
     cd ../utr3
     hgLoadRnaFold hg17 foldUtr3 fold
 # Parsed 38115 files
 #Warning: load of foldUtr3 did not go as planned: 38115 record(s), 2 row(s) skipped, 0 warning(s) loading 
 ./foldUtr3.tab
 
 # Clean up
     rm -r split fold err batch.bak
     cd ../utr5
     rm -r split fold err batch.bak
 
 # UPDATE GENE SORTER TABLES (AKA: FAMILY BROWSER) (DONE - 2004-11-29 - Fan)
 #	This should be done after knownGene tables are complete from known gene
 #	process.
 #
 # Cluster together various alt-splicing isoforms.
 #	Creates the knownIsoforms and knownCanonical tables
 ssh hgwdev
 mkdir /cluster/data/hg17/bed/geneSorter.2004-11-24
 # remove old symbolic link
 rm /cluster/data/hg17/bed/geneSorter
 ln -s /cluster/data/hg17/bed/geneSorter.2004-11-24 \
 	/cluster/data/hg17/bed/geneSorter
 cd /cluster/data/hg17/bed/geneSorter
 hgClusterGenes hg17 knownGene knownIsoforms knownCanonical
 
 # Extract peptides from knownGenes into fasta file
 # and create a blast database out of them.
 mkdir /cluster/data/hg17/bed/geneSorter/blastp
 cd /cluster/data/hg17/bed/geneSorter/blastp
 pepPredToFa hg17 knownGenePep known.faa
 #	You may need to build this binary in src/hg/near/pepPredToFa
 /scratch/blast/formatdb -i known.faa -t known -n known
 #	This command is in /projects/compbio/bin/$MACH/formatdb
 
 # Copy over database to bluearc
 rm -fr /cluster/bluearc/hg17/blastp
 mkdir -p /cluster/bluearc/hg17/blastp
 cp -p /cluster/data/hg17/bed/geneSorter/blastp/known.* \
 	/cluster/bluearc/hg17/blastp
 
 #	Had to pick up a new blastall binary (2004-06-15)
 #	Our old one would no longer run on our systems that have
 #	updated Linux versions
 mkdir /cluster/bluearc/blast2210
 cd /cluster/bluearc/blast2210
 wget --timestamping \
     ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/2.2.10/blast-2.2.10-ia32-linux.tar.gz
 wget --timestamping \
     ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/2.2.10/ChangeLog.txt
 wget --timestamping \
     ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/2.2.10/ReleaseNotes.txt
 tar xvzf blast-2.2.10-ia32-linux.tar.gz
 
 # Split up fasta file into bite sized chunks for cluster
 cd /cluster/data/hg17/bed/geneSorter/blastp
 mkdir split
 faSplit sequence known.faa 8000 split/kg
 
 # Make parasol run directory
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/self
 cd /cluster/data/hg17/bed/geneSorter/blastp/self
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 -p blastp -d /cluster/bluearc/hg17/blastp/known -i $1 -o $2 \
 -e 0.01 -m 8 -b 1000
 '_EOF_'
     # << keep emacs happy
 chmod +x blastSome
 
 # Make gensub2 file
 cat  << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #	'ls ../../split/*.fa' is too much, hence the echo
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 # Wait a couple of minutes, and do a para check,  if all is good
 # then do a
 para push
 # This should finish in ~15 minutes if the cluster is free.
 Completed: 7748 of 7748 jobs
 CPU time in finished jobs:     191136s    3185.59m    53.09h    2.21d  0.006 y
 IO & Wait Time:                 66703s    1111.72m    18.53h    0.77d  0.002 y
 Average job time:                  33s       0.55m     0.01h    0.00d
 Longest job:                      370s       6.17m     0.10h    0.00d
 Submission to last job:           747s      12.45m     0.21h    0.01d
 
 # Load into database.  This takes about 30 minutes
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/self/run/out
 time hgLoadBlastTab hg17 knownBlastTab *.tab
 # Scanning through 7748 files
 # Loading database with 12810133 rows
 # 306.480u 54.190s 26:35.50 22.6% 0+0k 0+0io 206pf+0w
 
 cd /cluster/data/hg17/bed/geneSorter
 # Create table that maps between known genes and RefSeq
 hgMapToGene hg17 refGene knownGene knownToRefSeq
 #	may need to build this command in src/hg/near/hgMapToGene
 #	hgsql -e "select count(*) from knownToRefSeq;" hg17
 #	row count changed 37611
 
 # Create table that maps between known genes and LocusLink
 hgsql --skip-column-names -e "select mrnaAcc,locusLinkId from refLink" hg17 \
 	> refToLl.txt
 hgMapToGene hg17 refGene knownGene knownToLocusLink -lookup=refToLl.txt
 #	hgsql -e "select count(*) from knownToLocusLink;" hg17
 #	row count changed to 37611
 
 # Create table that maps between known genes and Pfam domains
 hgMapViaSwissProt hg17 knownGene name proteinID Pfam knownToPfam
 #	hgsql -e "select count(*) from knownToPfam;" hg17
 #	row count changed to 36302
 
 # Create table to map between known genes and GNF Atlas2
 # expression data.
     hgMapToGene hg17 gnfAtlas2 knownGene knownToGnfAtlas2 '-type=bed 12'
 #	hgsql -e "select count(*) from knownToGnfAtlas2;" hg17
 #	row count changed to 36373
 
 # Create expression distance table - takes about an hour
     hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
     	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
 	-lookup=knownToGnfAtlas2 &
 # Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
 # Got 36373 unique elements in hgFixed.gnfHumanAtlas2MedianRatio
 
 #	hgsql -e "select count(*) from gnfAtlas2Distance;" hg17
 #	row count changed to 36373000
 
 # Create a table that maps between known genes and 
 # the nice affy expression data.
 hgMapToGene "-type=bed 12" hg17 affyUclaNorm knownGene knownToU133
 #	hgsql -e "select count(*) from knownToU133;" hg17
 #	row count changed to 37299
 
 # Create expression distance table.  This will take about 2.5 hours
 cd /tmp
 cp -p ~/kent/src/hg/near/hgExpDistance/affyUcla.weight .
 time hgExpDistance hg17 affyUclaNorm affyUclaExp knownExpDistance \
 	-weights=affyUcla.weight -lookup=knownToU133 &
 # Have 43039 elements in affyUclaNorm
 # 211 genes, 42 weights, 26.500000 total wieght
 # Got 37299 unique elements in affyUclaNorm
 # 8212.320u 217.310s 2:38:07.84 88.8%     0+0k 0+0io 267pf+0w
 # Create table that maps between known genes and 
 # the GNF data.
 cd /tmp
 hgMapToGene hg17 affyU95 knownGene knownToU95
 #	row count changed to 18791
 #	hgFixed.gnfHumanU95Exps argument is unused, no need to exist
 hgExpDistance hg17 hgFixed.gnfHumanU95MedianRatio \
 	hgFixed.gnfHumanU95Exps gnfU95Distance  -lookup=knownToU95 &
 # Have 11545 elements in hgFixed.gnfHumanU95MedianRatio
 # Got 17682 unique elements in hgFixed.gnfHumanU95MedianRatio
 #	row count changed to 17682000 
 
 # Create known gene mapping table and expression distance tables
 # for GNF Atlas 2.  (The hgExpDistance takes only 10 minutes.)
 
 hgMapToGene hg17 affyGnf1h knownGene knownToGnf1h
 hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
 	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
 	-lookup=knownToGnf1h &
 # Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
 # Got 10273 unique elements in hgFixed.gnfHumanAtlas2MedianRatio
 
 cd /cluster/data/hg17/bed/geneSorter
 hgMapToGene hg17 affyU133Plus2 knownGene knownToU133Plus2
 #	row count changed to 40015 
 
 # Make sure that GO database is up to date.
 # UPDATE GO DATABASE (DONE 11/24/04 Fan)
 
 # Download the terms and make the database.
 ssh hgwdev
 mkdir /cluster/store1/geneOntology/20041124
 cd /cluster/store1/geneOntology/20041124
 
 wget --timestamping http://www.godatabase.org/dev/database/archive/latest/go_200411-assocdb-data.gz
 
 hgsql mysql <<end
 create database go041124;
 end
 zcat go_*data.gz | hgsql go041124
 
 wget -timestamping ftp://ftp.geneontology.org/pub/go/gene-associations/gene_association.goa_sptr.gz
 
 wget --timestamping ftp://ftp.geneontology.org/pub/go/gene-associations/gene_association.goa_uniprot.gz
 
 zcat gene_association.goa_uniprot.gz | hgGoAssociation go041124 goaPart stdin 
 # Passed 4502016 of 5291097 of 5291097, 85.09%
 
 # Ask sys-admin to switch the database pointer go to point to go041124.
 
 cd /cluster/data/hg17/bed/geneSorter
 
 XXX - DO NOT YET HAVE ensGene table - must wait on Ensembl to release that
 XXX - have not created the knownToEnsembl table yet - 2004-07-15 - Hiram
 # Create knownToEnsembl column
 hgMapToGene hg17 ensGene knownGene knownToEnsembl
 #	table row count went from previous version: 36068 to 38251
 
 # Make knownToCdsSnp table (Heather did this table, Nov 29, 2004)
   ssh hgwdev
   nice hgMapToGene hg17 snp knownGene knownToCdsSnp -all -cds
 # row count 168336
 # approx. 5 minutes running time
 
 # Make C. elegans ortholog column using blastp on wormpep.
 # First make C. elegans protein database and copy it to cluster/bluearc
 # There was no /cluster/bluearc/ce1/blastp, so get the latest wormpep from Sanger
 cd /cluster/data/ce1/bed/blastp
 mkdir old
 cp -p * old
 wget --timestamp ftp://ftp.sanger.ac.uk/pub/databases/wormpep/wormpep134/wormpep134
 mv wormpep134 wormPep.faa
 formatdb -i wormPep.faa -t wormPep -n wormPep
 #copy them to /cluster/bluearc
 ssh kkr1u00
 mkdir -p /cluster/bluearc/ce1/blastp
 cp /cluster/data/ce1/bed/blastp/wormPep.p?? /cluster/bluearc/ce1/blastp
 
 #	The blast jobs below can be run on the kk or kk9 clusters
 # Create the ceBlastTab
 ssh kk9
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/ce1
 cd /cluster/data/hg17/bed/geneSorter/blastp/ce1
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 	-p blastp -d /cluster/bluearc/ce1/blastp/wormPep \
 	-i $1 -o $2 -e 0.01 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 #	Only takes 10 minutes on an idle cluster
 # Completed: 7748 of 7748 jobs
 # CPU time in finished jobs:      33235s     553.91m     9.23h    0.38d  0.001 y
 # IO & Wait Time:                 19891s     331.52m     5.53h    0.23d  0.001 y
 # Average job time:                   7s       0.11m     0.00h    0.00d
 # Longest job:                       68s       1.13m     0.02h    0.00d
 # Submission to last job:           653s      10.88m     0.18h    0.01d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/ce1/run/out
 hgLoadBlastTab hg17 ceBlastTab -maxPer=1 *.tab
 #	row count changed to 28252
 
 # Make mouse ortholog column using blastp on mouse known genes.
 # First make mouse protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This already exists.  See makeMm5.doc for procedure
 #	the directory: /cluster/bluearc/scratch/mus/mm5/blastp should have data
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/mm5
 cd /cluster/data/hg17/bed/geneSorter/blastp/mm5
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 	-p blastp -d /cluster/bluearc/scratch/mus/mm5/blastp/known \
 	-i $1 -o $2 -e 0.001 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #	this echo trick is used because otherwise the command line is
 #	too long and you can not do a simple ls
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7748 of 7748 jobs
 # CPU time in finished jobs:     141842s    2364.04m    39.40h    1.64d  0.004 y
 # IO & Wait Time:                 52251s     870.85m    14.51h    0.60d  0.002 y
 # Average job time:                  25s       0.42m     0.01h    0.00d
 # Longest job:                      254s       4.23m     0.07h    0.00d
 # Submission to last job:           540s       9.00m     0.15h    0.01d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/mm5/run/out
 hgLoadBlastTab hg17 mmBlastTab -maxPer=1 *.tab
 # Scanning through 7748 files
 #	row count changed to 37549 
 
 # Make rat ortholog column using blastp on rat known genes.
 # First make rat protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This already exists.  See makeRn3.doc for procedure.
 #	Files were put in this directory: /cluster/bluearc/rn3/blastp/
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/rn3
 cd /cluster/data/hg17/bed/geneSorter/blastp/rn3
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
     -p blastp -d /cluster/bluearc/rn3/blastp/known \
     -i $1 -o $2 -e 0.001 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #	this echo trick is used because otherwise the command line is
 #	too long and you can not do a simple ls
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7748 of 7748 jobs
 # CPU time in finished jobs:      31786s     529.77m     8.83h    0.37d  0.001 y
 # IO & Wait Time:                 25795s     429.91m     7.17h    0.30d  0.001 y
 # Average job time:                   7s       0.12m     0.00h    0.00d
 # Longest job:                       75s       1.25m     0.02h    0.00d
 # Submission to last job:           157s       2.62m     0.04h    0.00d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/rn3/run/out
 hgLoadBlastTab hg17 rnBlastTab -maxPer=1 *.tab
 # Scanning through 7748 files
 #Loading database with 26133 rows
 
 # Make Danio rerio (zebrafish) ortholog column using blastp on Ensembl.
 # First make protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 # NOTE: data used to reside in /cluster/bluearc/dr1/blastp
 mv /cluster/bluearc/dr1/blastp /cluster/bluearc/danRer1/blastp
 #	the directory: /cluster/bluearc/danRer1/blastp should have data
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/danRer1
 cd /cluster/data/hg17/bed/geneSorter/blastp/danRer1
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 	-p blastp -d /cluster/bluearc/danRer1/blastp/ensembl \
 	-i $1 -o $2 -e 0.005 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7748 of 7748 jobs
 # CPU time in finished jobs:     102324s    1705.39m    28.42h    1.18d  0.003 y
 # IO & Wait Time:                 47203s     786.72m    13.11h    0.55d  0.001 y
 # Average job time:                  19s       0.32m     0.01h    0.00d
 # Longest job:                      230s       3.83m     0.06h    0.00d
 # Submission to last job:           427s       7.12m     0.12h    0.00d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/danRer1/run/out
 hgLoadBlastTab hg17 drBlastTab -maxPer=1 *.tab
 # Loading database with 33852 rows
 
 # Make Saccharomyces cerevisiae (yeast) ortholog column using blastp on RefSeq.
 # First make protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This is already done, see makeMm3.doc for procedure
 #	the directory: /cluster/bluearc/sc1/blastp should have data
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/sc1
 cd /cluster/data/hg17/bed/geneSorter/blastp/sc1
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 	-p blastp -d /cluster/bluearc/sc1/blastp/sgd \
 	-i $1 -o $2 -e 0.01 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7748 of 7748 jobs
 # CPU time in finished jobs:      20983s     349.72m     5.83h    0.24d  0.001 y
 # IO & Wait Time:                 25513s     425.21m     7.09h    0.30d  0.001 y
 # Average job time:                   6s       0.10m     0.00h    0.00d
 # Longest job:                       37s       0.62m     0.01h    0.00d
 # Submission to last job:           106s       1.77m     0.03h    0.00d
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/sc1/run/out
 hgLoadBlastTab hg17 scBlastTab -maxPer=1 *.tab
 # Loading database with 18489 rows
 
 # Make Drosophila melanagaster ortholog column using blastp on FlyBase.
 # First make SwissProt protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This is already done, see makeMm3.doc for procedure
 #	the directory: /cluster/bluearc/dm1/blastp should have data
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/dm1
 cd /cluster/data/hg17/bed/geneSorter/blastp/dm1
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 	-p blastp -d /cluster/bluearc/dm1/blastp/flyBase \
 	-i $1 -o $2 -e 0.01 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7748 of 7748 jobs
 # CPU time in finished jobs:      83377s    1389.62m    23.16h    0.97d  0.003 y
 # IO & Wait Time:                 39913s     665.21m    11.09h    0.46d  0.001 y
 # Average job time:                  16s       0.27m     0.00h    0.00d
 # Longest job:                      167s       2.78m     0.05h    0.00d
 # Submission to last job:           365s       6.08m     0.10h    0.00d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/dm1/run/out
 hgLoadBlastTab hg17 dmBlastTab -maxPer=1 *.tab
 # Loading database with 30067 rows
 
 # update knownToHInv table
 # Verified that there is now new release of HInv data.
 hgMapToGene hg17 HInvGeneMrna knownGene knownToHInv
 # count changed to 33236
 
 #### BUILD Ensembl cross-reference table, ensemblXref3 (DONE - 2004-11-30 - Fan)
 
     # Get the ensembl gene/protein cross-reference data from
     # http://www.ensembl.org/Homo_sapiens/martview
     # Follow this sequence through the pages:
     # Page 1) Make sure that the Homo_sapiens choice is selected. Hit next.
     # Page 2) Uncheck the "Limit to" box in the region choice. Then hit next.
     # Page 3) Choose the "Feature" box, select gene, transcript, protein,
               SPTrEMBL ID, SWISSPROT ID, and SWISSPROT AC
     # Page 4) Choose "Text, tab separated".  choose gzip compression.  hit export.
     # Save as ensXref.txt
 
     sed ensXref.txt -e 's/\./\t/g' > ensemblXref3.tab
 
     hgsql hg17 -e "drop table ensemblXref3"
     hgsql hg17 < ~/src/hg/lib/ensemblXref3.sql
 
     hgsql hg17 -e 'load data local infile "ensemblXref3.tab" into table ensemblXref3 ignore 1 lines'
 
 #### BUILD SUPERFAMILY RELATED TABLES (DONE - 2004-11-30 - Fan)
 
 # Download Superfamily data files and build the Superfamily DB
 # from supfam.mrc-lmb.cam.ac.uk
 
     mkdir /cluster/store8/superfamily/041128
     ln -s /cluster/store8/superfamily/041128 /cluster/data/superfamily/041128
     cd /cluster/data/superfamily/041128
 # ftp over the following two files:
 ass_28-Nov-2004.tab.gz
       supfam_28-Nov-2004.sql.gz
     
     gzip -d *.gz
 # Load the Superfamily database
     hgsql hg17 -e "create database superfam041128"
     hgsql superfam041128 < supfam_28-Nov-2004.sql
 # This may take about an hour.
 
 # Make sure to add an index on id of the des table of superfam041128.
     hgsql superfam041128 -e "create index id on des(id);"
 
     hgsql superfam041128 < ~/src/hg/lib/sfAssign.sql
     hgsql superfam041128 -e 'load data local infile "ass_28-Nov-2004.tab" into table 
 superfam041128.sfAssign;'
 
 # Build or rebuild Superfamily track and create sf tables needed for PB
 
    hgsql hg17 < ~/src/hg/lib/sfAssign.sql
 
    cd /cluster/data/superfamily/041128  
    hgsql hg17 -e 'load data local infile "ass_28-Nov-2004.tab" into table hg17.sfAssign;'
 
 # If hg17.sfDes already exists, drop it.
 
    hgsql superfam041128 -e "select * from des" >sfDes.tab
    hgsql hg17 < ~/src/hg/lib/sfDes.sql
    hgsql hg17 -e 'load data local infile "sfDes.tab" into table hg17.sfDes ignore 1 lines;'
 
 # If hg17.superfamily already exists, drop it.
    cd /cluster/data/hg17/bed
    mkdir /cluster/data/hg17/sf.2004-1128
    ln -s sf.2004-1128 sf
    hgSuperfam hg17 > sf.log
 
 # It is normal that many proteins does not have corresponding Superfamily entries.
 
 # If hg17.sfDescription exists, drop it.
 
    hgsql hg17 < ~/src/hg/lib/sfDescription.sql
    hgsql hg17 -e 'LOAD DATA local INFILE "sfDescription.tab"  into table hg17.sfDescription;'
 
 # Finally, load the superfamily table.
 
    hgLoadBed hg17 superfamily superfamily.tab -tab
 
 # Create knownToSuperfamily table
 # Note hs is changed into ht for this Superfamily release.
    
    cat /cluster/data/superfamily/041128/ass_28-Nov-2004.tab \
    | hgKnownToSuper hg17 hs stdin
 # created 25287 rows in knownToSuper
 
 ### HG17 PROTEOME BROWSER TABLES RE-BUILD ####  (DONE - 2004-12-01 - Fan)
 # These are instructions for rebuilding tables 
 # needed for the Proteome Browser.  
 # DON'T START THESE UNTIL TABLES FOR KNOWN GENES AND kgProtMap table
 # ARE REBUILT.  
 # This update is based on proteins DBs dated 041115.
 
 # Create the working directory
 
     ssh hgwdev
     mv /cluster/data/hg17/bed/pb /cluster/data/hg17/bed/pb.2004-06-11
     mkdir /cluster/data/hg17/bed/pb.2004-12-01
     cd /cluster/data/hg17/bed
     ln -s /cluster/data/hg17/bed/pb.2004-12-01 pb
 
 # Move the existing PB tables by:
 
 hgsql hg17
 create database hg17Sav2;
 
 alter table hg17.pepCCntDist rename as hg17Sav2.pepCCntDist;
 alter table hg17.pepExonCntDist rename as hg17Sav2.pepExonCntDist;
 alter table hg17.pepHydroDist rename as hg17Sav2.pepHydroDist;
 alter table hg17.pepIPCntDist rename as hg17Sav2.pepIPCntDist;
 alter table hg17.pepMolWtDist rename as hg17Sav2.pepMolWtDist;
 alter table hg17.pepMwAa rename as hg17Sav2.pepMwAa;
 alter table hg17.pepPi rename as hg17Sav2.pepPi;
 alter table hg17.pepPiDist rename as hg17Sav2.pepPiDist;
 alter table hg17.pepResDist rename as hg17Sav2.pepResDist;
 
 alter table hg17.pbAaDistA rename as hg17Sav2.pbAaDistA;
 alter table hg17.pbAaDistC rename as hg17Sav2.pbAaDistC;
 alter table hg17.pbAaDistD rename as hg17Sav2.pbAaDistD;
 alter table hg17.pbAaDistE rename as hg17Sav2.pbAaDistE;
 alter table hg17.pbAaDistF rename as hg17Sav2.pbAaDistF;
 alter table hg17.pbAaDistG rename as hg17Sav2.pbAaDistG;
 alter table hg17.pbAaDistH rename as hg17Sav2.pbAaDistH;
 alter table hg17.pbAaDistI rename as hg17Sav2.pbAaDistI;
 alter table hg17.pbAaDistK rename as hg17Sav2.pbAaDistK;
 alter table hg17.pbAaDistL rename as hg17Sav2.pbAaDistL;
 alter table hg17.pbAaDistM rename as hg17Sav2.pbAaDistM;
 alter table hg17.pbAaDistN rename as hg17Sav2.pbAaDistN;
 alter table hg17.pbAaDistP rename as hg17Sav2.pbAaDistP;
 alter table hg17.pbAaDistQ rename as hg17Sav2.pbAaDistQ;
 alter table hg17.pbAaDistR rename as hg17Sav2.pbAaDistR;
 alter table hg17.pbAaDistS rename as hg17Sav2.pbAaDistS;
 alter table hg17.pbAaDistT rename as hg17Sav2.pbAaDistT;
 alter table hg17.pbAaDistV rename as hg17Sav2.pbAaDistV;
 alter table hg17.pbAaDistW rename as hg17Sav2.pbAaDistW;
 alter table hg17.pbAaDistY rename as hg17Sav2.pbAaDistY;
 alter table hg17.pbAnomLimit rename as hg17Sav2.pbAnomLimit;
 alter table hg17.pbResAvgStd rename as hg17Sav2.pbResAvgStd;
 alter table hg17.pbStamp rename as hg17Sav2.pbStamp;
 
 quit
 
 # Define pep* tables in hg17 DB
 
 	cat ~/kent/src/hg/lib/pep*.sql > pepAll.sql
 
 #  First edit out pepPred table definition, then
 
 	hgsql hg17 < pepAll.sql
 
 # Build the pepMwAa table
 
   hgsql proteins041115 -e "select info.acc, molWeight, aaSize from sp041115.info, sp041115.accToTaxon where 
 accToTaxon.taxon=9606 and accToTaxon.acc = info.acc" > pepMwAa.tab
 
 hgsql hg17 <<end
 load data local infile "pepMwAa.tab" into table hg17.pepMwAa ignore 1 lines;
 end
 
 o Build the pepPi table
 
   hgsql proteins041115 -e "select info.acc from sp041115.info, sp041115.accToTaxon where 
 accToTaxon.taxon=9606 and accToTaxon.acc = info.acc" > protAcc.lis
 
   pbCalPi protAcc.lis sp041115 pepPi.tab
 
 hgsql hg17 <<end
 load data local infile "pepPi.tab" into table hg17.pepPi;
 end
 
 # Calculate and load pep distributions
 
   pbCalDist sp041115 proteins041115 9606 hg17 >pbCalDist.out
 
     cat pbCalDist.out
     wc  pbCalDist.out
 
     hgsql hg17
 
     load data local infile "pepExonCntDist.tab" into table hg17.pepExonCntDist;
     load data local infile "pepCCntDist.tab" into table hg17.pepCCntDist;
     load data local infile "pepHydroDist.tab" into table hg17.pepHydroDist;
     load data local infile "pepMolWtDist.tab" into table hg17.pepMolWtDist;
     load data local infile "pepResDist.tab" into table hg17.pepResDist;
     load data local infile "pepIPCntDist.tab" into table hg17.pepIPCntDist;
     load data local infile "pepPiDist.tab" into table hg17.pepPiDist;
     quit
 
 # Calculate frequency distributions
 
     pbCalResStd 041115 9606 hg17
 
 # Create pbAnomLimit and pbResAvgStd tables
 
    hgsql hg17 < ~/src/hg/lib/pbAnomLimit.sql
    hgsql hg17 < ~/src/hg/lib/pbResAvgStd.sql
 
    hgsql hg17 -e 'load data local infile "pbResAvgStd.tab" into table hg17.pbResAvgStd;'
    hgsql hg17 -e 'load data local infile "pbAnomLimit.tab" into table hg17.pbAnomLimit;'
 
 # Create pbStamp table for PB
    
   hgsql hg17 < ~/src/hg/lib/pbStamp.sql
   hgsql hg17Sav2 -e 'select * from pbStamp' > pbStamp.tab
 
   hgsql hg17 -e 'load data local infile "pbStamp.tab" into table hg17.pbStamp ignore 1 lines;'
 
 # Adjust drawing parameters for Proteome Browser stamps
 
   Now invoke Proteome Browser and adjust various drawing parameters
   (mostly the ymax of each stamp) if necessary, by updating the 
   pbStamp.tab file and then delete and reload the pbStamp table. 
 
 # Perform preliminary review of Proteome Browser for hg17, then
   notify QA for formal review.
 
 ####  Blat knownGene proteins to determine exons  (braney DONE 12/11/04)
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mkdir blat.hg17KG.2004-12-08
     rm blat.hg17KG
     ln -s  blat.hg17KG.2014-12-08 blat.hg17KG
     cd blat.hg17KG
     pepPredToFa hg17 knownGenePep known.fa
     ssh kk
     cd /cluster/data/hg17/bed/blat.hg17KG
     cat << '_EOF_' > blatSome
 #!/bin/csh -fe
 /cluster/bin/i386/blat -t=dnax -q=prot -out=pslx $1 $2 $3
 '_EOF_'
     cat << '_EOF_' > blatGsub
 #LOOP
 blatSome $(path1) {check in line $(path2)} {check out line psl/$(root1)/$(root2).psl}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
     chmod +x blatSome
     ls -1S /scratch/hg/gs.18/build35/bothMaskedNibs/*.nib > human.lst
     mkdir kgfa
     cd kgfa
     faSplit sequence ../known.fa 3010 kg
     cd ..
     ls -1S kgfa/*.fa > kg.lst
     gensub2 human.lst kg.lst blatGsub blatSpec
     mkdir psl
     cd psl
     foreach i (`cat ../human.lst`)
 	mkdir `basename $i .nib`
     end
     cd ..
     para create blatSpec
     para push
 
 # Completed: 134130 of 134136 jobs
 # Crashed: 6 jobs
 # CPU time in finished jobs:   29801114s  496685.23m  8278.09h  344.92d  0.945 y
 # IO & Wait Time:               1983513s   33058.55m   550.98h   22.96d  0.063 y
 # Average job time:                 237s       3.95m     0.07h    0.00d
 # Longest job:                    63306s    1055.10m    17.59h    0.73d
 # Submission to last job:        169384s    2823.07m    47.05h    1.96d
 
 # did 6 crashed jobs on small cluster
     ssh eieio
     cd /cluster/data/hg17/bed/blat.hg17KG
     pslSort dirs raw.psl /tmp psl/*
     pslReps -nohead -minCover=0.9 -minAli=0.9 raw.psl cooked.psl /dev/null
     pslUniq cooked.psl hg17KG.psl
     pslxToFa hg17KG.psl hg17KG_ex.fa -liftTarget=genome.lft -liftQuery=protein.lft
     kgName hg17 hg17KG.psl blastKGRef01
     cut -f 10 hg17KG.psl > kgName.lst
     faSomeRecords known.fa kgName.lst hg17KG.fa
     hgPepPred hg17 generic blastKGPep01 hg17KG.fa
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/blat.hg17KG
     hgsql hg17 < ~/kent/src/hg/lib/blastRef.sql
     echo "rename table blastRef to blastKGRef01" | hgsql hg17
     echo "load data local infile 'blastKGRef01' into table blastKGRef01" | hgsql hg17
 
 #### TIGR GENE INDEX (DONE 2004-12-04 Fan)
     mkdir -p /cluster/data/hg17/bed/tigr
     cd /cluster/data/hg17/bed/tigr
     wget -timestamp ftp://ftp.tigr.org/pub/data/tgi/Homo_sapiens/TGI_track_HumanGenome_build35.tgz
     tar xvzf TGI*.tgz
 
     foreach f (*cattle*)
       set f1 = `echo $f | sed -e 's/cattle/cow/g'`
       mv $f $f1
     end
 
     foreach o (mouse cow human pig rat)
       echo $o
       setenv O $o
       foreach f (chr*_$o*s)
         tail +2 $f | perl -wpe 's /THC/TC/; s/(TH?C\d+)/$ENV{O}_$1/;' > $f.gff
       end
     end
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/tigr
     hgsql hg17 -e "drop table tigrGeneIndex"
     hgsql hg17 < ~/kent/src/hg/lib/tigrGeneIndex.sql
 
     foreach f (*.gff)
         echo Processing $f ...
         /cluster/home/fanhsu/bin/i386/ldHgGene -oldTable -exon=TC hg17 tigrGeneIndex $f
         hgsql hg17 -e "select count(*) from tigrGeneIndex"
     end
     # Total of 401322 entries created in tigrGeneIndex table.
 
     hgsql hg17 -e "update tigrGeneIndex set cdsStart = txStart;"
     hgsql hg17 -e "update tigrGeneIndex set cdsEnd = txEnd;"
 
     checkTableCoords hg17 tigrGeneIndex
 
     gzip *.gff *TCs
 
 # BLASTZ FOR ZEBRAFISH (danRer2) (DONE, 2004-12-09, hartera)
     ssh kkr1u00
     # blastz requires lineage-specific repeats
     # Treat all repeats as lineage-specific.
  # /iscratch/i/gs.18/build35/linSpecRep.notInZebrafish exists (makeDanRer1.doc)
     mkdir -p /iscratch/i/danRer2/linSpecRep.notInHuman
     foreach f (/iscratch/i/danRer2/rmsk/chr*.fa.out)
       cp -p $f /iscratch/i/danRer2/linSpecRep.notInHuman/$f:t:r:r.out.spec
     end
     iSync
 
     ssh kk
     mkdir -p /cluster/data/hg17/bed/blastz.danRer2.2004-12-08
     ln -s /cluster/data/hg17/bed/blastz.danRer2.2004-12-08 \
           /cluster/data/hg17/bed/blastz.danRer2
     cd /cluster/data/hg17/bed/blastz.danRer2
     # Set L=6000 and abridge repeats - these are the same parameters used
     # for hg16 and Fugu and similar to those for hg16-galgal2
                                                                                 
     cat << '_EOF_' > DEF
 # human (hg17) vs zebrafish (danRer2)
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin
                                                                                 
 ALIGN=blastz-run
 BLASTZ=blastz
                                                                                 
 # Reuse parameters from hg16-fr1.
 BLASTZ_H=2000
 BLASTZ_Y=3400
 BLASTZ_L=6000
 BLASTZ_K=2200
 BLASTZ_Q=/cluster/data/blastz/HoxD55.q
 BLASTZ_ABRIDGE_REPEATS=1
                                                                                 
 # TARGET: Human (hg17)
 SEQ1_DIR=/iscratch/i/gs.18/build35/bothMaskedNibs
 SEQ1_RMSK=
 SEQ1_FLAG=
 SEQ1_SMSK=/iscratch/i/gs.18/build35/linSpecRep.notInZebrafish
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
                                                                                 
 # QUERY: zebrafish (danRer2)
 SEQ2_DIR=/iscratch/i/danRer2/nib/
 SEQ2_RMSK=
 SEQ2_FLAG=
 SEQ2_SMSK=/iscratch/i/danRer2/linSpecRep.notInHuman
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=10000000
 SEQ2_LAP=0
                                                                                 
 BASE=/cluster/data/hg17/bed/blastz.danRer2
                                                                                 
 DEF=$BASE/DEF
 RAW=$BASE/raw
 CDBDIR=$BASE
 SEQ1_LEN=$BASE/S1.len
 SEQ2_LEN=$BASE/S2.len
 
 #DEBUG=1
 '_EOF_'
     # << this line keeps emacs coloring happy
     chmod +x DEF
     # Save the DEF file in the current standard place
     cp DEF ~angie/hummus/DEF.hg17-danRer2.2004-12-08
     # prepare first cluster run
     ssh kk
     cd /cluster/data/hg17/bed/blastz.danRer2
     bash # if a csh/tcsh user
     . ./DEF
     /cluster/data/hg17/jkStuff/BlastZ_run0.sh
     cd run.0
     para try, check, push, check ...etc.
 # para time
 # Completed: 58993 of 58993 jobs
 # CPU time in finished jobs:   19583036s  326383.93m  5439.73h  226.66d  0.621 y
 # IO & Wait Time:                471090s    7851.50m   130.86h    5.45d  0.015 y
 # Average job time:                 340s       5.67m     0.09h    0.00d
 # Longest job:                      885s      14.75m     0.25h    0.01d
 # Submission to last job:         78245s    1304.08m    21.73h    0.91d
 
     ssh kki
     cd /cluster/data/hg17/bed/blastz.danRer2
     bash # if a csh/tcsh user
     . ./DEF
     /cluster/data/hg17/jkStuff/BlastZ_run1.sh
     cd run.1
     para try, check, push, etc ...
 # para time
 # Completed: 341 of 341 jobs
 # CPU time in finished jobs:        789s      13.14m     0.22h    0.01d  0.000 y
 # IO & Wait Time:                  2992s      49.87m     0.83h    0.03d  0.000 y
 # Average job time:                  11s       0.18m     0.00h    0.00d
 # Longest job:                       34s       0.57m     0.01h    0.00d
 # Submission to last job:           391s       6.52m     0.11h    0.00d
 
     #   Third cluster run to convert lav's to axt's
     ssh kki
     cd /cluster/data/hg17/bed/blastz.danRer2
     mkdir axtChrom
     # a new run directory
     mkdir run.2
     cd run.2
 cat << '_EOF_' > do.csh
 #!/bin/csh
 cd $1
 cat `ls -1 *.lav | sort -g` \
 | lavToAxt stdin /iscratch/i/gs.18/build35/bothMaskedNibs \
  /iscratch/i/danRer2/nib stdout \
 | axtSort stdin $2
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod a+x do.csh
     cat << '_EOF_' > gsub
 #LOOP
 ./do.csh {check in exists $(path1)} {check out line+ /cluster/data/hg17/bed/blastz.danRer2/axtChrom/$(root1).axt}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
     \ls -1Sd ../lav/chr* > chrom.list
     gensub2 chrom.list single gsub jobList
     wc -l jobList
     head jobList
     para create jobList
     para try, check, push, check,...
 # para time
 # Completed: 45 of 46 jobs
 # Crashed: 1 jobs
 # CPU time in finished jobs:         99s       1.64m     0.03h    0.00d  0.000 y
 # IO & Wait Time:                   862s      14.37m     0.24h    0.01d  0.000 y
 # Average job time:                  21s       0.36m     0.01h    0.00d
 # Longest job:                       92s       1.53m     0.03h    0.00d
 # Submission to last job:           456s       7.60m     0.13h    0.01d
     # crashed job: chr6_hla_hap1.axt is empty - has no alignments
     # translate sorted axt files into psl
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.danRer2
     mkdir -p pslChrom
     set tbl = "blastzDanRer2"
     foreach f (axtChrom/chr*.axt)
       set c=$f:t:r
       echo "Processing chr $c"
       /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
     end
                                                                                
     # Load database tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.danRer2/pslChrom
                                                                                
     foreach f (./*.psl)
       /cluster/bin/i386/hgLoadPsl -noTNameIx hg17 $f
       echo "$f Done"
     end
 # try different parameters for blastz with chr1 of hg17
 # featureBits -chrom=chr1 hg17 refGene:cds blastzDanRer1 -enrichment
 # refGene:cds 1.301%, blastzDanRer1 3.934%, both 0.874%, cover 67.23%, 
 # enrich 17.09x
 # H=2000, Y=3400, L=6000, K=2200 and HoxD55.q scoring matrix
 # featureBits -chrom=chr1 hg17 refGene:cds blastzDanRer2 -enrichment
 # refGene:cds 1.301%, blastzDanRer2 3.845%, both 0.879%, cover 67.55%, 
 # enrich 17.57x
 # same parameters as above but L=8000
 # featureBits -chrom=chr1 hg17 refGene:cds blastzDanRer2L8k -enrichment
 # refGene:cds 1.301%, blastzDanRer2L8k 2.309%, both 0.778%, cover 59.81%, 
 # enrich 25.91x
 # enrichment went up but coverage dropped quite a bit.
 # Default parameters with H=2000
 # featureBits -chrom=chr1 hg17 refGene:cds blastzDanRer2Default -enrichment
 # refGene:cds 1.301%, blastzDanRer2Default 1.701%, both 0.846%, cover 65.04%, 
 # enrich 38.24x
 # same as first run but with no Y option set (default Y)
 # featureBits -chrom=chr1 hg17 refGene:cds blastzDanRer2NoY -enrichment
 # refGene:cds 1.301%, blastzDanRer2NoY 3.980%, both 0.877%, cover 67.47%, 
 # enrich 16.95x
 
 # row count:
 # danRer2  122160
 # danRer2L8k 62815
 # danRer2Default 75818
 # danRer2NoY 124129
 # can be pruned at the chaining step.
 # trackDb - change Zebrafish Blastz to danRer1 Blastz and display this track
 # for danRer2 as Zebrafish Blastz
 
 # RESCORE DANRER2 BLASTZ (DONE, 2004-12-09, hartera)
     # Low scores can occur with repeats abridged and using the
     # HoxD55.q matrix. PSU's restore_rpts program rescored alignments
     # with the default matrix instead of the BLASTZ_Q matrix.
     # Rescore them here so the chainer sees the higher scores:
                                                                                 
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.danRer2
     mkdir axtChrom.rescore
     foreach f (axtChrom/chr*.axt)
         axtRescore -scoreScheme=/cluster/data/blastz/HoxD55.q \
         $f axtChrom.rescore/$f:t
     end
     mv axtChrom axtChrom.orig
     mv axtChrom.rescore axtChrom
                                                                                 
 #   psl files and blastz tables will be the same regardless of score so
 #   no need to reload
 
 # CHAIN DANRER2 BLASTZ (DONE, 2004-12-09, hartera)
 # RELOAD CHAINS WIH FILTERING (DONE, 2004-12-10, hartera)
 # APPLY chainAntiRepeat TO REMOVE CHAINS THAT ARE THE RESULTS OF REPEATS
 # AND DEGENERATE DNA (DONE, 2004-12-22, hartera)
     # Run axtChain on little cluster
     ssh kki
     cd /cluster/data/hg17/bed/blastz.danRer2
     mkdir -p axtChain/run1
     cd axtChain/run1
     mkdir out chain
     # create input list
     ls -1S /cluster/data/hg17/bed/blastz.danRer2/axtChrom/*.axt \
         > input.lst
     cat << '_EOF_' > gsub
 #LOOP
 doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
                                                                                 
     # Reuse gap penalties from hg16 vs chicken run.
     cat << '_EOF_' > ../../chickenHumanTuned.gap
 tablesize^V     11
 smallSize^V     111
 position^V      1^V     2^V     3^V     11^V    111^V   2111^V  12111^V 32111^V
 72111^V 152111^V        252111
 qGap^V  325^V   360^V   400^V   450^V   600^V   1100^V  3600^V  7600^V  15600^V
 31600^V 56600
 bothGap^V       625^V   660^V   700^V   750^V   900^V   1400^V  4000^V  8000^V
 16000^V 32000^V 57000
 '_EOF_'
     # << this line makes emacs coloring happy
     cat << '_EOF_' > gsub
 #LOOP
 doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
 #ENDLOOP
 '_EOF_'
     # << this line makes emacs coloring happy
 cat << '_EOF_' > doChain
 #!/bin/csh
 axtChain -scoreScheme=/cluster/data/blastz/HoxD55.q \
     -linearGap=../../chickenHumanTuned.gap $1 \
     /iscratch/i/gs.18/build35/bothMaskedNibs \ 
     /iscratch/i/danRer2/nib $2 >& $3
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod a+x doChain
     gensub2 input.lst single gsub jobList
     para create jobList
     para try, check, push, check...
 # para time
 # Completed: 45 of 46 jobs
 # Crashed: 1 jobs
 # CPU time in finished jobs:       1837s      30.62m     0.51h    0.02d  0.000 y
 # IO & Wait Time:                   441s       7.35m     0.12h    0.01d  0.000 y
 # Average job time:                  51s       0.84m     0.01h    0.00d
 # Longest job:                      106s       1.77m     0.03h    0.00d
 # Submission to last job:           419s       6.98m     0.12h    0.00d
 
     # crashed job is chr6_hla_hap1 which has no alignments
     # now on the cluster server, sort chains
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
     chainMergeSort run1/chain/*.chain > all.chain
     chainSplit chain all.chain
                                                                                 
     # take a look at score distr's
     foreach f (chain/*.chain)
       grep chain $f | awk '{print $2;}' | sort -nr > /tmp/score.$f:t:r
       echo $f:t:r >> hist5000.out
       textHistogram -binSize=5000 /tmp/score.$f:t:r >> hist5000.out
       echo ""
     end
 
     # apart from chr19 not too many with chains with scores < 5000
     # load chr1 chain into table and check
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.danRer2/axtChain/chain
     hgLoadChain hg17 chr1_chainDanRer2 chr1.chain
 # featureBits -chrom=chr1 hg17 refGene:cds chainDanRer2Link -enrichment
 # refGene:cds 1.301%, chainDanRer2Link 3.676%, both 0.877%, cover 67.42%,
 # enrich 18.34x
 # featureBits -chrom=chr1 hg17 refGene:cds chainDanRer2 -enrichment
 # refGene:cds 1.301%, chainDanRer2 32.611%, both 1.034%, cover 79.52%, 
 # enrich 2.44x
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
     mv all.chain all.chain.unfiltered
     chainFilter -minScore=5000 all.chain.unfiltered > all.chain
     chainSplit chainFilt5k all.chain
 
     # load chr1 filtered chains and check
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.danRer2/axtChain/chainFilt5k
     hgLoadChain hg17 chr1_chainDanRer2Filt5k chr1.chain
 # featureBits -chrom=chr1 hg17 refGene:cds chainDanRer2Filt5kLink -enrichment
 # refGene:cds 1.301%, chainDanRer2Filt5kLink 2.907%, both 0.870%, cover 66.86%,
 # enrich 23.00x
 # featureBits -chrom=chr1 hg17 refGene:cds chainDanRer2Filt5k -enrichment
 # refGene:cds 1.301%, chainDanRer2Filt5k 31.343%, both 1.028%, cover 79.02%, 
 # enrich 2.52x
 # checked browser - when filtered on minScore=5000, the low scoring 
 # alignments removed are small and/or poor alignments so use this version.
     
     # remove repeats from filtered chains and reload into database
     # (2004-12-22, hartera)
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
     mv chainFilt5k chainRaw
     mkdir chain
     cd chainRaw
     foreach f (*.chain)
        set c = $f:r
        echo $c
        nice chainAntiRepeat /cluster/bluearc/hg17/bothMaskedNibs \
                        /cluster/bluearc/danRer2/nib $f \
                        ../chain/$c.chain
     end
     cd ..
     chainMergeSort ./chain/*.chain > all.chain.antirepeat
     chainSplit chainAR all.chain.antirepeat
     # load filtered chains and check
     ssh hgwdev
     echo 'drop table chr1_chainDanRer2Filt5k;' | hgsql hg17
     echo 'drop table chr1_chainDanRer2Filt5kLink;' | hgsql hg17
     # reload filtered chains instead of unfiltered (2004-12-10, hartera)
     # reload filtered chains with repeats removed (2004-12-22, hartera)
     cd /cluster/data/hg17/bed/blastz.danRer2/axtChain/
     cd chainAR
     foreach i (*.chain)
         set c = $i:r
         hgLoadChain hg17 ${c}_chainDanRer2 $i
         echo done $c
     end
 # trackDb - change Zebrafish Chain to danRer1 Chain and display this track
 # for danRer2 as Zebrafish Chain.
 # after chainAntiRepeat
 # featureBits -chrom=chr1 hg17 refGene:cds chainDanRer2Link -enrichment
 # refGene:cds 1.304%, chainDanRer2Link 2.742%, both 0.872%, cover 66.81%, 
 # enrich 24.36x 
 
 # NET DANRER2 BLASTZ (DONE, 2004-12-09, hartera)
 # RE-CREATE NET WITH FILTERED CHAINS (DONE, 2004-12-10, hartera)
 # RE-DO NET WITH CHAINS FILTERED BY chainAntiRepeat (DONE, 2004-12-22, hartera) 
     ssh kksilo
     cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
     rm -r preNet
     mkdir preNet
     cd chainAR
     foreach i (*.chain)
        echo preNetting $i
        /cluster/bin/i386/chainPreNet $i ../../S1.len ../../S2.len \
                                      ../preNet/$i
     end
     cd ..
     mkdir n1
     cd preNet
     foreach i (*.chain)
       set n = $i:r.net
       echo primary netting $i
       /cluster/bin/i386/chainNet $i -minSpace=1 ../../S1.len ../../S2.len \
                                  ../n1/$n /dev/null
     end
     cd ..
     cat n1/*.net | /cluster/bin/i386/netSyntenic stdin noClass.net
     # memory usage 133443584, utime 905 s/100, stime 139
                                                                                 
     # Add classification info using db tables:
     # netClass looks for ancient repeats in one of the databases
     # hg17 has this table - hand-curated by Arian but this is for
     # human-rodent comparisons so do not use here, use -noAr option
     mkdir -p /cluster/bluearc/danRer2/linSpecRep.notInHuman
     # linSpecRep.notInZebrafish exists for hg17 
     cp /iscratch/i/danRer2/linSpecRep.notInHuman/* \
        /cluster/bluearc/danRer2/linSpecRep.notInHuman
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
     time netClass noClass.net hg17 danRer2 zfishdanRer2.net \
          -tNewR=/cluster/bluearc/hg17/linSpecRep.notInZebrafish \
          -qNewR=/cluster/bluearc/danRer2/linSpecRep.notInHuman -noAr
     # 97.230u 54.290s 5:37.50 44.8%   0+0k 0+0io 217pf+0w
     # load net into database
     cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
     netFilter -minGap=10 zfishdanRer2.net |  hgLoadNet hg17 netDanRer2 stdin
 # trackDb - change Zebrafish Net to danRer1 Net and display this track
 # for danRer2 as Zebrafish Net.
 # after chainAntiRepeat:
 # featureBits hg17 refGene:cds netDanRer2 -enrichment
 # refGene:cds 1.015%, netDanRer2 22.898%, both 0.783%, cover 77.15%, 
 # enrich 3.37x
 
     # index had NULL cardinality, analyze table to fix (2005-1-18, Heather)
     hgsql hg17
     analyze table netDanRer2
 
 # LOAD ACEMBLY TRACK (DONE, 2005-01-24, hartera)
 # ACEMBLY TABLE RELOADED AND FINISHED COLOR CODING CODE IN 
 # hgTracks (2005-01-28, hartera)
 # FINISHED CODE FOR FILTERING BY GENE CLASS (2005-02-03, hartera)
     mkdir -p /cluster/data/hg17/bed/acembly
     cd /cluster/data/hg17/bed/acembly
     # Data is obtained from 
     # Danielle et Jean Thierry-Mieg     mieg@ncbi.nlm.nih.gov
  
     wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35.human.genes/acembly.ncbi_35.genes.proteins.fasta.tar.gz
     wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35.human.genes/acembly.ncbi_35.genes.gff.tar.gz
     wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35.human.genes/acembly.ncbi_35.mrnas.fasta.tar.gz
     wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35.human.genes/acembly.ncbi_35.genes.pfamhits.tar.gz
     tar xvzf acembly.ncbi_35.genes.gff.tar.gz
     tar xvzf acembly.ncbi_35.genes.proteins.fasta.tar.gz
     cd acembly.ncbi_35.genes.gff
     # the acembly dataset for hg16 had problems with reverse blocks so
     # check for these
 cat << '_EOF_' > checkReversedBlocks
 for i in x1*.gff
 do
         echo -n "$i working ..."
         awk -F"\t" '
 {
 if ($4 > $5) {
     printf "reverse blocks problem for $1"
         printf "\n"
 }
 }
 ' $i > $i.fixed
         echo " done"
 done
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod +x checkReversedBlocks
     ./checkReversedBlocks  
     ls -l *.fixed
     # all *.fixed files are empty so remove - there is no reversing of blocks
     rm *.fixed
 
     foreach f (x1.acemblygenes.*.gff)
       set c=$f:r:e
       egrep '^[a-zA-Z0-9]+\|NT_[0-9][0-9][0-9][0-9][0-9][0-9]' $f | \
         perl -wpe 's/^(\w+)\|(\w+)/$1\/$2/' > ctg-chr${c}_random.gff
       if (-e ../../../$c/lift/random.lft) then
         liftUp chr${c}_random.gff ../../../$c/lift/random.lft warn \
           ctg-chr${c}_random.gff
       endif
       grep -v ^$c\| $f | grep -v ^Hs | perl -wpe 's/^/chr/;' | \
         grep -v "^chr//" > chr$c.gff
       echo "done $c"
     end
     
     #- Load into database - use extended genePred
     ssh hgwdev
     cd /cluster/data/hg17/bed/acembly
     # Reloaded without -genePredExt 1/6/05:
     ldHgGene -gtf hg17 acembly acembly.ncbi_35.genes.gff/chr*.gff
     # for entry with 28212470 from chr6.gff, change to chr6 
     # and for 29124352 in chr6.gff, change to chr6 (1/13/05)
     echo 'update acembly set chrom = "chr6" where chrom = "chr28212470";' \
          | hgsql hg17 
     echo 'update acembly set chrom = "chr6" where chrom = "chr29124352";' \
          | hgsql hg17 
     
     # checkTableCoords and runGeneCheck to check data
     # a number of errors so leave on hgwdev for the moment
 # checkTableCoords:
 # rah.acembly has 16 records with chrom not described in chromInfo.
 # rah.acembly item RPL10A.sNov04 chr6:35544172-35546520: end of last block (35546519) is not the same as chromEnd (35546520).
 # rah.acembly has 1 records with blockEnd[n-1] != end.
 # rah.acembly has 1 records with end > chromSize.
 # chr6    acembly exon    35545934        35546101        .       +       0  
 # gene_id RPL10A; transcript_id RPL10A.sNov04; exon_number 5
 # chr6    acembly intron  35546102        35546520        .       +       0
 # gene_id RPL10A; transcript_id RPL10A.sNov04; intron_type fuzzy
 # chr6    acembly CDS     35546335        35546384        .       +       0
 # gene_id RPL10A; transcript_id RPL10A.sNov04; exon_number 6
 # chr6    acembly exon    35546335        35546519        .       +       0
 # gene_id RPL10A; transcript_id RPL10A.sNov04; exon_number 6
 # chr6    acembly stop_codon      35546382        35546384        .       +
 # 0       gene_id RPL10A; transcript_id RPL10A.sNov04;
 # here the intron overlaps exon 6 so take 35546519 to be txEnd
       echo 'update acembly set txEnd = 35546519 where name = "RPL10A.sNov04";'\
            | hgsql hg17
 # for record where end > chromSize
       echo 'select * from acembly as a, chromInfo as c where c.chrom = a.chrom \            and c.size < a.txEnd;' | hgsql hg17
       # KIR2DL5.bNov04 on chr19_random, chr19_random size is 301858, 
       # txEnd is 305266 delete this record
       echo 'delete from acembly where name = "KIR2DL5.bNov04";' | hgsql hg17
 # from runGeneCheck:
 # 5780   inFrameStop
 # 110664 noStart
 # 23085 badCdsSplice
 # 23848 noStop
 # 14957 badUtrSplice
 # 3661 gap
 # 4726 badFrame
 # 261066 lines in genePred.tab
 
     # e-mailed authors of data (2004-12-21, hartera) 
 #  notiri.aNov04 - has ctg instead of atg at start. others have no start specified: sirora.nNov04
 # sirora.zmNov04 - chr1:19389-19392 is AAC (gtt) (-)
 # sirora.sNov04 - chr1:8925-8928 CAA (ttg) (-)
 # sirora.rNov04 - chr1:8925-8928 CAA (ttg) (-)
     # for entries with 28212470 and 29124352 instead of chr6 change to chr6
     # Re-process this x1 file to chr6.gff (2005-01-24)
     mv x1.acemblygenes.6.gff x1.acemblygenes.6.gff.broken
     sed -e "s/^28212470/6/" x1.acemblygenes.6.gff.broken | sed -e \
            "s/^29124352/6/"  > x1.acemblygenes.6.gff
     grep -v ^6\| x1.acemblygenes.6.gff | grep -v ^Hs | perl -wpe 's/^/chr/;' | \
         grep -v "^chr//" > chr6.gff
     # Received a list of genes from Jean and Danielle Mieg
     # showing genes that are "main", "putative" or "cloud" - there should be
     # no "cloud" genes in our data set (2005-01-11)
     # download acembly_gene_lists.tar.gz from e-mail
     cd /cluster/data/hg17/bed/acembly
     tar xvzf acembly_gene_lists.tar.gz
     cd acembly_gene_lists
 cat << '_EOF_' > getIDs.pl
 #!/usr/bin/perl -w
 use strict;
 
 while (<STDIN>) {
    my @f = split(/\s+/);
    for (my $i =0; $i <= $#f; $i++) {
        if ($f[$i] =~ /gene_id$/) {
           # if field is ID type then next value is the ID
           my $id = $f[$i+1];
           # remove ";" at end and print ID
           chop $id;
           print "$id\n";
        }
    }
 }
 '_EOF_'
     chmod +x getIDs.pl
     # get gene IDs from gff files
     foreach f (../acembly.ncbi_35.genes.gff/chr*.gff)
        echo "Processing $f"
        perl getIDs.pl < $f >> genesGffs.ids
     end
     # remove back slash from some names
     sort genesGffs.ids | uniq > genesGffs.ids.uniq
     # reformat gene list to get just the genes and remove first 2 lines and sort
     foreach g (*.list)
        sed -e 's/"//g;' $g | sed -e 's/Gene : //;' | sed -e '1,2d' \
               | sort | uniq > $g.IDsort
     end 
     # remove back slash from some names
     perl -pi.bak -e 's/\\//' *.IDsort
     # check if cloud genes appear in gff files list of genes
     # list of genes in cloud but not in gff
     comm -13 genesGffs.ids.uniq cloud_gene.list.IDsort > gffvscloud.out
     diff gffvscloud.out cloud_gene.list.IDsort
     # there is no difference so none of the cloud genes are in the gff files
     # check if all the other genes in the main and putative lists are in gffs
     comm -13 genesGffs.ids.uniq main_gene.list.IDsort > gffvsmain.out
     comm -13 genesGffs.ids.uniq putative_gene.list.IDsort > gffvsputative.out
     wc -l *.out
     # 14 gffvsmain.out
     # 0 gffvsputative.out
 
     # there are 14 genes in the main set not in the gff files
     # actually there are 12, as FCA/MR and SLA/LP are in the gff files
     # all putative genes are in the gff set
     wc -l main_gene.list.IDsort putative_gene.list.IDsort
     # 52467 main_gene.list.IDsort
     # 43978 putative_gene.list.IDsort
     # 96445 total
     wc -l genesGffs.ids.uniq    
     # 97042 genesGffs.ids.uniq
     # check discrepancy
     cat main_gene.list.IDsort putative_gene.list.IDsort > mp.ids
     sort mp.ids > mp.sort
     comm -23 genesGffs.ids.uniq mp.sort > gffNotMP.out
     wc -l gffNotMP.out
     # 609 gffNotMP.out
 # create table of Acembly gene classifications
 # see http://www.ncbi.nlm.nih.gov/IEB/Research/Acembly/index.html?human
 # in FAQ, describes main, putative and cloud genes. The cloud genes are not 
 # well confirmed and so they are not in this data set. 
     # NEED TO FILTER GENES AND RELOAD TABLES:
     # authors Jean and Danielle Mieg e-mailed back. The 12 genes in the 
     # putative list that are not in the gff files were not exported
     # as they did not find a single putative protein to describe so they 
     # were not added to the gffs. They will be added at a later date.
     # Remove these from the acemblyClass table (2005-01-21, hartera)
     # Reload acemblyClass table as problems with the gene names
     # the class table has gene IDs and the acembly table has transcript IDs
     # it is hard to look up class in the class table since just removing the 
     # transcript ID suffixes (e.g. "aNov04" after a ".") does not work as 
     # some gene IDs have a "." in them anyway.
 
     ssh kksilo
     cd /cluster/data/hg17/bed/acembly/acembly_gene_lists
     comm -13 gffvsmain.out main_gene.list.IDsort > main_gene.list.filt
     wc -l main_gene.list.filt
     # 52455 main_gene.list.filt
     ssh hgwdev
     cd /cluster/data/hg17/bed/acembly/acembly_gene_lists
     # drop acemblyClass table and recreate (2005-01-27, hartera)
     echo 'drop table acemblyClass;' | hgsql hg17
     # prepare a file of genes and classification
     # use transcript IDs - get these and corresponding gene IDs from gff files
     # if Gene IDs used the difficult to parse transcript ID (name column) from 
     # the acembly genePred table. e.g. notiri.aNov04 is a transcript ID so can
     # remove suffix after "." to obtain gene ID, some gene names have "." in
     # them and not all have the suffix.
     # 260446 transcript IDs (use allFiltered.gff - see below) 
     perl getClass.pl main_gene.list.filt putative_gene.list.IDsort \
          ../acembly.ncbi_35.genes.gff/allFiltered.gff 
     foreach f (main_gene.list.filt putative_gene.list.IDsort)
        if ($f == "main_gene.list.filt") then
           set t = "main"
        endif
        if ($f == "putative_gene.list.IDsort") then
           set t = "putative"
        endif
        awk 'BEGIN {OFS="\t"} {print $1, "'$t'"}' $f >> class.txt
     end
     sort classes.txt | uniq > geneIDtxID.class
     # get transcript ID and class fields for acemblyClass table
     awk 'BEGIN {OFS="\t"} {print $2,$3}' geneIDtxID.class > acemblyClass.tab
     wc -l acemblyClass.tab
     # 260446 acemblyClass.tab
     # make change to acemblyClass.as and check in:
     # change name to be transcript ID instead of gene ID
 cat << '_EOF_' > $HOME/kent/src/hg/lib/acemblyClass.as
 table acemblyClass
 "Class for Acembly genes"
    (
     string name; "Transcript ID for Acembly gene"
     string class; "Class of gene"   
    )
 '_EOF_'
     cd $HOME/kent/src/hg/lib/
     autoSql acemblyClass.as acemblyClass
     mv acemblyClass.h $HOME/kent/src/hg/inc
     # do make to check it works and commit the .as, .sql, .c and .h files to CVS
     cd /cluster/data/hg17/bed/acembly/acembly_gene_lists
     echo "drop table acemblyClass" | hgsql hg17
     hgsql hg17 < ~/kent/src/hg/lib/acemblyClass.sql
     # reload table with transcript IDs 
     echo "load data local infile 'acemblyClass.tab' into table acemblyClass" \
          | hgsql hg17
     
     # There were also 609 genes in the gff files that are not in the
     # main, putative or cloud gene lists. Jean and Danielle Mieg say that
     # these were filtered out from their data set but not from the gff files.
     # Remove these from the gff files. (gffNotMP.out) (2005-01-24)
     cd /cluster/data/hg17/bed/acembly/acembly.ncbi_35.genes.gff
     cat chr*.gff > all.gff
 cat << '_EOF_' > removeGenes.pl
 #!/usr/bin/perl -w
 use strict;
                                                                                 
 my $genes = $ARGV[0];
 my $gff = $ARGV[1];
                                                                                 
 open(GENES, $genes) || die "Can not open $genes:$!\n";
 open(GFF, $gff) || die "Can not open $gff:$!\n";
 open(OUT, ">removed.out") || die "Can not open removed.out:$!\n";
                                                                                 
 my %genes;
                                                                                 
 while (<GENES>) {
    chomp;
    my $g = $_;
    $genes{$g} = 1;
 }
 close GENES;
                                                                                 
 while (<GFF>) {
    my $l = $_;
    my $id;
    my @line = split(/\s+/);
    for (my $i = 0; $i <= $#line; $i++) {
       if ($line[$i] eq "gene_id") {
          $id = $line[$i+1];
       }
    }
    $id =~ s/;//;
    print "id is now $id\n";
    if (!exists($genes{$id})) {
       print $l;
    }
    else {
       print OUT $l;
    }
 }
 '_EOF_'
     perl removeGenes.pl ../acembly_gene_lists/gffNotMP.out all.gff \
                         > allFiltered.gff 
     # checked that gene IDs in the removed.out file are the same
     # same as those in gffNotMP.out
     # reload into the acembly table
     ssh hgwdev
     cd /cluster/data/hg17/bed/acembly
     echo 'drop table acembly;' | hgsql hg17
     # Reloaded with filtered set 2005-01-23, reload again 2005-01-28 with
     # the genePredExt option to get gene ID in name 2 field
     ldHgGene -gtf -genePredExt hg17 acembly \
              acembly.ncbi_35.genes.gff/allFiltered.gff
     # Read 260446 transcripts in 3656676 lines in 1 files
     #  260446 groups 41 seqs 1 sources 5 feature types
     #  260446 gene predictions
     # remove cdsStartStat, cdsEndStat and exonFrames fields
     echo 'alter table acembly drop column cdsStartStat;' | hgsql hg17
     echo 'alter table acembly drop column cdsEndStat;' | hgsql hg17
     echo 'alter table acembly drop column exonFrames;' | hgsql hg17
 
     # fix problem data found by checkTableCoords
     # here the intron overlaps exon 6 so take 35546519 to be txEnd
     echo 'update acembly set txEnd = 35546519 where name = "RPL10A.sNov04";'\
            | hgsql hg17
     # for record where end > chromSize
     echo 'select * from acembly as a, chromInfo as c where c.chrom = a.chrom \            and c.size < a.txEnd;' | hgsql hg17
     # KIR2DL5.bNov04 on chr19_random, size is 301858, txEnd is 305266
     # delete this record
     echo 'delete from acembly where name = "KIR2DL5.bNov04";' | hgsql hg17
     # acembly peptide table
     # need to just grab same sequences that are in acembly
     cd ./acembly.ncbi_35.genes.proteins.fasta
     echo 'select name from acembly;' | hgsql -N hg17 > acembly.name
     cat *.fasta > allPep.fa
     faSomeRecords allPep.fa acembly.name acemblyPep.fa
     # PEPTIDE SEQUENCES NOT LOADED
     # There are 236,554 peptide names that do not match transcript IDs in
     # the acembly table and 110,278 transcript IDs in acembly that do not
     # have a corresponding peptide. Waiting for repsonse about this from
     # Jean and Danielle (2005-01-31)  
     # hgPepPred hg17 generic acemblyPep \
     #         acembly.ncbi_35.genes.proteins.fasta/*.fasta
 
     # Edit hgTracks.c to get colour coded tracks based on the gene class
     # for each gene as read from the acemblyClass table.
     # Edits to hui.c, hgTrackUi.c and hgTracks.c to allow filtering of 
     # genes based on class.
 # acembly trackDb entry:
 # track acembly
 # shortLabel Acembly Genes
 # longLabel AceView Gene Models With Alt-Splicing
 # group genes
 # priority 41
 # visibility dense
 # color 155,0,125
 # type genePred acemblyPep acemblyMrna
 # url http://www.ncbi.nih.gov/IEB/Research/Acembly/av.cgi?db=hg17&l=$$
 # itemClassTbl acemblyClass
 # geneClasses main putative
 # gClass_main 128,0,125
 # gClass_putative 200,0,125
 # urlLabel Transcript ID:
 
 # search added:
 # searchTable acembly
 # searchType genePred
 # searchMethod prefix
 # termRegex [^[:space:]]+
 # searchPriority 50
 
     # Received data with gene product relationship from Jean Thierry-Mieg
     # (2005-02-17)
     ssh eieio
     cd /cluster/data/hg17/bed/acembly/acembly.ncbi_35.genes.proteins.fasta
     wget --timestamp ftp://ftp.ncbi.nlm.nih.gov/repository/acedb/human/acembly.ncbi_35.gene2product.txt.gz
     gunzip acembly.ncbi_35.gene2product.txt.gz
     # these are gene ID and product mappings, need transcript ID to product
     # mappings. E-mailed Jean Thierry-Mieg to ask for this information
     
 # BUILD WGRNA TRACK (DONE, 2004-12-13, Fan)
 
 # Grab data from original miRNA track and convert them into wgRna .tab format.
     hgsql hg17 --skip-column-names -e 'select * from miRNA' >miRNA.out
     cat miRNA.out  | awk {'print $2"\t"$3"\t"$4"\t"$5"\t"$6"\t"$7"\t"$8"\t"$9"\t""miRna"'} >wgRna.tab
 
 # Break the original custom track data file, hsa-snoRNA_track.txt, into two files j1 and j2, 
 # then remove header and blank lines.
     cat j1 | awk {'print $1"\t"$2"\t"$3"\t"$4"\t"$5"\t"$6"\t"0"\t"0"\t""CDBox"'} >>wgRna.tab
     cat j2 | awk {'print $1"\t"$2"\t"$3"\t"$4"\t"$5"\t"$6"\t"0"\t"0"\t""HAcaBox"'} >>wgRna.tab
 
 # load into wgRna table
     hgLoadBed -sqlTable=/cluster/home/fanhsu/hg/lib/wgRna.sql hg17 wgRna wgRna.tab
 # create and edit wgRna.html under src/hg/makeDb/trackDb/human/hg17.
 
 # RELOADED wgRna DATA USING wgRNA_corrected.txt SENT BY MICHEL WEBER
 # Manually removed the first header line and the first column of the bin field and removed 
 # the last empty line.
 
     cut -f 2- wgRNA_corrected.txt >wgRna.tab
     vi wgRna.tab
     hgLoadBed -sqlTable=/cluster/home/fanhsu/hg/lib/wgRna.sql hg17 wgRna wgRna.tab
 
 
 # UPDATED WGRNA DATA PER EMAIL FROM WEBER (2004-12-14, Fan).
 
 # Added the following 3 lines to j1
 chr3 161715396 161715726 U90 480 -
 chr11 93104316 93104387 Z40 480 -
 chr11 93106041 93106114 mgh28S-2410 480 -
     
 # Regenerated wgRna table
     cat miRNA.out  | awk {'print $2"\t"$3"\t"$4"\t"$5"\t"$6"\t"$7"\t"$8"\t"$9"\t""miRna"'} >wgRna.tab
     cat j1 | awk {'print $1"\t"$2"\t"$3"\t"$4"\t"$5"\t"$6"\t"0"\t"0"\t""CDBox"'} >>wgRna.tab
     cat j2 | awk {'print $1"\t"$2"\t"$3"\t"$4"\t"$5"\t"$6"\t"0"\t"0"\t""HAcaBox"'} >>wgRna.tab
     hgLoadBed -sqlTable=/cluster/home/fanhsu/hg/lib/wgRna.sql hg17 wgRna wgRna.tab
 
 # Changed the following records to RNA type scaRna.
 
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U85"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U87"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U88"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U89"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U90"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U91"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U92"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U93"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U100"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA26"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA35"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA45"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA47"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA57"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="HBII-382"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="mgU2-19/30"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="mgU2-25/61"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="mgU2-22/U4-8"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U85"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U87"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U88"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U89"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U90"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U91"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U92"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U93"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U100"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA26"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA35"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA45"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA47"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA57"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="HBII-382"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="mgU2-19/30"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="mgU2-25/61"'
 hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="mgU2-22/U4-8"'
 
 # Updated .../trackDb/human/hg17/wgRna.html.
 
 # MAKE VSDANRER2 DOWNLOADABLES (DONE, 2004-12-14, hartera)
 # REMAKE FOR CHAINS AND NET AFTER USING chainAntiRepeat
 # (DONE, 2004-12-22, hartera)
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.danRer2/axtChrom
     set gp = /usr/local/apache/htdocs/goldenPath/hg17
     mkdir -p $gp/vsDanRer2/axtChrom
     cp -p *.axt $gp/vsDanRer2/axtChrom
     cd $gp/vsDanRer2/axtChrom
     gzip *.axt
     md5sum *.gz > md5sum.txt
                                                                                 
     # copy chains and nets to downloads area
     # re-make chains and net downloadables (2004-12-22, hartera)
     rm $gp/vsDanRer2/zebrafish*.gz $gp/vsDanRer2/md5sum.txt
     cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
     gzip -c all.chain.antirepeat > \
             /cluster/data/hg17/zip/zebrafishDanRer2.chain.gz
     gzip -c zfishdanRer2.net > /cluster/data/hg17/zip/zebrafishDanRer2.net.gz
     cd $gp/vsDanRer2
     mv /cluster/data/hg17/zip/zebrafish*.gz .
     md5sum *.gz > md5sum.txt
     # Copy over & edit README.txt w/pointers to chain, net formats.
                                                                                 
 # CLEANUP DANRER2 BLASTZ (DONE, 2004-12-14, hartera)
 # RE-DONE (DONE, 2004-12-22, hartera)
 # REMOVED RAW AND LAV DIRS (DONE, 2005-02-24, hartera)
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.danRer2
     nice rm axtChain/run1/chain/* &
     nice rm -fr axtChain/n1 axtChain/noClass.net &
     nice gzip axtChrom/* pslChrom/* axtChain/all.chain axtChain/all.chain.unfiltered axtChain/*.net &
     nice gzip axtChain/all.chain.antirepeat axtChain/chainAR/*.chain &
     nice rm -fr axtChain/chain axtChain/chainRaw axtChain/preNet &
     nice rm -rf raw &
     nice rm -rf lav &
 
 # EXTRACT AXT'S AND MAF'S FROM TETRAODON (tetNig1) NET 
 # (DONE, 2004-12-15, hartera) 
 # Redo to remove overlaps (2006-04-07 kate)
     ssh eieio
     # create axts
     cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
     netSplit tetNig1.net tetraodonNet
     mkdir -p ../axtNet
 cat > axtNet.csh << 'EOF'
     foreach f (tetraodonNet/chr*.net)
         set c = $f:t:r
         echo "axtNet on $c"
         netToAxt tetraodonNet/$c.net chain/$c.chain \
         /cluster/data/hg17/nib /cluster/data/tetNig1/nib ../axtNet/$c.axt
         echo "Complete: $c.net -> $c.axt"
         end
 'EOF'
 
     chmod +x axtNet.csh
     csh axtNet.csh >&! axtNet.log &
     tail -100f axtNet.log
 
     # sort axts before making mafs - must be sorted for multiz
     cd /cluster/data/hg17/bed/blastz.tetNig1
     mv axtNet axtNet.unsorted
     mkdir axtNet
     foreach f (axtNet.unsorted/*.axt)
         set c = $f:t:r
         echo "Sorting $c"
         axtSort $f axtNet/$c.axt
     end
 
     # create maf
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.tetNig1
     cd axtNet
     mkdir ../mafNet
 
 cat > makeMaf.csh << 'EOF'
     foreach f (chr*.axt)
       set maf = $f:t:r.tetNig1.maf
       echo translating $f to $maf
       axtToMaf $f \
             /cluster/data/hg17/chrom.sizes /cluster/data/tetNig1/chrom.sizes \
             ../mafNet/$maf -tPrefix=hg17. -qPrefix=tetNig1.
     end
 'EOF'
     csh makeMaf.csh >&! makeMaf.log &
     tail -100f makeMaf.log
 
     nice gzip axtChrom/* pslChrom/* axtChain/all.chain axtChain/*.net &
 
     # redo axt's and maf's to remove overlaps (2006-04-07 kate)
     ssh kkstore02
     cd /cluster/data/hg17/bed/blastz.tetNig1
     mv axtNet axtNet.old
     mv mafNet mafNet.old
     mkdir -p axtNet mafNet
     cd axtChain
 
 cat > fix.csh << 'EOF'
     date
     foreach f (tetraodonNet/chr*.net)
         set c = $f:t:r
         echo $c
         netToAxt tetraodonNet/$c.net chain/$c.chain \
             /cluster/data/hg17/nib /cluster/data/tetNig1/nib stdout | \
                 axtSort stdin ../axtNet/$c.axt
         echo "Complete: $c.net -> $c.axt"
         axtToMaf ../axtNet/$c.axt \
             /cluster/data/hg17/chrom.sizes /cluster/data/tetNig1/chrom.sizes \
             ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=tetNig1.
     end
     date
 'EOF'
     csh fix.csh >&! fix.log &
     cd /san/sanvol1/scratch/hg17/mafNet
     rm -fr tetNig1
     cp -rp /cluster/data/hg17/bed/blastz.tetNig1/mafNet tetNig1
 
 
 # 10-WAY MULTIZ -- 8-WAY PLUS FROG AND TETRA (DONE 2004-12-22 kate)
 #       Use older multiz (not v10) till bugs fixed
 
     ssh eieio
     cd /cluster/data/hg17/bed
     rm multiz10way
     mkdir multiz.2004-12-22
     ln -s multiz.2004-12-22 multiz10way
     cd multiz10way
 
     cat > tree.nh << 'EOF'
 ((((((hg17,panTro1),(rn3,mm5)),canFam1),galGal2),xenTro1),((fr1,tetNig1),danRer1))
 'EOF'
 
     mkdir /cluster/bluearc/hg17/multiz.2004-12-22
     cd /cluster/bluearc/hg17
     mkdir 2004-12-22
     rm multiz10way
     ln -s multiz.2004-12-17 multiz10way.v10
     ln -s multiz.2004-12-22 multiz10way
     # reuse pairwise MAF's on bluearc
     mv multiz10way.v10/{canFam1,danRer1,fr1,galGal2,mm5,panTro1,rn3,tetNig1,xenTro1} multiz10way
     # NOTE: pairwise mafs were moved to /cluster/bluearc/hg17/mafNet
 
     # make output dir and run dir
     ssh kk9
     cd /cluster/data/hg17/bed
     cd multiz10way
     mkdir -p maf
     mkdir -p run
     cd run
 
     # create scripts to run multiz on cluster
 cat > oneMultiz.csh << 'EOF'
 #!/bin/csh -fe
     set c = $1
     set multi = /scratch/$user/multiz10way.$c
     set pairs = /cluster/bluearc/hg17/multiz10way
 
     # special mode --
     # with 1 arg, cleanup
     if ($#argv == 1) then
         rm -fr $multi
         exit
     endif
 
     set s1 = $2
     set s2 = $3
 
     # locate input files -- in pairwise dir, or multiple dir
     set d1 = $multi
     set d2 = $multi
     if (-d $pairs/$s1) then
         set d1 = $pairs
     endif
     if (-d $pairs/$s2) then
         set d2 = $pairs
     endif
     set f1 = $d1/$s1/$c.maf
     set f2 = $d2/$s2/$c.maf
     # write to output dir
     set out = $multi/${s2}${s1}
     mkdir -p $out
 
     # check for empty input file
     if (-s $f1 && -s $f2) then
         echo "Aligning $f1 $f2"
         /cluster/bin/penn/multiz $f1 $f2 - > $out/$c.maf
     else if (-s $f1) then
         cp $f1 $out
     else if (-s $f2) then
         cp $f2 $out
     endif
 'EOF'
 # << for emacs
 chmod +x oneMultiz.csh
 
 cat > allMultiz.csh << 'EOF'
 #!/bin/csh -fe
 
 set c = $1
 oneMultiz.csh $c mm5 panTro1
 oneMultiz.csh $c rn3 panTro1mm5 
 oneMultiz.csh $c canFam1 panTro1mm5rn3 
 oneMultiz.csh $c galGal2 panTro1mm5rn3canFam1
 oneMultiz.csh $c xenTro1 panTro1mm5rn3canFam1galGal2
 oneMultiz.csh $c fr1 panTro1mm5rn3canFam1galGal2xenTro1
 oneMultiz.csh $c tetNig1 panTro1mm5rn3canFam1galGal2xenTro1fr1
 oneMultiz.csh $c danRer1 panTro1mm5rn3canFam1galGal2xenTro1fr1tetNig1
 # get final alignment file
 cp /scratch/$user/multiz10way.$c/panTro1mm5rn3canFam1galGal2xenTro1fr1tetNig1danRer1/$c.maf /cluster/data/hg17/bed/multiz10way/maf/$c.maf
 #cleanup
 oneMultiz.csh $c
 'EOF'
 # << for emacs
 chmod +x allMultiz.csh
 
 cat > gsub << 'EOF'
 #LOOP
 allMultiz.csh $(root1) {check out line+ /cluster/data/hg17/bed/multiz10way/maf/$(root1).maf}
 #ENDLOOP
 'EOF'
 # << for emacs
     cut -f 1 /cluster/data/hg17/chrom.sizes > chrom.lst
     gensub2 chrom.lst single gsub jobList
     para create jobList
     para try; para check
     para push
 
     # post-process multiz maf with maf_project to "glue" short
     # alignment blocks together
     ssh eieio
     cd /cluster/data/hg17/bed/multiz10way.v8
     mkdir -p mafGlued
     cd maf
     foreach f (*.maf)
         set c = $f:r
         echo "gluing $f"
         /cluster/bin/penn/maf_project $f hg17.$c > ../mafGlued/$c.maf
     end
     # filter out alignment blocks with no alignments in non-reference species,
     # and low-scoring alignments based on Webb Miller's latest
     # recommendations (score < -5 * ncol^2 * nrow)
     # NOTE: Webb hasn't approved the filtered alignments yet,
     # so leaving them in for now.
     #mkdir -p mafFiltered
     #cd ../mafGlued
     #foreach f (*.maf)
         #set c = $f:r
         #echo "filtering $f"
         #~kate/bin/i386/mafFilter -factor $f > ../mafFiltered/$c.maf
     #end
     #cd ..
 
     grep score mafGlued/chr1.maf | wc -l
     grep score mafFiltered/chr1.maf | wc -l
     grep score mafGlued/bad | wc -l
         # 43692
     grep score=0.0 bad | wc -l
         # 10206
     # load alignments into tables
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz10way.v8
     set mafDir = /gbdb/hg17/mafNet
     mkdir -p $mafDir
 
     # multiple alignment
     set mafDir = /gbdb/hg17/multiz10way/maf
     mkdir -p $mafDir/multiz10way
     cd /cluster/data/hg17/bed/multiz10way.v8/mafGlued
     ln -s `pwd`/*.maf $mafDir/multiz10way
     hgLoadMaf hg17 -warn multiz10way -pathPrefix=$mafDir/multiz10way
 
     # load summary table to replace pairwise
     cd /cluster/data/hg17/bed/multiz10way.v8/mafGlued/
     time cat chr*.maf | hgLoadMafSummary -minSize=10000 -mergeGap=500 -maxSize=50000 hg17 multiz10waySummary stdin
     # Processed 27314693 components in 9081437 mafs from stdin
     # 30 minutes
 
 
 # CONSERVATION SCORING WITH PHASTCONS (DONE 2005-01-14 kate)
 # 1. Partition multiple alignment into windows, using "msa_split"
 # 2. Create starting tree model, with branch lengths
 #       use "phyloFit" on alignments
 # 3. Estimate GC avg. over all species, use "msa_view" on maf
 # 4. Estimate other model params, using phastCons (via doEstimate script)
 
     # NOTE: no alignment filtering done -- the scores don't look
     # particularly meaningful w/ this version of multiz.
     # Next time, run on "glued" (maf_projected) 
 
     ssh eieio
     cd /cluster/data/hg17/bed/multiz10way.v8
     set mafDir = /cluster/bluearc/hg17/multiz10way.v8/maf
     mkdir -p $mafDir
     cp -r maf/*.maf $mafDir
 
     ssh kk9
     cd /cluster/data/hg17/bed/multiz10way.v8
     mkdir cons
     cd cons
 
     # break up the genome-wide MAFs into pieces
     # NOTE: chrom fasta files are already on the bluearc
     # from previous run
     mkdir /cluster/bluearc/hg17/chrom
     cd /cluster/data/hg17
     foreach f (`cat chrom.lst`)
         echo $f
         cp -r $f/*.fa /cluster/bluearc/hg17/chrom
     end
 
     cd /cluster/data/hg17/bed/multiz10way.v8/cons
     mkdir run.split
     cd run.split
     set WINDOWS = /cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS
     rm -fr $WINDOWS
     mkdir -p $WINDOWS
     cat << 'EOF' > doSplit.sh
 #!/bin/sh
 
 PHAST=/cluster/bin/phast
 FA_SRC=/cluster/bluearc/hg17/chrom
 WINDOWS=/cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS
 
 maf=$1
 c=`basename $maf .maf`
 echo $c
 mkdir -p /scratch/msa_split
 ${PHAST}/msa_split $maf -i MAF -M ${FA_SRC}/$c.fa -O hg17,panTro1,mm5,rn3,canFam1,galGal2,xenTro1,fr1,tetNig1,danRer1 -w 1000000,0 -r /scratch/msa_split/$c -o SS -I 1000 -B 5000
 [ $? -eq 0 ] || exit 1
 echo "Copying..."
 cd /scratch/msa_split
 for file in $c.*.ss ; do gzip -c $file > ${WINDOWS}/$file.gz ; done
 [ $? -eq 0 ] || exit 1
 rm -f /scratch/msa_split/$c.*.ss
 echo "Done copying"
 echo "Done" >> ${WINDOWS}/$c.done
 'EOF'
 # << for emacs
 
     set WINDOWS = /cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS
     chmod +x doSplit.sh
     rm -f jobList
     foreach file (/cluster/bluearc/hg17/multiz10way.v8/maf/*.maf) 
         set c = $file:t:r
 	echo "doSplit.sh $file {check out line+ $WINDOWS/$c.done}" >> jobList
     end
     
     para create jobList
         # 46 jobs
     para try
     para check
 
     # TODO: cleanup
     # rm -fr $mafDir
     
     # now generate conservation scores and predicted elements
     set path = ($path /cluster/bin/phast); rehash
     cd /cluster/data/hg17/bed/multiz10way.v8/cons
     mkdir run.elements   
     cd run.elements
 
     # create a starting tree model from a chr1 ss files in WINDOWS dir.
     ssh kolossus
     cd /cluster/data/hg17/bed/multiz10way.v8/cons/run.elements
     gunzip -c /cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS/chr1.14996059-15998256.ss.gz \
                 > /tmp/phastCons.$$
      phyloFit -i SS /tmp/phastCons.$$ --out-root starting-tree --tree \
         "((((((hg17,panTro1),(mm5,rn3)),canFam1),galGal2),xenTro1),((fr1,tetNig1),danRer1))" 
     rm /tmp/phastCons.$$
     cat starting-tree.mod
 #ALPHABET: A C G T 
 #ORDER: 0
 #SUBST_MOD: REV
 #TRAINING_LNL: -2635749.517410
 #BACKGROUND: 0.247225 0.248374 0.250827 0.253574 
 #RATE_MAT:
 #-0.997890    0.201447    0.648573    0.147870 
 #0.200515   -1.020796    0.190184    0.630096 
 #0.639258    0.188324   -1.025170    0.197587 
 #0.144168    0.617176    0.195447   -0.956791 
 #TREE: ((((((hg17:0.006401,panTro1:0.008342):0.099376,(mm5:0.083404,rn3:0.105411):0.242694):0.020883,canFam1:0.221922):0.099131,galGal2:0.275759):0.041997,xenTro1:0.280306):0.064815,((fr1:0.137674,tetNig1:0.091463):0.118573,danRer1:0.250847):0.064815);
 
     # estimate model parameters
     # estimate avg. cross-species avg. GC content from chr1 maf's
     ssh kolossus
     set path = ($path /cluster/bin/phast); rehash
     cd /cluster/data/hg17/bed/multiz10way.v8/cons/run.elements
     msa_view --aggregate hg17,panTro1,rn3,mm5,canFam1,galGal2,xenTro1,danRer1,tetNig1,fr1 \
         -i MAF \
         --summary-only /cluster/data/hg17/bed/multiz10way.v8/maf/chr1.maf\
             > maf_summary.txt
     awk '$1 == "[aggregate]" {printf "%0.3f\n", $3 + $4}' maf_summary.txt
        # 0.424
 
     # generate models from random sample of genome (use 90 1Mb windows,
     # to conveniently run on rack 9 100-node cluster)
     # On first pass, used parameters from 8way alignment:
     #   expected-lengths 12 -taret-coverage .17
     # NOTE: there may be a cleverer way to select the first length param
     # On second pass, used parameters below, based on consEntropy
     # and featureBits coverage of elements, below
     cat << 'EOF' > doEstimate.sh
 #!/bin/sh
 zcat $1 | /cluster/bin/phast/phastCons - starting-tree.mod --gc 0.424 --nrates 1,1 --no-post-probs --ignore-missing --expected-lengths 11 --target-coverage 0.20 --quiet --log $2 --estimate-trees $3
 'EOF'
     chmod u+x doEstimate.sh
     rm -fr LOG TREES
     mkdir -p LOG TREES
     ls /cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS/*.gz > all.windows
     /cluster/bin/phast/chooseLines -k 90 all.windows > subset.windows
     rm -f jobs.lst
     foreach f (`cat subset.windows`)
         set root = $f:t:r:r
 	echo doEstimate.sh /cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS/$f LOG/$root.log TREES/$root >> jobs.lst
     end
     # run cluster job (about an hour)
     ssh kk9
     cd /cluster/data/hg17/bed/multiz10way.v8/cons/run.elements
     para create jobs.lst
        # 90 jobs written to batch
     para try; para check
     para push
 # 2 jobs crashed with out-of-mem; as we are just taking a sample
 # this is probably OK, but I've notified Adam
 # Average job time:                1055s      17.58m     0.29h    0.01d
 # Longest job:                     3647s      60.78m     1.01h    0.04d
 
     # NOTE: should have used ave.noncons.mod to improve parameter estimation
     # cp nave.noncons.mod starting-tree.mod
 
     ls TREES/*.cons.mod > cons.txt
     /cluster/bin/phast/phyloBoot --read-mods '*cons.txt' --output-average ave.cons.mod > cons_summary.txt
     grep TREE ave.cons.mod
 # TREE: ((((((hg17:0.002313,panTro1:0.002931):0.036375,(mm5:0.029849,rn3:0.039008):0.095334):0.003258,canFam1:0.078205):0.047189,galGal2:0.158045):0.020103,xenTro1:0.169387):0.028857,((fr1:0.071610,tetNig1:0.057766):0.091165,danRer1:0.138905):0.028857);
     ls TREES/*.noncons.mod > noncons.txt
     /cluster/bin/phast/phyloBoot --read-mods '*noncons.txt' --output-average ave.noncons.mod > noncons_summary.txt
     grep TREE ave.noncons.mod
 # TREE: ((((((hg17:0.007342,panTro1:0.009340):0.116009,(mm5:0.095037,rn3:0.124288):0.304355):0.010633,canFam1:0.249367):0.151476,galGal2:0.507037):0.064317,xenTro1:0.549121):0.094733,((fr1:0.231246,tetNig1:0.185161):0.296288,danRer1:0.446734):0.094733);
 
 
     # analyze conservation genome-wide
     cat << 'EOF' > doPhastCons.sh
 #!/bin/sh
 
 mkdir -p /cluster/bluearc/hg17/multiz10way.v8/phastCons/POSTPROBS /cluster/bluearc/hg17/multiz10way.v8/phastCons/ELEMENTS
 pref=`basename $1 .ss.gz`
 chr=`echo $pref | awk -F\. '{print $1}'`
 tmpfile=/scratch/phastCons.$$
 zcat $1 | /cluster/bin/phast/phastCons - ave.cons.mod,ave.noncons.mod --expected-lengths 11 --target-coverage 0.20 --quiet --seqname $chr --idpref $pref --viterbi /cluster/bluearc/hg17/multiz10way.v8/phastCons/ELEMENTS/$pref.bed --score --require-informative 0 > $tmpfile
 gzip -c $tmpfile > /cluster/bluearc/hg17/multiz10way.v8/phastCons/POSTPROBS/$pref.pp.gz
 rm $tmpfile
 'EOF'
     chmod u+x doPhastCons.sh
 
     rm -fr /cluster/bluearc/hg17/multiz10way.v8/phastCons/POSTPROBS /cluster/bluearc/hg17/multiz10way.v8/phastCons/ELEMENTS
     rm -f jobs2.lst
     foreach f (/cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS/*.ss.gz)
         echo doPhastCons.sh $f >> jobs2.lst
     end
 
     # run cluster job (it's quick -- 10 minutes or so)
     ssh kk
     cd /cluster/data/hg17/bed/multiz10way.v8/cons/run.elements
     para create jobs2.lst
         # 2932 jobs written to batch
     para try; para check
     para push
 # Average job time:                  80s       1.33m     0.02h    0.00d
 # Longest job:                      157s       2.62m     0.04h    0.00d
 # Submission to last job:           583s       9.72m     0.16h    0.01d
 
     # combine predictions and transform scores to be in 0-1000 interval
     # do in a way that avoids limits on numbers of args
     rm -f splitfiles* all.raw.bed
     find /cluster/bluearc/hg17/multiz10way.v8/phastCons/ELEMENTS -name "*.bed" > files
     split files splitfiles
     foreach s (splitfiles*)
        awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", $1, $2, $3, $5, $5;}' `cat $s` >> all.raw.bed
     end
     /cluster/bin/scripts/lodToBedScore all.raw.bed > all.bed
     rm files splitfiles* 
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz10way.v8/cons/run.elements
     hgLoadBed hg17 phastConsElements10way all.bed
     sort -rn -k 5 all.bed | sed -n '1,100000p' > top100K.bed
     hgLoadBed hg17 phastConsElements10wayTop100K top100K.bed
 
     # check coverage -- reran estimation and conservation steps with new parameters till
     # coverage close to 5% and expected-length parameter is close to consEntropy recommended length
     
     featureBits hg17 phastConsElements10way
         # first pass
         #       .17 12
         # 132657993 bases of 2866216770 (4.628%) in intersection
         # second pass -- used this
         #       .20 11
         # 143386170 bases of 2866216770 (5.003%) in intersection
     featureBits hg17 phastConsElements
         # 137850739 bases of 2866216770 (4.810%) in intersection
 
     # check expected-length parameter
     # first pass
     /cluster/bin/phast/consEntropy .17 12 \
                         ave.cons.mod ave.noncons.mod --NH 9.78
         # recommended length 10.4
     # second pass -- good enough according to Adam
     /cluster/bin/phast/consEntropy .20 11 \
                         ave.cons.mod ave.noncons.mod --NH 9.78
         #( Solving for new omega: 11.000000 12.243251 12.155776 12.155369 )
         #Transition parameters: gamma=0.200000, omega=11.000000, mu=0.090909, nu=0.022727
         #Relative entropy: H=1.263205 bits/site
         #Required length: N=7.548911 sites
         #Total entropy: NH=9.535821 bits
         #Recommended expected length: omega=12.155369 sites (for NH=9.780000)
  
     # create wiggle data files
     ssh eieio
     cd /cluster/data/hg17/bed/multiz10way.v8/cons
     # sort post-prob files by chrom position using filename, then 
     #  use wigEncode to create binary files for wiggle
     find /cluster/bluearc/hg17/multiz10way.v8/phastCons/POSTPROBS \
         -name "*.pp.gz" | sort -t\. -k2,2n | xargs zcat | \
         wigEncode stdin phastCons10way.wig phastCons10way.wib
 
     hgWiggle -db=hg17 -doHistogram -hBinSize=0.001 \
         -hBinCount=1000 -hMinVal=0.0 -db=hg17 phastCons >histo.8way.data
     hgWiggle -db=hg17 -doHistogram -hBinSize=0.001 \
         -hBinCount=1000 -hMinVal=0.0 -db=hg17 phastCons10way >histo.10way.data
     hgWiggle -db=hg17 -doStats \
          phastCons > stats.8way.data
     hgWiggle -db=hg17 -doStats \
          phastCons10way > stats.10way.data
 
     # Load gbdb and database with wiggle.
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz10way.v8/cons
     set wibDir = /gbdb/hg17/multiz10way/wib/phastCons10way
     mkdir -p $wibDir
     ln -s `pwd`/phastCons10way.wib $wibDir
     hgLoadWiggle hg17 phastCons10way phastCons10way.wig \
                 -pathPrefix=$wibDir
 
     # create tree image:
         # edit tree.nh to create species.nh with common names
         /cluster/bin/phast/draw_tree -b -s species.nh > species10.ps
         # photoshop to enhance, then save as gif/jpg
         cp /cluster/data/hg17/bed/multiz10way.v8/species10.jpg \
             /usr/local/apache/htdocs/images/phylo/10way.jpg
 
     # get stats on the track
     ssh hgwdev
         
     featureBits hg17 -enrichment refGene:cds phastConsElements10way
         # refGene:cds 1.020%, phastConsElements10way 5.003%, both 0.711%, cover 69.73%, enrich 13.94x
         # compare to previous elements (generated from 8way)
     featureBits hg17 -enrichment refGene:cds phastConsElements
         # refGene:cds 1.020%, phastConsElements 4.810%, both 0.747%, cover 73.22%, enrich 15.22x
 
     # see how gluing reduces number of alignments
     ssh eieio
     cd /cluster/data/hg17/bed/multiz10way.v8
     mkdir stats
     grep score maf/chr22.maf | grep -v 0.0 | wc -l
         #179576
     grep score mafGlued/chr22.maf | grep -v 0.0 | wc -l
         #110550
 
     # look at distribution of alignment sizes after gluing
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz10way.v8
     mkdir mafTemp
     ln -s `pwd`/maf/chr1.maf mafTemp
     # load temp table
     hgLoadMaf hg17 -pathPrefix=mafTemp multiz10wayChr1
         #Loaded 1246727 mafs
     # again, compare to glued:
     echo "SELECT COUNT(*) FROM multiz10way"
         # 738030
         # again, ~40% fewer
 
     cd stats
     echo "SELECT chromEnd - chromStart FROM multiz10way WHERE chrom='chr1'" | \
         hgsql -N hg17 | sort -n > chr1.maf.glued.sizes
     echo "SELECT chromEnd - chromStart FROM multiz10wayChr1"| \
         hgsql -N hg17 | sort -n > chr1.maf.sizes
     # cleanup
     hgsql hg17 -e "DROP TABLE multiz10wayChr1"
     rm -fr ../mafTemp
 
     # coverage of multiple alignment, and pairs
     ssh kolossus
     cd /cluster/data/hg17/bed/multiz10way.v8
     cd stats
     nice mafRanges -notAllOGap ../mafGlued/chr1.maf hg17 \
                 hg17.chr1.mafRanges.bed
     nice mafRanges -notAllOGap /cluster/data/hg17/bed/multiz8way/maf/chr1.maf \
                 hg17 hg17.8way.chr1.mafRanges.bed
     foreach db (panTro1 canFam1 mm5 rn3 galGal2 xenTro1 fr1 tetNig1 danRer1)
         echo $db
         nice mafRanges /cluster/data/hg17/bed/blastz.$db/mafNet/chr1.*maf \
                -notAllOGap hg17 $db.chr1.mafRanges.bed
         ls /cluster/data/hg17/bed/blastz.$db/mafNet/chr1.*maf
     end
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz10way.v8/stats
     nice featureBits -chrom=chr1 hg17 refGene:cds hg17.chr1.mafRanges.bed -enrichment
 # refGene:cds 1.308%, hg17.chr1.mafRanges.bed 95.725%, both 1.307%, cover 99.94%, enrich 1.04x
     nice featureBits -chrom=chr1 hg17 refGene:cds hg17.8way.chr1.mafRanges.bed -enrichment
 # refGene:cds 1.308%, hg17.8way.chr1.mafRanges.bed 95.742%, both 1.307%, cover 99.97%, enrich 1.04x
 
     foreach db (panTro1 canFam1 mm5 rn3 galGal2 xenTro1 fr1 tetNig1 danRer1)
         nice featureBits -chrom=chr1 -enrichment hg17 refGene:cds $db.chr1.mafRanges.bed
     end
 
 #refGene:cds 1.308%, panTro1.chr1.mafRanges.bed 93.472%, both 1.264%, cover 96.65%, enrich 1.03x
 #refGene:cds 1.308%, canFam1.chr1.mafRanges.bed 55.377%, both 1.277%, cover 97.64%, enrich 1.76x
 #refGene:cds 1.308%, mm5.chr1.mafRanges.bed 37.342%, both 1.280%, cover 97.92%, enrich 2.62x
 #refGene:cds 1.308%, rn3.chr1.mafRanges.bed 35.429%, both 1.257%, cover 96.14%, enrich 2.71x
 #refGene:cds 1.308%, galGal2.chr1.mafRanges.bed 3.840%, both 0.936%, cover 71.61%, enrich 18.65x
 #refGene:cds 1.308%, xenTro1.chr1.mafRanges.bed 3.059%, both 0.881%, cover 67.36%, enrich 22.02x
 #refGene:cds 1.308%, fr1.chr1.mafRanges.bed 1.892%, both 0.854%, cover 65.29%, enrich 34.50x
 #refGene:cds 1.308%, tetNig1.chr1.mafRanges.bed 1.384%, both 0.805%, cover 61.57%, enrich 44.50x
 #refGene:cds 1.308%, danRer1.chr1.mafRanges.bed 2.716%, both 0.847%, cover 64.81%, enrich 23.86x
 
 
 # MAKE HG17-RN3 OVER.CHAIN FOR LIFTOVER  (DONE 1/25/05 angie)
     ssh kolossus
     set chainDir = /cluster/data/hg17/bed/blastz.rn3/axtChain
     netChainSubset $chainDir/rat.net.gz $chainDir/all.chain.gz \
       /cluster/data/hg17/bed/bedOver/hg17ToRn3.over.chain
     
 
 # MAKE HG17-GALGAL2 OVER.CHAIN FOR LIFTOVER  (DONE 1/25/05 angie)
     ssh kolossus
     set chainDir = /cluster/data/hg17/bed/blastz.galGal2/axtChain
     netChainSubset $chainDir/human.net $chainDir/all.chain \
       /cluster/data/hg17/bed/bedOver/hg17ToGalGal2.over.chain
     
 
 # DOWNLOADS FOR 10-WAY MULTIZ (2005-01-24 kate)
 #   Use "glued" mafs
 
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17
     mkdir -p multiz10way
     cd multiz10way
     foreach f (/cluster/data/hg17/bed/multiz10way.v8/mafGlued/*.maf)
         set c = $f:r:t
         echo $c
         nice gzip -c $f > $c.maf.gz
     end
     # copy README and edit
 
     # Create upstream files for download 
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz10way.v8
     echo hg17 panTro1 mm5 rn3 canFam1 galGal2 xenTro1 fr1 tetNig1 danRer1 > org.txt
     # mafFrags takes a while
     foreach i (1000 2000 5000)
         echo "making upstream$i.maf"
         nice featureBits hg17 refGene:upstream:$i -fa=/dev/null -bed=up.bad
         awk -F '\t' '{printf("%s\t%s\t%s\t%s\t%s\t%s\n", $1, $2, $3, substr($4, 0, 9), 0, $5)}' up.bad > up.bed
         rm up.bad
         nice mafFrags hg17 multiz10way up.bed upstream$i.maf -orgs=org.txt
         rm up.bed
     end
     ssh eieio
     cd /cluster/data/hg17/bed/multiz10way.v8
     nice gzip upstream{1000,2000,5000}.maf
         # 6 mins.
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17
     mv /cluster/data/hg17/bed/multiz10way.v8/upstream*.maf.gz multiz10way
     cd multiz10way
     md5sum *.gz > md5sum.txt
 
     #	Create histogram of this phastCons data  (Hiram - 2005-02-07)
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz.2004-12-22/cons
     time hgWiggle -doHistogram \
         -hBinSize=0.001 -hBinCount=1000 -hMinVal=0.0 -verbose=2 \
 	    -db=hg17 phastCons > histogram.data 2>&1
     #	34 minutes
 
     cat << '_EOF_' > histo.gp
 set terminal png small color \
         x000000 xffffff xc000ff x66ff66 xffff00 xff0000 xff0000
 set size 1.4, 0.8
 set key left box
 set grid noxtics
 set grid ytics
 set title " Hg17 Histogram phastCons track"
 set xlabel "Hg17 phastCons score"
 set ylabel "p-Value"
 set y2label "Cumulative Probability Distribution"
 set y2range [0:1]
 set y2tics
 
 plot "histogram.data" using 2:5 title " pValue" with impulses, \
         "histogram.data" using 2:7 axes x1y2 title " CPD" with lines
 '_EOF_'
     
     gnuplot histo.gp > histo.png
     display histo.png &
 
 
 # BLASTZ BOREOEUTHERIAN (BOREUT1) (DONE 1/29/05 braney)
     ssh kk
     mkdir /cluster/data/borEut1/bed/zb.hg17 
     ln -s /cluster/data/borEut1/bed/zb.hg17 /cluster/data/hg17/bed/blastz.borEut1
     cd /cluster/data/hg17/bed/blastz.borEut1
     # Use default (Human-Mouse) settings for starters.
     cat << '_EOF_' > DEF
 # human vs. dog
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin
 
 ALIGN=blastz-run
 BLASTZ=blastz
 
 # Default
 BLASTZ_H=2000
 BLASTZ_ABRIDGE_REPEATS=1
 
 # TARGET: Human
 SEQ1_DIR=/scratch/hg/gs.18/build35/bothMaskedNibs
 SEQ1_RMSK=
 SEQ1_FLAG=
 SEQ1_SMSK=/scratch/hg/gs.18/build35/linSpecRep.notInDog
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY: Dog
 SEQ2_DIR=/iscratch/i/borEut1/nib
 SEQ2_RMSK=
 SEQ2_FLAG=
 SEQ2_SMSK=/scratch/hg/canFam1/linSpecRep.notInHuman
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=10000000
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/blastz.borEut1
 
 DEF=$BASE/DEF
 RAW=$BASE/raw
 CDBDIR=$BASE
 SEQ1_LEN=$BASE/S1.len
 SEQ2_LEN=$BASE/S2.len
 '_EOF_'
     # << this line keeps emacs coloring happy
 
     /cluster/data/hg17/jkStuff/BlastZ_run0.sh
     cd run.0
     para push
 # Completed: 2728 of 2728 jobs
 # CPU time in finished jobs:     621440s   10357.34m   172.62h    7.19d  0.020 y
 # IO & Wait Time:                 19079s     317.98m     5.30h    0.22d  0.001 y
 # Average job time:                 235s       3.91m     0.07h    0.00d
 # Longest job:                     2340s      39.00m     0.65h    0.03d
 # Submission to last job:          2837s      47.28m     0.79h    0.03d
 
     ssh kki
     cd /cluster/data/hg17/bed/blastz.borEut1
     /cluster/data/hg17/jkStuff/BlastZ_run1.sh
 
     cd run.1
     para push
 # Completed: 341 of 341 jobs
 # CPU time in finished jobs:         95s       1.58m     0.03h    0.00d  0.000 y
 # IO & Wait Time:                   825s      13.75m     0.23h    0.01d  0.000 y
 # Average job time:                   3s       0.04m     0.00h    0.00d
 # Longest job:                       10s       0.17m     0.00h    0.00d
 # Submission to last job:            73s       1.22m     0.02h    0.00d
 
     ssh kk
     cd /cluster/data/hg17/bed/blastz.borEut1
     /cluster/data/hg17/jkStuff/BlastZ_run2.sh
     cd run.2
     para push
 
 # /cluster/data/hg17/bed/blastz.borEut1/axtChrom/chr18_random.axt is empty
 # /cluster/data/hg17/bed/blastz.borEut1/axtChrom/chr19_random.axt is empty
 # ..
 # Completed: 44 of 46 jobs
 # Crashed: 2 jobs
 # CPU time in finished jobs:        104s       1.73m     0.03h    0.00d  0.000 y
 # IO & Wait Time:                   482s       8.04m     0.13h    0.01d  0.000 y
 # Average job time:                  13s       0.22m     0.00h    0.00d
 # Longest job:                      134s       2.23m     0.04h    0.00d
 # Submission to last job:           142s       2.37m     0.04h    0.00d
 
 # END BLASTZ BOREOEUTHERIAN
 
 
 ##########################################################################
 # MAKE tfbsConsSites and tfbsConsFactors for TFBS conserved track (DONE braney 1/15/05)
 # Questions?  weirauch@soe.ucsc.edu or braney@soe.ucsc.edu
 #	tfbsConsSites table reloaded 2006-11-03 - Hiram - see below:
 ## reload tfbsCons table - it was based on a newer version of tfbs names that
 
 ssh hgwdev
 mkdir /cluster/data/hg17/bed/tfbsCons
 cd /cluster/data/hg17/bed/tfbsCons
 
 # Define all parameters in 'PARAMS.txt'
 # Define all chromosomes in 'CHROMS.txt'
 # Get tfbsConsUtils.tar.gz from Matt Weirauch with Perl scripts  weirauch@soe.ucsc.edu
 set tarfile=/cluster/data/hg17/bed/tfbsCons/tfbsConsUtils.tar.gz
 tar zxf $tarfile
 
 nice ./getRefseqStats.pl &
 nice ./getBatchQueries.pl &
 
 ssh kk
 mkdir /cluster/bluearc/braney/tfloc
 # Copy ./tmp/ctfbs_batch_list.txt to this dir
 # Copy ./scripts/doit to this dir
 para create ctfbs_batch_list.txt
 para try
 para push
 
 # When the run is done (within a day or so), the results will be in individual dirs, one for each chromosome.
 
 ssh kksilo (or hgwdev, or whatever)
 nice ./getBedFile.pl &
 
 hgLoadBed -noSort hg17 tfbsConsSites \
 	-sqlTable=$HOME/kent/src/hg/lib/tfbsConsSites.sql \
 	tfbsConsSites.bed -tab
 hgLoadBed -noSort hg17 tfbsConsFactors \
 	-sqlTable=$HOME/kent/src/hg/lib/tfbsConsFactors.sql \
 	tfbsConsFactors.bed -tab
 
 # Feel free to delete or gzip anything in ./tmp
 #  (particularly the huge .maf and .bed files)
 #   after the final two bed files are sucessfully loaded
 
 ##########################################################################
 # CHICKEN RECIPROCAL-BEST NET FOR STRINGENT LIFTOVER (DONE 2/3/05 angie)
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.galGal2/axtChain
     # Run chainNet again, this time keeping both of its outputs:
     chainPreNet all.chain ../S1.len ../S2.len stdout \
     | chainNet stdin ../S1.len ../S2.len h_g.net g_h.net
     # Get the chicken chains from the chicken-referenced (but human-centric)
     # net:
     chainSwap all.chain g_h.chain
     netChainSubset g_h.net g_h.chain stdout \
     | chainSort stdin g_h.subset.chain
     # Net those (sorted) chicken chains, and keep both outputs, to get 
     # reciprocal best nets referenced to both species:
     chainPreNet g_h.subset.chain ../S2.len ../S1.len stdout \
     | chainNet stdin ../S2.len ../S1.len g_h.rbest.net h_g.rbest.net
     # Get the chains from the recip-best nets for stringent liftOver:
     netChainSubset g_h.rbest.net g_h.chain galGal2ToHg17.rbest.over.chain
     netChainSubset h_g.rbest.net all.chain hg17ToGalGal2.rbest.over.chain
 
 
 
 ####### RE-BUILD RGD HUMAN QTL TRACKS (DONE 2/5/05 Fan) ##############
 
 mkdir -p /cluster/store8/rgd/human050205
 rm /cluster/data/hg17/bed/rgdQtl
 ln -s /cluster/store8/rgd/human050205 /cluster/data/hg17/bed/rgdQtl
 cd /cluster/data/hg17/bed/rgdQtl
 
 # download data files from RGD
 
 wget --timestamp ftp://rgd.mcw.edu/pub/RGD_genome_annotations/human/human_QTL.gff
 
 # remove extra line feed character at the end of lines
 
 # !!! manually corrected the line of AASTH7_H because chromStart is greater than chrEnd
 
 rmLf human_QTL.gff > rgdQtl.gff
 
 # create rgdQtl.tab
 awk '{print $1"\t"$4"\t"$5"\t"$10}'  rgdQtl.gff |sed -e 's/Chr/chr/g'| \
 sed -e 's/"//g' |sed -e 's/RGD://g' | sed -e 's/;//g' > rgdQtl.tab
 
 # create rgdQtlLink.tab
 
 awk '{printf "%s\t%s\t", $12, $10; for (i = 14;i <= NF; ++i ) {printf "%s ", $i} printf "\n"} ' rgdQtl.gff | \
 sed -e 's/"//g'| sed -e 's/RGD://g' | sed -e 's/;//g'| sed -e 's/Note//g' > rgdQtlLink.tab
 
 # load rgdQtl table
 hgLoadBed hg17 rgdQtl rgdQtl.tab
 
 # check rgdQtl table
 checkTableCoords hg17 rgdQtl
 
 # load rgdQtlLink table
 hgsql hg17 -e "drop table hg17.rgdQtlLink;"
 hgsql hg17 <~/kent/src/hg/lib/rgdQtlLink.sql
 hgsql hg17 -e 'load data local infile "rgdQtlLink.tab" into table hg17.rgdQtlLink;'
 
 # updated trackDb.ra under /kent/src/hg/makeDb/trackDb/human/hg17 and
 # added rgdQtl.html.
 
 # GENOSCOPE TETRAODON (tetNig1) ECORES (DONE, 2005-02-08, hartera)
     ssh eieio
     mkdir -p /cluster/data/hg17/bed/ecoresTetNig1
     cd /cluster/data/hg17/bed/ecoresTetNig1
                                                                                 
     wget --timestamp \
          http://www.genoscope.cns.fr/externe//4ucsc/ExofishHs35Tnig1
     # this is in gff format
     # remove "Ecotig" from name field
     sed -e 's/Ecotig EG/EG/g' ExofishHs35Tnig1 > ExofishHs35Tnig1.gff
     # need to have tabs between fields not a space to load file into table
     sed -e 's/ /\t/g' ExofishHs35Tnig1.gff > Hs35Tnig1format.gff
     # if "ecore" is changed to "CDS" and "ecotig" to "transcript" this loads
     # correctly into the table.
     sed -e 's/ecore/CDS/' Hs35Tnig1format.gff | sed -e 's/ecotig/transcript/' \
            > Hg17vstetNig1.gff
     # add "chr" in front of the chromsome name in first field (2005-02-08)
     perl -pi.bak -e 's/^([0-9XYM]{1,2})/chr$1/' Hg17vstetNig1.gff
     rm *.bak
     # need to reload table
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/ecoresTetNig1
     echo 'drop table ecoresTetNig1;' | hgsql hg17
     nice ldHgGene hg17 ecoresTetNig1 Hg17vstetNig1.gff
     # Read 40172 transcripts in 186032 lines in 1 files
     #  40172 groups 42 seqs 1 sources 2 feature types
     # 40172 gene predictions
 # added ecoresTetNig1 entry to trackDb.ra in trackDb/human
 # and created ecoresTetNig1.html.  Genoscope will not be maintaining this
 # newest data in their Exofish comparative browser display.
 
 # UPDATE kgSpAlias TABLE WITH NEW UNIPROT DISPLAY ID ENTRIES (done 2/11/05 Fan)
 # Add new human protein display IDs to the alias table to support user search
     
     ssh hgwdev
     cd /cluster/data/hg17/bed/pb
     mkdir newDisplayId
     cd newDisplayId
  
     hgsql proteome -e 'select hg17.kgSpAlias.kgID, hg17.kgSpAlias.SpID, spOldNew.newDisplayId from spOldNew, hg17.kgSpAlias where spOldNew.acc=hg17.kgSpAlias.spID and oldDisplayId != newDisplayId' |sort -u >hg17.tab
     
     hgsql hg17 -e 'load data local infile "hg17.tab" into table hg17.kgSpAlias'
 
 # UPDATE kgProtAlias TABLE WITH NEW UNIPROT DISPLAY ID ENTRIES (done 2/11/05 Fan)
 # Add new hg17 protein display IDs to the alias table to support user search
     
     ssh hgwdev
     cd /cluster/data/hg17/bed/pb/newDisplayId
 
      hgsql proteome -e 'select hg17.kgSpAlias.kgID,spOldNew.oldDisplayId,spOldNew.newDisplayId from spOldNew, hg17.kgSpAlias where spOldNew.acc=hg17.kgSpAlias.spID and oldDisplayId != newDisplayId' |sort -u >hg17.kgProtAlias.tab
 
 # get rid of the header line at the end of the file
     vi hg17.kgProtAlias.tab 
 
     hgsql hg17 -e 'load data local infile "hg17.kgProtAlias.tab" into table hg17.kgProtAlias'
 
 
 
 # BLASTZ HUMAN TARGET, COW QUERY (DONE, Nov. 2004 - Jan. 2005, Heather)
   ssh kk
   # use /cluster/data/bosTau1 because more disk space there
   cd /cluster/data/bosTau1/bed
   mkdir zb.hg17
 
   # create DEF file 
   # for now, not doing ABRIDGE_REPEATS
   # this means I don't need to create lineage specific repeats
   # This is because blastz-run wouldn't take advantage of these
   # because my query is in scaffolds
 
 # human vs. cow
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin
 
 ALIGN=blastz-run
 BLASTZ=blastz
 
 # Default
 BLASTZ_H=2000
 BLASTZ_ABRIDGE_REPEATS=0
 
 # TARGET: Human
 SEQ1_DIR=/scratch/hg/gs.18/build35/bothMaskedNibs
 #SEQ1_DIR=/iscratch/i/hg17/bothMaskedNibs
 SEQ1_RMSK=
 SEQ1_FLAG=
 SEQ1_SMSK=
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY: Cow
 SEQ2_DIR=/iscratch/i/bosTau1/splitDir
 SEQ2_RMSK=
 SEQ2_FLAG=
 SEQ2_SMSK=
 SEQ2_IN_CONTIGS=1
 SEQ2_CHUNK=10000000
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/zb.hg17
 
 DEF=$BASE/DEF
 RAW=$BASE/raw
 CDBDIR=$BASE
 SEQ1_LEN=$BASE/S1.len
 SEQ2_LEN=$BASE/S2.len
 
 '_EOF_'
   # << this line keeps emacs coloring happy
   bash
   cd /cluster/data/bosTau1/bed/zb.hg17
   source DEF
   mkdir $RAW run.0
   # create S2.len so make-joblist doesn't have to
   /cluster/bin/scripts/blastz-make-joblist $DEF > $BASE/run.0/j
   sh ./xdir.sh
   cd run.0
   # check how many lines in j
   sed -e 's@^blastz-run@/cluster/bin/penn/blastz-run@' j > jobList
   para create jobList
   para try, para check, para push, para check....
 
   # convert out to lav
   ssh kki
   cd /cluster/data/bosTau1/bed/zb.hg17
   # run bash shell if not running it already
   source DEF
   mkdir -p $BASE/run.1
   mkdir -p $BASE/lav
   # create a new job list to convert out files to lav
   /cluster/bin/scripts/blastz-make-out2lav $DEF $BASE > run.1/jobList
   cd run.1
   # make sure the job list is OK
   wc -l jobList
   head jobList
   para create jobList
   para try
   para check
   para push
 
   # lavToAxt
   ssh kksilo
   cd /cluster/data/bosTau1/bed/zb.hg17
   mkdir axtTemp
   cd lav
   foreach i (*)
       catDir $i | lavToAxt stdin /cluster/data/hg17/nib \
         /cluster/data/bosTau1/bosTau1.2bit ../axtTemp/$i.axt
       echo done $i
   end
 
   # axtChain
   ssh kki
   cd /cluster/data/bosTau1/bed/zb.hg17
   mkdir -p axtChain/run1
   cd axtChain/run1
   mkdir out chainRaw
   ls -1S /cluster/data/bosTau1/bed/zb.hg17/axtTemp/*.axt > input.lst
 
   cat << '_EOF_' > gsub
   #LOOP
   doChain {check in exists $(path1)} {check out line+ chainRaw/$(root1).chain} {check out exists out/$(root1).out}
   #ENDLOOP
   '_EOF_'
   # << this line makes emacs coloring happy
     
   cat << '_EOF_' > doChain
   #!/bin/csh
 '_EOF_'
   axtChain $1 /iscratch/i/hg17/bothMaskedNibs /iscratch/i/bosTau1/nib/bosTau1.2bit $2 > $3
     
   # << this line makes emacs coloring happy
   chmod a+x doChain
   gensub2 input.lst single gsub jobList
   para create jobList
   para try
   para check
   para push
 
   # Completed: 46 of 46 jobs
   # Average job time:                  83s       1.39m     0.02h    0.00d
   # Longest job:                     1240s      20.67m     0.34h    0.01d
   # Submission to last job:          1326s      22.10m     0.37h    0.02d
 
   # mergesort
   ssh kksilo
   cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
   chainMergeSort run1/chainRaw/*.chain > all.chain.jan3
 
   # chainAntiRepeat
   ssh kksilo
   cd /cluster/data/bosTau1/bed/zb.hg17/axtChain/run1
   mkdir chainAntiRepeat
   # test with just one
   chainAntiRepeat /cluster/store5/gs.18/build35/nib /cluster/data/bosTau1/bosTau1.2bit \
     chainRaw/chr18.chain chainAntiRepeat/chr18.chain
   # do them all
   foreach f (chainRaw/*.chain)
     set f1 = $f:t
     echo $f1
       chainAntiRepeat /cluster/store5/gs.18/build35/nib /cluster/data/bosTau1/bosTau1.2bit \
         $f chainAntiRepeat/$f1
   end
 
   # mergesort again
   ssh kksilo
   cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
   chainMergeSort run1/chainAntiRepeat/*.chain > all.chain.jan5
   gzip all.chain.jan3
 
   # split
   mkdir chain
   chainSplit chain all.chain.jan5
 
   # look at the distribution
   foreach f (chain/*.chain)
     grep chain $f | awk '{print $2;}' | sort -nr > /tmp/score.$f:t:r
     echo $f:t:r
     textHistogram -binSize=5000 /tmp/score.$f:t:r
     echo ""
   end
   # see files histogram.out and histogram.interesting
 
   # run chainFilter
   chainFilter -minScore=5000 all.chain.jan5 > all.chain.jan5.filtered
   gzip all.chain.jan5
 
   # split
   rm chain/*
   chainSplit chain all.chain.jan5.filtered
   gzip all.chain.jan5.filtered
 
   # load
   ssh hgwdev
   cd /cluster/data/bosTau1/bed/zb.hg17/axtChain/chain
   foreach i (*.chain)
     set c = $i:r
     echo loading $c
     hgLoadChain hg17 ${c}_chainBosTau1 $i
   end
 
     # featureBits -chrom=chr1 hg17 chainBosTau1Link
     # 103272818 bases of 222827847 (46.346%) in intersection
     # featureBits -chrom=chr2 hg17 chainBosTau1Link
     # 105920345 bases of 237506229 (44.597%) in intersection
     # featureBits -chrom=chr3 hg17 chainBosTau1Link
     # 89582887 bases of 194635740 (46.026%) in intersection
     # featureBits -chrom=chr4 hg17 chainBosTau1Link
     # 77513949 bases of 187161218 (41.416%) in intersection
     # featureBits -chrom=chr5 hg17 chainBosTau1Link
     # 80428726 bases of 177702766 (45.260%) in intersection
     # featureBits -chrom=chr6 hg17 chainBosTau1Link
     # 71830264 bases of 167317699 (42.930%) in intersection
     # featureBits -chrom=chr7 hg17 chainBosTau1Link
     # 64561289 bases of 154759139 (41.717%) in intersection
     # featureBits -chrom=chr8 hg17 chainBosTau1Link
     # 55896735 bases of 142612826 (39.195%) in intersection
     # featureBits -chrom=chr9 hg17 chainBosTau1Link
     # 52068957 bases of 117781268 (44.208%) in intersection
     # featureBits -chrom=chr10 hg17 chainBosTau1Link
     # 57427282 bases of 131613628 (43.633%) in intersection
     # featureBits -chrom=chr11 hg17 chainBosTau1Link
     # 58412709 bases of 131130853 (44.545%) in intersection
     # featureBits -chrom=chr12 hg17 chainBosTau1Link
     # 56076163 bases of 130259811 (43.049%) in intersection
     # featureBits -chrom=chr13 hg17 chainBosTau1Link
     # 37951944 bases of 95559980 (39.715%) in intersection
     # featureBits -chrom=chr14 hg17 chainBosTau1Link
     # 39896970 bases of 88290585 (45.188%) in intersection
     # featureBits -chrom=chr15 hg17 chainBosTau1Link
     # 37507979 bases of 81341915 (46.112%) in intersection
     # featureBits -chrom=chr16 hg17 chainBosTau1Link
     # 33883573 bases of 78884754 (42.953%) in intersection
     # featureBits -chrom=chr17 hg17 chainBosTau1Link
     # 31871034 bases of 77800220 (40.965%) in intersection
     # featureBits -chrom=chr18 hg17 chainBosTau1Link
     # 30359555 bases of 74656155 (40.666%) in intersection
 
 
 
 # NET 
 # run in stages to avoid memory problems
   ssh kolossus
   cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
 
   # PRE
   /cluster/bin/x86_64/chainPreNet all.chain.jan5.filtered ../S1.len ../S2.len chainPreNet.out
 
   # chainNet
   /cluster/bin/x86_64/chainNet chainPreNet.out \
     -minSpace=1 ../S1.len ../S2.len bosTau1.net.raw /dev/null
 
   # syntenic (using revision 1.6)
   /cluster/home/heather/bin/x86_64/netSyntenic bosTau1.net.raw bosTau1.net.syn
   # memory usage 2757492736, utime 13404 s/100, stime 616
 
   # backup/compress
   ssh kksilo
   cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
   gzip bosTau1.net.raw
   cp bosTau1.net.syn bosTau1.net.syn.backup
 
   # netClass
   # takes about 4 hours
   ssh hgwdev
   cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
   netClass -noAr bosTau1.net.syn hg17 bosTau1 bosTau1.net
 
   # backups
   ssh kksilo
   cp bosTau1.net bosTau1.net.backup
   rm bosTau1.net.syn.backup
 
   # load
   ssh hgwdev
   cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
   netFilter -minGap=10 bosTau1.net | hgLoadNet hg17 netBosTau1 stdin
   rm bosTau1.net.backup
 
   # index has NULL cardinality; analyze to fix
   hgsql hg17
   analyze table netBosTau1
 
   # generate axts 
   ssh kksilo
   cd /cluster/data/bosTau1/bed/zb.hg17
   mkdir axtNet
 
   # split first (not required?)
   cd axtChain
   mkdir net
   netSplit bosTau1.net.syn net
   cd net
   foreach i (*.net)
     netToAxt $i ../chain/$i:r.chain /cluster/data/hg17/nib /cluster/data/bosTau1/bosTau1.2bit ../../axtNet/$i:r.axt
   end
   gzip bosTau1.net.syn
   gzip bosTau1.net
 
   # axtSort (takes about 5 minutes)
   ssh kksilo
   cd /cluster/data/bosTau1/bed/zb.hg17
   mkdir axtNetSort
   foreach f ( axtNet/*.axt )
     set c = $f:t:r
     echo "axtSort on $c"
     axtSort $f axtNetSort/$c.axt
   end
 
   # make maf files
   mkdir mafNet
   foreach f (axtNetSort/*.axt)
     set c = $f:t:r
     echo "axtToMaf on $c"
     axtToMaf $f /cluster/data/hg17/chrom.sizes /cluster/data/bosTau1/chrom.sizes mafNet/$c.maf -tPrefix=hg17. -qPrefix=bosTau1.
   end
 
 # MAKE VSBOSTAU1 DOWNLOADABLES (DONE Feb. 15, 2005 Heather)
     ssh hgwdev
     cd /usr/local/apache/htdocs/goldenPath/hg17
     mkdir vsBosTau1
     cd vsBosTau1
     mkdir axtNet
     cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
     cp -p all.chain.gz  /usr/local/apache/htdocs/goldenPath/hg17/vsBosTau1/cow.chain.gz
     cp -p bosTau1.net.gz /usr/local/apache/htdocs/goldenPath/hg17/vsBosTau1/cow.net.gz
     cd ../axtNet
     cp -p * /usr/local/apache/htdocs/goldenPath/hg17/vsBosTau1/axtNet
 
     cd /usr/local/apache/htdocs/goldenPath/hg17/vsBosTau1
     # Make a README.txt which explains the files & formats.
     md5sum *.gz > md5sum.txt
     cd axtNet
     md5sum *.gz > md5sum.txt
 
 # YALE PSEUDOGENES (started Robert Baertsch, finished JK 2/21/05)
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mkdir pseudoYale
     cd pseudoYale
     # Place file obtained from Mark Gerstein at yale in pseudoYale.gtf
     ldHgGene hg17 pseudoYale pseudoYale.gtf
     # Note - I'm guessing how this goes.  Robert left no record. -jk
 
 # added xenoRefGene track (markd ~2005-02-20)
     add to /cluster/data/genbank/genbank.con
     hg17.refseq.mrna.xeno.load  = yes
     hg17.refseq.mrna.xeno.loadDesc = yes
 
 # BUILD ccdsGene and ccdsInfo tables (markd 2005-02-25)
     # download files to the genbank data area, as this will eventually
     # be done automatically as part of the genbank build process.
     cd  /cluster/data/genbank
     mkdir -p data/ccds/hg17/2005-02-25
     cd data/ccds/hg17/2005-02-25
 
     # get the basic text dumps of the data, rather than the database dumps
     wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/CCDS/
     # ends up with:
         About-NcbiHinxton.txt
         NcbiHinxton.txt
         NcbiHinxtonAllAccessions.txt
 
     # this is a preliminary release, it contained 2 PAR genes that had
     # bad coordinates and 7 genes that were determined at be pseudogenes
     # at the last minute.  The accessions for these 9 genes were
     # placed in skip.ccds and then removed:
     fgrep -v -f skip.ccds  /scratch/markd/gene-sets/ncbiDb/set1.5/NcbiHinxtonAllAccessions.txt >  /scratch/markd/gene-sets/ncbiDb/set1.5/NcbiHinxtonAllAccessions.cleaned.txt
 
     # create the tab files to load in the database
     /cluster/data/genbank/bin/i386/ccdsImport NcbiHinxtonAllAccessions.cleaned.txt ccdsGene.gp ccdsInfo.tab
 
     # load ccdsInfo
     hgsql hg17 <../../../../../lib/ccdsInfo.sql 
     hgsql -e 'load data local infile "ccdsInfo.tab" into table ccdsInfo'  hg17
 
     # load cdsGene.gp and check
     ldHgGene -predTab -genePredExt hg17 ccdsGene ccdsGene.gp 
     checkTableCoords hg17 -verbose=2 ccdsGene
     rm *.tab
     gzip -9 NcbiHinxton*.txt
   
     
 # BUILD refSeqKg TABLE TO SUPPORT CCDS GENES (RE-DONE, Fan 2/26/05)
 
     hgsql hg17 -N -e "select * from knownGene" >kg.gp
     hgsql hg17 -N -e "select * from refGene"   >ref.gp
 
     overlapSelect -inCds -strand  -idOutput -fraction=fraction.out -selectCds -overlapSimilarity=0.90 -selectFmt=genePred -inFmt=genePred kg.gp ref.gp refSeqKg.90.tab
     cat fraction.out|sort -u >refSeqKg.tab
 
     hgsql hg17 -e 'drop table refSeqKg'
     hgsql hg17 < ~/src/hg/lib/refSeqKg.sql
     hgsql hg17 -e 'load data local infile "refSeqKg.tab" into table refSeqKg'
 
    rm fraction.out
 
 # BUILD ccdsGene and ccdsInfo tables  (markd, reone 2005-03-17)
     cd /cluster/store5/genbank/data/ccds/hg17
     wget ftp://ftp.ncbi.nlm.nih.gov/pub/hcds/Hs35.1/CDSTrackDB/CCDS.20050303.tar.gz
     mkdir /scratch/tmp/ccds
     cd /scratch/tmp/ccds
     tar -zxf  /cluster/store5/genbank/data/ccds/hg17/CCDS.20050303.tar.gz     
 
     # import ccds database tables
     hgsql -e 'create database ccds'
     hgsql ccds </cluster/data/genbank/etc/createTables.sql 
     hgsql ccds </cluster/data/genbank/etc/createKeys.sql 
     /cluster/data/genbank/bin/i386/ccdsImport ccds data/[A-Z]*.txt
 
     # create and load ccdsGene and ccdsInfo tables from imported database
     /cluster/data/genbank/bin/i386/ccdsMkTables -loadDb ccds hg17 ccdsInfo ccdsGene
 
     # refSeqKg table
 
     hgsql -N -e "select * from knownGene" hg17 >kg.gp
     hgsql -N -e "select * from refGene" hg17 >ref.gp
 
     overlapSelect -statsOutput -strand -inCds -selectCds -overlapSimilarity=0.90 kg.gp ref.gp stdout | tail +2 | sort -u >refSeqKg.tab
 
     hgsql hg17 -e 'drop table refSeqKg'
     hgsql hg17 < ~/compbio/kent/src/hg/lib/refSeqKg.sql
     hgsql hg17 -e 'load data local infile "refSeqKg.tab" into table refSeqKg'
     cd ..
     rm -r ccds
 
 
 # COW BACENDS (Done, Heather, Mar. 21, 2005)
  
 ssh hgwdev
 cd /cluster/data/hg17/bed
 mkdir bacendsCow
 cd bacendsCow
  
 # Obtain GFF file from Denis; unzip into BACendhg15.gff
 
 # Convert into BED 6:
  
   makebed.pl < BACendhg17.gff > BACendhg17.bed
   hgLoadBed -noBin hg17 bacendsCow BACendhg17.bed
   # 53403 warnings
  
 # add to kent/src/hg/makeDb/trackDb/human/hg17/trackDb.ra
 
 # make map between ccds and known genes (markd 2005/03/08)
 # this should be run whenever either known genes or ccds is updated
 /cluster/data/genbank/bin/i386/mkCcdsGeneMap  -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap
 
 # UPDATE WGRNA TRACK (DONE, 2004-12-13, Fan)
 
 # Received updated data file, wg_track_april2005.txt, from Michel Weber by email.
 
    cut -f 2-10  wg_track_april2005.txt |tail +2 >wg_track_april2005.tab
 
 # Use editor to remove the last blank line.
 
    hgLoadBed -sqlTable=/cluster/home/fanhsu/hg/lib/wgRna.sql hg17 wgRna wg_track_april2005.tab
 
 # Asked Donna to update Reference section according to Michel's email.  
 
 ## refresh vega tracks with vega build30        (done 5/4/04 Robert)
 ##download vega mysql tables
 cd /cluster/store8/ensembl
 mkdir vega30_35c
 cd vega30_35c
 ln /cluster/store8/ensembl/vega30_35c /cluster/data/hg17/bed/vega30 -s
 
 for i in `cat tables` ; do wget -N ftp://ftp.ensembl.org/pub/human-30.35c/data/mysql/homo_sapiens_vega_30_35c/$i.gz ; done
 wget -N ftp://ftp.ensembl.org/pub/human-30.35c/data/mysql/homo_sapiens_vega_30_35c/homo_sapiens_vega_30_35c_mysql40_compatible..sql.gz 
 gunzip *.gz
 ##create mysql database
 mysql 
 create database vega30
 use vega30
 source homo_sapiens_vega_30_35c_mysql40_compatible.sql
 source dropMt.sql
 source load.sql
 exit
 
 
 hgsql vega30 -N -B < vegaGene.sql > vegaGene.tab
 awk -f vegaGene.awk < vegaGene.tab > vegaGene.gp     
 ldHgGene hg17 vegaGene -predTab vegaGene.gp -gtf -genePredExt 
 hgsql vega30 -N -B < vegaPseudo.sql > vegaPseudo.tab
 awk -f vegaPseudo.awk < vegaPseudo.tab > vegaPseudo.gp     
 ldHgGene hg17 vegaPseudoGene -predTab vegaPseudo.gp -gtf -genePredExt 
 
 #load processed pseudogenes
 grep Processed vegaPseudo.tab > vegaProcPseudo.tab
 awk -f vegaPseudo.awk < vegaProcPseudo.tab > vegaProcPseudo.gp
 ldHgGene hg17 vegaProcessedPseudo -predTab vegaProcPseudo.gp -gtf -genePredExt
 
 #load vegaInfo
 hgsql vega30 -N -B < vegaGeneInfo.sql > vegaInfo.tab
 hgsql vega30 -N -B < vegaPseudoInfo.sql >> vegaInfo.tab
 
 hgsql hg17 -N -B < /cluster/home/baertsch/kent/src/hg/lib/vegaInfo.sql
 echo 'load data local infile "vegaInfo.tab" into table vegaInfo' | hgsql hg17 -N -B
 
 #load down to hg16
 liftOver vegaGene.gp /gbdb/hg17/liftOver/hg17ToHg16.over.chain vegaGeneHg16.gp unMapped.gp -genePred
 liftOver vegaPseudo.gp /gbdb/hg17/liftOver/hg17ToHg16.over.chain vegaPseudoGeneHg16.gp unMappedPseudo.gp -genePred
 
 ldHgGene hg16 vegaGene -predTab vegaGeneHg16.gp -gtf 
 ldHgGene hg16 vegaPseudoGene -predTab vegaPseudoGeneHg16.gp -gtf 
 echo 'truncate table vegaInfo' | hgsql hg16 -N -B
 echo 'load data local infile "vegaInfo.tab" into table vegaInfo' | hgsql hg16 -N -B
 
 
 #########################################################################
 # MOUSE NET/CHAINS MM6 - Info contained in makeMm6.doc (200503 Hiram)
 
 
 ####################################################################################
 # RE-BUILD KNOWN GENES TABLES, 2ND TRIAL WITH VARIANT PROTEINS (Started 5/13/05 Fan)
 
 # First build protein databases, sp050415 and proteins050415
 # See makeProteins050415.doc for details.
 
 # Create working subdirectories and temporary databases (kgHg17F)
 
   ssh hgwdev
   cd /cluster/store10/kg
   mkdir kgHg17F  
   ln -s /cluster/store10/kg/kgHg17F /cluster/store6/kgDB/bed/kgHg17F
   ln -s /cluster/store10/kg/kgHg17F /cluster/data/hg17/bed/kgHg17F
 
   hgsql hg17 -e "create database kgHg17F"   
   hgsql hg17 -e "create database kgHg17FTemp"
 
   mkdir /cluster/bluearc/kgDB/kgHg17F
   mkdir /cluster/bluearc/kgDB/kgHg17F/protBlat
   ln -s /cluster/bluearc/kgDB/kgHg17F/protBlat /cluster/store10/kg/kgHg17F/protBlat
   cd /cluster/store10/kg/kgHg17F/protBlat
 
 ################################################################# 
 # VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV
 # The protBlat.psl was built during the first KG II build trial
 # The results are still valid, except that kgHg17E was used
 # instead of kgHg17F
 
 # Create working subdirectories and temporary databases (kgHg17E)
 
   ssh hgwdev
   cd /cluster/store10/kg
   mkdir kgHg17E  
   ln -s /cluster/store10/kg/kgHg17E /cluster/store6/kgDB/bed/kgHg17E
   ln -s /cluster/store10/kg/kgHg17E /cluster/data/hg17/bed/kgHg17E
 
   hgsql hg17 -e "create database kgHg17E"   
   hgsql hg17 -e "create database kgHg17ETemp"
 
   mkdir /cluster/bluearc/kgDB/kgHg17E
   mkdir /cluster/bluearc/kgDB/kgHg17E/protBlat
   ln -s /cluster/bluearc/kgDB/kgHg17E/protBlat /cluster/store10/kg/kgHg17E/protBlat
   cd /cluster/store10/kg/kgHg17E/protBlat
 
 # Get all human protein sequences
 
   hgsql -N sp050415 -e \
   'select proteins050415.spXref3.accession,protein.val from proteins050415.spXref3,protein where division="9606" and acc=accession' \
   |awk '{print ">" $1;print $2}' >humanProt.fa
 
 # Prepare and perform cluster run for protein/genome alignment
 
   ssh kk
   cd /cluster/data/hg17/bed/kgHg17E/protBlat
   mkdir prot
   faSplit sequence humanProt.fa 1000 prot/prot
   ls /cluster/bluearc/kgDB/kgHg17E/protBlat/prot/* > prot.lis
 
   ssh hgwdev
   cd /cluster/data/hg17/bed/kgHg17E/protBlat
   hgsql hg17 -N -e 'select chrom from chromInfo' > chrom.lis
   exit
   
   cat << '_EOF_' > gsub
 #LOOP
 /cluster/bin/i386/blat -noHead -t=dnax -q=prot /cluster/data/hg17/nib/$(path1).nib $(path2) {check out line+ /cluster/bluearc/kgDB/kgHg17E/protBlat/result/$(root1)_$(root2).psl}
 #ENDLOOP
 '_EOF_'
 
   mkdir result
   gensub2 chrom.lis prot.lis gsub jobList
 
   para create jobList
   para try
   para check
   para push
   para check ...
 # many output .psl will be empty, the warnings are OK.
 [kk:protBlat> para check
 45494 jobs in batch
 0 jobs (including everybody's) in Parasol queue.
 Checking finished jobs
 tracking errors: 1
 crashed: 12643
 ranOk: 32850
 total jobs in batch: 45494
 [kk:protBlat> para time
 45494 jobs in batch
 0 jobs (including everybody's) in Parasol queue.
 Checking finished jobs
 Completed: 32850 of 45494 jobs
 Crashed: 12643 jobs
 para.results: file not found.  paraHub can't write to this dir?
 CPU time in finished jobs:   36153510s  602558.50m 10042.64h  418.44d  1.146 y
 IO & Wait Time:               1585456s   26424.27m   440.40h   18.35d  0.050 y
 Average job time:                1149s      19.15m     0.32h    0.01d
 Longest running job:                0s       0.00m     0.00h    0.00d
 Longest finished job:          155120s    2585.33m    43.09h    1.80d
 Submission to last job:        276342s    4605.70m    76.76h    3.20d
 
 # This cluster run took about 3 days.  Crashed jobs are due to empty BLAT result.  It is OK.
 
 # collect BLAT results
 
    ssh hgwdev
    cd /cluster/data/hg17/bed/kgHg17E/protBlat
 
    mkdir result2
    mkdir result3
 
    cat chrom.lis |sed -e 's/chr/do1 chr/g' >doall
 
    cat << '_EOF_' > do1.1
 echo processing $1
 cat result/$1_prot*.psl >result2/$1.psl
 '_EOF_'
 
    cat << '_EOF_' > do1.2
 echo processing $1
 pslReps -nohead -minCover=0.80 -minAli=0.80 -nearTop=0.002 result2/$1.psl result3/$1.psl /dev/null >>j.out
 '_EOF_'
 
    chmod +x do*
 
    cp do1.1 do1
    doall
    cp do1.2 do1
    doall
 
    cat result3/*.psl >protBlat.psl
 
 #^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 # The end of protBlat.psl build, using kgHg17E
 ################################################################################
 
 
 
 
 ############################################################################
 # This part process the variant splice proteins.
 
 # First build variant splice protein tables.
 
 # Get all variant isoform human protein sequences
 
   ssh hgwdev
   cd /cluster/data/swissprot/050415/build
   wget --timestamp \
  ftp://us.expasy.org/databases/uniprot/current_release/knowledgebase/complete/uniprot_sprot_varsplic.fasta.gz
   wget --timestamp \
 ftp://us.expasy.org/databases/uniprot/current_release/knowledgebase/complete/uniprot_trembl_varsplic.fasta.gz
 
   gzip -d *varsplic.fasta.gz
 
   faToTab -type=protein uniprot_trembl_varsplic.fasta splicTrembl.tab
   faToTab -type=protein uniprot_sprot_varsplic.fasta splicSprot.tab
 
   cat splicTrembl.tab splicSprot.tab >varProtein.tab
   hgsql sp050415 < ~/src/hg/lib/varProtein.sql
   hgsql sp050415 -e 'load data local infile "varProtein.tab" into table varProtein'
 
   cat varProtein.tab |cut -f 1>j1
   cut -f 1 j1|sed -e 's/-/\t/g' >j2
   paste j1 j2 >splicProt.tab
 
   hgsql kgHg17FTemp -e 'drop table splicProt'
   hgsql kgHg17FTemp <~/src/hg/lib/splicProt.sql
   hgsql kgHg17FTemp -e 'load data local infile "splicProt.tab" into table splicProt'
 
   hgsql kgHg17FTemp -N -e \
   'select varAcc, varProtein.val from sp050415.varProtein,splicProt,proteins050415.spXref3 where accession=parAcc and varProtein.acc=splicProt.varAcc and division="9606"'| \
    awk '{print ">" $1;print $2}' >humanVarProt.fa
 
 
    cd /cluster/data/hg17/bed/kgHg17F
 
 # get all Human splicProtBlat records
 
 hgsql hg17 -N -e \
 'select splicProtBlat.* from splicProtBlat,proteins050415.spXref3,kgHg17FTemp.splicProt where qName=splicProt.varAcc and parAcc=accession and division="9606"'\
 |cut -f 2-22 \
 >humanVarProtBlat.psl
   
 # Combine the regular protein protBlat records with the variant protein psl records.
   cd /cluster/store10/kg/kgHg17F
   cat ../kgHg17E/protBlat/protBlat.psl humanVarProtBlat.psl >protBlat.psl
 
   hgLoadPsl hg17 protBlat.psl
 # Processing protBlat.psl
 # load of protBlat did not go as planned: 104064 record(s), 0 row(s) skipped, 1484 warning(s) loading psl.tab
 # Looked into the cause of these 1484 warnings.  It was due to that qBaseInsert and tBaseInsert 
 # have negative values, probably due to that this is protein alignment.
 
 
 # create all_mrna.psl and tight_mrna.psl
    hgsql hg17 -N -e "select * from all_mrna" |cut -f 2-22 >all_mrna.psl
 
    pslReps -minCover=0.40 -minAli=0.97 -nearTop=0.002 \
            all_mrna.psl tight_mrna.psl /dev/null
 
 # Use overlapSelect to get protein and mRNA alignment overlaps   
    overlapSelect  -statsOutput  -dropped=protOut.psl -overlapThreshold=0.90 \
    -selectFmt=psl -inFmt=psl tight_mrna.psl  protBlat.psl protMrna.stat
 
    overlapSelect  -mergeOutput  -dropped=protOut.psl -overlapThreshold=0.90 -selectFmt=psl \
    -inFmt=psl tight_mrna.psl  protBlat.psl protMrna.out
 
 # Create protein/mRNA pair and protein lists
    cut -f 10,31 protMrna.out|sort -u >spMrna.tab
    cut -f 10    protMrna.out|sort -u >protein.lis
 
 # Load spMrna.tab into spMrna table in temp DB.
    hgsql kgHg17FTemp < ~/src/hg/lib/spMrna.sql
    hgsql kgHg17FTemp -e 'load data local infile "spMrna.tab" into table spMrna'
    hgsql kgHg17FTemp -e 'create index mrnaID on spMrna(mrnaID)'
 
 # Prepare and perform cluster run of protein/mRNA alignment
 
 # Get mRNA fa file.
    cd /cluster/data/hg17/bed/kgHg17F
    /cluster/data/genbank/bin/i386/gbGetSeqs -native -db=hg17 \
    -gbRoot=/cluster/data/genbank genbank mrna mrna.fa
 
 # Create mrnaSeq table in kgHg17FTemp DB.
 
    faToTab mrna.fa mrnaSeq.tab
 
    hgsql kgHg17FTemp -e 'drop table mrnaSeq'
    hgsql kgHg17FTemp <~/src/hg/lib/mrnaSeq.sql
    hgsql kgHg17FTemp -e 'load data local infile "mrnaSeq.tab" into table mrnaSeq'
 # Prepare files for cluster run
    ~/src/hg/protein/KG2.sh kgHg17F hg17 050415
 
 # Perform cluster run of protein/mRNA alignment
    ~/src/hg/protein/KG3.sh kgHg17F hg17 050415
 
 # Collect cluster run results
    cd kgBestMrna
 
    ls out | sed -e 's/prot/do1 prot/g' >doall
 
 # create do1 with the following 2 lines:
    cat << '_EOF_' > do1
 echo processing $1
 cat out/$1/*.out >>protMrnaRaw.psl
 '_EOF_'
 
    chmod +x do*
    doall
 
 # Filter out low quality alignments
    pslReps -nohead -singleHit -minAli=0.9 protMrnaRaw.psl protMrnaBlat.psl /dev/null
    cut -f 10,14 protMrnaBlat.psl |sort -u >protMrna.lis
    wc protMrna.lis
 
 # Load BLAT results into temp DB.
    hgsql kgHg17FTemp < ~/src/hg/lib/protMrnaBlat.sql
    hgsql kgHg17FTemp -e 'load data local infile "protMrnaBlat.psl" into table protMrnaBlat'
    hgsql kgHg17FTemp -e 'create index tName on protMrnaBlat(tName)'
 
 # Create CDS files from protein/mRNA alignment results.
    hgsql kgHg17FTemp -N -e \
    'select qName,"_",tName,tStart+1,":",tEnd+3 from protMrnaBlat order by qName,tName,tEnd-tStart desc'\
    |sed 's/\t_\t/_/g'|sed 's/\t:\t/../g' >protMrna.cds
 
 # Create protMrna.psl with proteinID_mrnaID as query ID.
    cut -f 22-30 ../protMrna.out > j1.tmp
    cut -f 32-42 ../protMrna.out > j2.tmp
    cut -f 10,31 ../protMrna.out|sed -e 's/\t/_/g' >j3.tmp
    paste j1.tmp j3.tmp j2.tmp >protMrna.psl
    rm j1.tmp j2.tmp j3.tmp
 
 # Run mrnaToGene to create protMrna.gp
    bash
    mrnaToGene -cdsFile=protMrna.cds protMrna.psl protMrna.gp 2>protMrna.err >protMrna.log
    exit
 
 # Prepare refGene and all_mrna gp files.
 
    cd ..
    hgsql hg17 -N -e 'select * from refGene' >ref.gp
 
    hgsql hg17 -N -e \
    'select gbCdnaInfo.acc,cds.name from gbCdnaInfo,cds,all_mrna where all_mrna.qName=gbCdnaInfo.acc and   gbCdnaInfo.cds=cds.id' \
    |sort -u > all_mrna.cds
 
    bash
    mrnaToGene -cdsFile=all_mrna.cds all_mrna.psl all_mrna.gp 2>all_mrna.err > all_mrna.log
    exit
 
 # Align proteins to RefSeq.
 
    overlapSelect -inCds -statsOutput -overlapThreshold=0.90 -selectFmt=psl -inFmt=genePred\
    protBlat.psl ref.gp ref.stat
    overlapSelect -inCds -dropped=refOut1.gp -overlapThreshold=0.90 -selectFmt=psl -inFmt=genePred\
    protBlat.psl ref.gp protRef.gp
 
    overlapSelect -mergeOutput -selectCds -dropped=protOut1.psl -overlapThreshold=0.80 -inFmt=psl\
    -selectFmt=genePred ref.gp protBlat.psl protRef.out
 
    cut -f 10,22 protRef.out | sort -u >spRef.tab
    cut -f 10 protRef.out    | sort -u >protRef.lis
 
    hgsql kgHg17FTemp -e 'drop table spRef'
    hgsql kgHg17FTemp <~/src/hg/lib/spRef.sql
    hgsql kgHg17FTemp -e 'load data local infile "spRef.tab" into table spRef'
 
 # Prepare and perform cluster runs for protein/RefSeq alignments
 
    ~/src/hg/protein/KGRef2.sh kgHg17F hg17 050415
    ~/src/hg/protein/KGRef3.sh kgHg17F hg17 050415
 
    cd kgBestRef
    ls out | sed -e 's/prot/do1 prot/g' >doall
 
    cat << '_EOF_' > do1
 echo processing $1
 cat out/$1/*.out >>protRefRaw.psl
 '_EOF_'
 
    chmod +x do*
    doall
 
 # Filter out low quality alignments.
    pslReps -nohead -singleHit -minAli=0.9 protRefRaw.psl protRefBlat.psl /dev/null
    cut -f 10,14 protRefBlat.psl |sort -u >protRef.lis
    wc protRef.lis
 
    hgsql kgHg17FTemp -e 'drop table protRefBlat'
    hgsql kgHg17FTemp < ~/src/hg/lib/protRefBlat.sql
    hgsql kgHg17FTemp -e 'load data local infile "protRefBlat.psl" into table protRefBlat'
    hgsql kgHg17FTemp -e 'create index tName on protRefBlat(tName)'
 
 # Run gene-check to filter out invalid gp entries
    cd /cluster/data/hg17/bed/kgHg17F
    cat ref.gp kgBestMrna/protMrna.gp all_mrna.gp >kgCandidate0.gp
    gene-check  -incl-ok -ok-genepred-out kgCandidate0.passed.gp -nib-dir /cluster/data/hg17/nib kgCandidate0.gp kgCandidate0.check
 
    hgsql kgHg17FTemp -e 'drop table kgCandidate0'
    hgsql kgHg17FTemp < ~/src/hg/lib/kgCandidate0.sql 
    hgsql kgHg17FTemp -e  'load data local infile "kgCandidate0.gp" into table kgCandidate0'
 
    hgsql kgHg17FTemp -e 'drop table geneCheck'
    hgsql kgHg17FTemp < ~/src/hg/lib/geneCheck.sql
    hgsql kgHg17FTemp -e  'load data local infile "kgCandidate0.check" into table geneCheck ignore 2 lines'
 
 
 # Run kgCheck to get all KG candidates that pass the KG gene check criteria
 
    kgCheck kgHg17FTemp hg17 kgCandidate0 geneCheck kgCandidate.tab
    hgsql kgHg17FTemp -e  'drop table kgCandidate'
    hgsql kgHg17FTemp < ~/src/hg/lib/kgCandidate.sql
    hgsql kgHg17FTemp -e  'load data local infile "kgCandidate.tab" into table kgCandidate'
    hgsql kgHg17FTemp -e 'create index alignID on kgCandidate(alignID)'
 
 # ####### NEXT TIME AROUND PUT IN AN EXTRA STEP TO BRING IN ITEMS ON A "PUT BACK" LIST
 # FOR SPECIAL CASES LIKE SELENOCYSTEINE, NON-AUG INITIATION CODON, RIBOSOMAL SLIPPAGE, ETC.
 # #######
 
 # Construct the kgCandidateX table that has alignID in the name field. 
    cut -f 2-10 kgCandidate.tab >j2.tmp
    cut -f 11 kgCandidate.tab >j1.tmp
    paste j1.tmp j2.tmp >kgCandidateX.tab
 
    hgsql kgHg17FTemp -e  'drop table kgCandidateX'
    hgsql kgHg17FTemp < ~/src/hg/lib/kgCandidateX.sql
    hgsql kgHg17FTemp -e  'load data local infile "kgCandidateX.tab" into table kgCandidateX'
 
 # Score protein/mRna and protein/RefSeq alignments
 
    kgResultBestMrna2 050415 kgHg17FTemp hg17|sort -u >protMrnaBlatScore.tab
    kgResultBestRef2  050415 kgHg17FTemp hg17|sort -u >protRefScore.tab
 
 # Combine scoring results and load them into temp DB.
    cat protMrnaBlatScore.tab protRefScore.tab >protMrnaScore.tab
    hgsql kgHg17FTemp -e 'drop table protMrnaScore'
    hgsql kgHg17FTemp < ~/src/hg/lib/protMrnaScore.sql
    hgsql kgHg17FTemp -e 'load data local infile "protMrnaScore.tab" into table protMrnaScore'
    hgsql kgHg17FTemp -e 'create index mrnaAcc on protMrnaScore(mrnaAcc)'
 
 
 # Run kgGetCds to get CDS structure of each gene
 
    kgGetCds kgHg17FTemp kgCandidateX jY.tmp
    cat jY.tmp |sort -u >kgCandidateY.tab
    rm jY.tmp
    hgsql kgHg17FTemp -e  'drop table kgCandidateY'
    hgsql kgHg17FTemp < ~/src/hg/lib/kgCandidateY.sql
    hgsql kgHg17FTemp -e  'load data local infile "kgCandidateY.tab" into table kgCandidateY'
 
 # Run kgPickPrep to replace long cds structure string with cdsId.
    kgPickPrep kgHg17FTemp kgCandidateZ.tab
    hgsql kgHg17FTemp -e  'drop table kgCandidateZ'
    hgsql kgHg17FTemp < ~/src/hg/lib/kgCandidateZ.sql
    hgsql kgHg17FTemp -e  'load data local infile "kgCandidateZ.tab" into table kgCandidateZ'
    hgsql kgHg17FTemp -e 'create index cdsId on kgCandidateZ(cdsId)'
 
 # Run kgPick to pick the representative a mrna/protein pair for each unique CDS structure.
 
    kgPick kgHg17FTemp hg17 proteins050415 kg3.tmp dupSpMrna.tmp
    sort -u dupSpMrna.tmp >dupSpMrna.tab
 
 # Sort KG genes to make the kg3.gp table file.
    ~/kent/src/hg/protein/sortKg.pl kg3.tmp >kg3.gp
 
    hgsql kgHg17FTemp -e  'drop table knownGene'
    hgsql kgHg17FTemp < ~/src/hg/lib/knownGene.sql
    hgsql kgHg17FTemp -e  'load data local infile "kg3.gp" into table knownGene'
 
    hgsql hg17 -e  'drop table kg3'
    hgsql hg17 < ~/src/hg/lib/kg3.sql
    hgsql hg17 -e  'load data local infile "kg3.gp" into table kg3'
 
 # Perform analysis before renaming the kg3 table to knownGene.
 
 # Load data into hg17 knownGene table.
    hgsql hg17 -e  'drop table knownGene'
    hgsql hg17 < ~/src/hg/lib/knownGene.sql
    hgsql hg17 -e  'load data local infile "kg3.gp" into table knownGene'
 
 # Build knownGeneMrna and knownGenePep tables.
 
    kgPepMrna kgHg17FTemp hg17 050415
    hgsql hg17 -e  'drop table knownGeneMrna'
    hgsql hg17 < ~/src/hg/lib/knownGeneMrna.sql
    hgsql hg17 -e  'load data local infile "knownGeneMrna.tab" into table knownGeneMrna'
    hgsql hg17 -e  'drop table knownGenePep'
    hgsql hg17 < ~/src/hg/lib/knownGenePep.sql
    hgsql hg17 -e  'load data local infile "knownGenePep.tab" into table knownGenePep'
 
 
 # Build kgXref table
 
    kgXref2 kgHg17FTemp 050415 hg17
 
    hgsql hg17 -e  'drop table kgXref'
    hgsql hg17 < ~/src/hg/lib/kgXref.sql
    hgsql hg17 -e  'load data local infile "kgXref.tab" into table kgXref'
 
 # Build spMrna table
 
    hgsql hg17 -N -e 'select name, proteinID from knownGene' >kgSpMrna.tab
 
    hgsql hg17 -e  'drop table spMrna'
    hgsql hg17 <~/src/hg/lib/spMrna.sql
    hgsql hg17 -e 'load data local infile "kgSpMrna.tab" into table spMrna'
 
 # Build kgProtMap table
 
     ~/src/hg/protein/kgProtMap2.sh kgHg17F hg17 050415
 
 # Update and clean up kgResultBestMrna2.c and then check it in.
 
 #####################################
 # Build alias tables.		DONE 5/18/05 Fan.
 #	kgAliasM reads from proteins050415.hugo.symbol, proteins050415.hugo.aliases
 #	proteins050415.hugo.withdraws, hg17.kgXref.kgID
 #	to create kgAliasM.tab and geneAlias.tab
 #	by picking out those kgID items from kgXref where
 #	kgXref.geneSymbol == hugo.symbol
 
    kgAliasM hg17 proteins050415
 
 #	kgAliasKgXref reads from hg17.knownGene.proteinID,
 #	hg17.knownGene.name, hg17.kgXref.geneSymbol
 #	to create kgAliasKgXref.tab
 
    kgAliasKgXref hg17
 
 
 #	kgAliasRefseq reads from hg17.knownGene.name,
 #	hg17.knownGene.proteinID, hg17.kgXref.refseq
 #	to create kgAliasRefseq.tab
 
    kgAliasRefseq hg17
 
    hgsql sp050415 -N -e 'select name,gene.val from hg17.knownGene,displayId,gene where displayId.val=proteinID and displayId.acc=gene.acc' \
    | sort -u  > kgAliasP.tab
 
    hgsql hg17 -N -e 'select name, name from knownGene' >kgAliasDup.tab
    hgsql hg17 -N -e 'select mrnaID, dupMrnaID from dupSpMrna' >>kgAliasDup.tab
    
    cat kgAliasM.tab kgAliasRefseq.tab kgAliasKgXref.tab kgAliasP.tab kgAliasDup.tab| \
    sort |uniq > kgAlias.tab
 
    hgsql -e "drop table kgAlias;" hg17 
    hgsql hg17 < ~/kent/src/hg/lib/kgAlias.sql
    hgsql hg17 -e 'LOAD DATA local INFILE "kgAlias.tab" into table kgAlias' 
 
 #	kgProtAlias reads from hg17.knownGene.name,
 #	hg17.knownGene.proteinID, hg17.knownGene.alignID,
 #	proteins050415.spXref3.accession, proteins050415.spSecondaryID, proteins050415.pdbSP.pdb
 #	to create kgProtAlias.tab
 #
 
    kgProtAlias hg17 050415
 
    hgsql hg17 -N -e \
    'select kgID, spDisplayID, protAcc from kgXref where protAcc != ""'\
    | sort -u >kgProtAliasNCBI.tab
 
 # include variant splice protein IDs
    
    hgsql hg17 -N -e \
    'select name, proteinID, parAcc from knownGene,sp050415.varAcc where varAcc=proteinID'\
    |sort -u >kgProtAliasDup.tab
 # include duplicate protein IDs from dupSpMrna table
    hgsql hg17 -N -e \
    'select name, knownGene.proteinID, dupProteinID from knownGene, dupSpMrna where name=mrnaID'\
    |sort -u >>kgProtAliasDup.tab
 
 # catch parent acc from dupProteinID too
    hgsql hg17 -N -e\
    'select name, knownGene.proteinID, parAcc from knownGene,dupSpMrna,sp050415.varAcc where name=mrnaID and dupProteinID=varAcc.varAcc'\
    |sort -u >>kgProtAliasDup.tab
     cat kgProtAliasNCBI.tab kgProtAlias.tab kgProtAliasDup.tab | sort -u > kgProtAliasAll.tab
 
     echo "`date` creating table kgProtAlias"
     hgsql hg17 -e "drop table kgProtAlias;"
     hgsql hg17 <~/src/hg/lib/kgProtAlias.sql; 
     hgsql hg17 -e 'LOAD DATA local INFILE "kgProtAliasAll.tab" into table kgProtAlias;'  
 
 # Build kgSpAlias table
 
     hgsql hg17 -e \
     'select kgXref.kgID, spID, alias from kgXref, kgAlias where kgXref.kgID=kgAlias.kgID' >j.tmp
     hgsql hg17 -e \
     'select kgXref.kgID, spID, alias from kgXref, kgProtAlias where kgXref.kgID=kgProtAlias.kgID'\
     >>j.tmp
     cat j.tmp|sort -u |grep -v 'kgID' >hg17.kgSpAlias.tab
     rm j.tmp
 
     hgsql hg17 -e 'drop table kgSpAlias';
     hgsql hg17 < ~/src/hg/lib/kgSpAlias.sql
     hgsql hg17 -e 'load data local infile "hg17.kgSpAlias.tab" into table kgSpAlias'
 
 # MAKE FOLDUTR TABLES (DONE 2005-05-19, Fan)
 # First set up directory structure and extract UTR sequence on hgwdev
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mkdir rnaStruct.2005-05-18
     rm rnaStruct
     ln -s rnaStruct.2005-05-18 rnaStruct
     cd rnaStruct
     mkdir -p utr3/split utr5/split utr3/fold utr5/fold
     utrFa hg17 knownGene utr3 utr3/utr.fa
     utrFa hg17 knownGene utr5 utr5/utr.fa
 
 # Split up files and make files that define job.
     ssh kk
     cd /cluster/data/hg17/bed/rnaStruct
     faSplit sequence utr3/utr.fa 50000 utr3/split/s
     faSplit sequence utr5/utr.fa 50000 utr5/split/s
     ls -1 utr3/split > utr3/in.lst
     ls -1 utr5/split > utr5/in.lst
     cd utr3
     cat > gsub <<end
 #LOOP
 rnaFoldBig split/\$(path1) fold
 #ENDLOOP
 end
     cp gsub ../utr5
 
 # Do cluster run for 3' UTRs
     gensub2 in.lst single gsub spec
     para create spec
     para try
     para push
 # Completed: 35692 of 35692 jobs
 # CPU time in finished jobs:    1272085s   21201.42m   353.36h   14.72d  0.040 y
 # IO & Wait Time:                102447s    1707.45m    28.46h    1.19d  0.003 y
 # Average job time:                  39s       0.64m     0.01h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:            6554s     109.23m     1.82h    0.08d
 # Submission to last job:          9100s     151.67m     2.53h    0.11d
 
 # Do cluster run for 5' UTRs 
     cd ../utr5
     gensub2 in.lst single gsub spec
     para create spec
     para try
     para push
 
 # Completed: 33693 of 33693 jobs
 # CPU time in finished jobs:     393764s    6562.74m   109.38h    4.56d  0.012 y
 # IO & Wait Time:                126205s    2103.41m    35.06h    1.46d  0.004 y
 # Average job time:                  15s       0.26m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:           51595s     859.92m    14.33h    0.60d
 # Submission to last job:         52057s     867.62m    14.46h    0.60d
 
 # Load database
     ssh hgwdev
     cd /cluster/data/hg17/bed/rnaStruct/utr5
     hgLoadRnaFold hg17 foldUtr5 fold
     cd ../utr3
     hgLoadRnaFold hg17 foldUtr3 fold
 
 # Clean up
     rm -r split fold err batch.bak
     cd ../utr5
     rm -r split fold err batch.bak
 
 # Build KEGG pathway tables.  DONE 5/19/05.  Fan.
    ssh hgwdev
    cd /cluster/store10/kg/kgHg17F
    md kegg
    cd kegg
 
    ~/src/hg/protein/KGpath.sh kgHg17F hg17 050415
 
    hgsql hg17 -e "drop table keggMapDesc"
    hgsql hg17 -e "drop table keggPathway"
    hgsql hg17 <~/src/hg/lib/keggMapDesc.sql
    hgsql hg17 <~/src/hg/lib/keggPathway.sql
    hgsql hg17 -e 'load data local infile "keggMapDesc.tab" into table keggMapDesc'
    hgsql hg17 -e 'load data local infile "keggPathway.tab" into table keggPathway'
   
 # Build CGAP pathway tables
 # RELOAD cgapAlias TABLE AFTER REMOVING REPLICATE ROWS (hartera, 2005-07-26)
 # duplicate rows. (hartera, 2005-07-26)
 # RELOADED cgapAlias AGAIN AS TOO MANY ROWS REMOVED BEFORE (hartera, 2005-10-06)
 
    cd ..
    ~/src/hg/protein/KGcgap.sh kgHg17F hg17 050415
    hgsql hg17 -e "drop table cgapAlias"
    hgsql hg17 -e "drop table cgapBiocDesc"
    hgsql hg17 -e "drop table cgapBiocPathway"
    hgsql hg17 <~/src/hg/lib/cgapAlias.sql
    hgsql hg17 <~/src/hg/lib/cgapBiocDesc.sql
    hgsql hg17 <~/src/hg/lib/cgapBiocPathway.sql
    hgsql hg17 -e 'load data local infile "cgapAlias.tab" \
                  into table cgapAlias'
    hgsql hg17 -e 'load data local infile "cgapBIOCARTAdescSorted.tab" into table cgapBiocDesc'
    hgsql hg17 -e 'load data local infile "cgapBIOCARTA.tab" into table cgapBiocPathway'
 
    # RELOAD cgapAlias TABLE. Sort and reload alias tab file to remove 
    # duplicate rows. (hartera, 2005-07-26)
    # DO TABLE RELOAD AGAIN AS sort -nu REMOVES MORE ROWS THAN sort -u
    # OR sort -n | uniq.
    #USE sort -n then uniq TO SORT ON THE IDs AND THEN UNIQ(hartera, 2005-10-06)
    cd /cluster/store10/kg/kgHg17F
    hgsql hg17 -e "drop table cgapAlias"
    # cgapAlias.tab has replicated rows so sort and unique before loading
    sort -n cgapAlias.tab | uniq > cgapAliasSorted.tab
    hgsql hg17 < ~/kent/src/hg/lib/cgapAlias.sql 
    hgsql hg17 -e 'load data local infile "cgapAliasSorted.tab" \
                  into table cgapAlias'
 
 # LOAD ENSEMBL GENES (DONE, 5/23/05, Fan)
 # Ensembl changed things again! Please note there are two subtle changes to make it work.
 
     mkdir /cluster/data/hg17/bed/ensembl
     cd /cluster/data/hg17/bed/ensembl
     mkdir new
     cd new
 
     # Get the ensembl protein data from 
     # http://www.ensembl.org/Homo_sapiens/martview
     # Follow this sequence through the pages:
     # Page 1) Make sure that the Homo_sapiens choice is selected. Hit next.
     # Page 2) Uncheck the "Limit to" box in the region choice. Then hit next.
     # Page 3) Choose the "Structures" box. 
     # Page 4) Choose GTF as the ouput.  choose gzip compression.  hit export.
     # Save as ensemblGene.gtf.gz
 
     # This time, there are some extra lines, like '		1;', 
     # that are causing problems, so added an extra filter in the beginning 
     # to get rid of them.
     # Ensembl handles random chromosomes differently than us, so we
     # strip this data.  Fortunately it just loses a couple of genes.
     # Add "chr" to front of each line in the gene data gtf file to make 
     # it compatible with our software.
     # Finally, get rid of the ".1" or ".2" after the name
 
 cat ensemblGene.gtf |sed -e 's/\t\t/xxxxx/g' \
     |grep -v xxxxx \
     | grep -v ^6_DR51 \
     | grep -v ^DR51 \
     | grep -v ^DR52 \
     | grep -v ^DR53 \
     | grep -v _NT_ \
     | perl -wpe 's/^([0-9]|X|Y|Un|MT)/chr$1/ \
                  || die "Line $. doesnt start with human chrom:\n$_"' \
     | sed -e 's/chrMT/chrM/g' \
     | sed -e 's/\..\"/\"/g' \
 >ensGene.gtf
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/ensembl/new
     /cluster/bin/i386/ldHgGene hg17 ensGene ensGene.gtf
 # Read 33581 transcripts in 699580 lines in 1 files
 # 33581 groups 25 seqs 1 sources 4 feature types
 # 33581 gene predictions
 
     # ensGtp associates geneId/transcriptId/proteinId for hgPepPred and 
     # hgKnownToSuper.  Use ensMart to create it as above, except:
     # Page 3) Choose the "Features" box. In "Ensembl Attributes", check 
     # Ensembl Gene ID, Ensembl Transcript ID, Ensembl Peptide ID.  
     # Choose Text, tab-separated as the output format.  Result name ensGtp.
     # Save file as ensGtp.txt.gz
     gunzip ensGtp.txt.gz
     hgsql hg17 -e 'drop table ensGtp'
     hgsql hg17 < ~/kent/src/hg/lib/ensGtp.sql
     hgsql hg17 -e 'load data local infile "ensGtp.txt" into table ensGtp ignore 1 lines'
 
 # ensMart has some problem with the resulting ensemblPep.fa.gz, so use different
 # processing step instead:
 
     wget -timestamp \
     ftp://ftp.ensembl.org/pub/current_human/data/fasta/pep/Homo_sapiens.NCBI35.may.pep.fa.gz
     zcat Homo_sapiens.NCBI35.may.pep.fa.gz | sed -e "s/transcript:/\n>/g" | grep -v 'gene:' >ensPep.fa
     faToTab -type=protein ensPep.fa ensPep.tab
     hgsql hg17 -e 'drop table ensPep'
     hgsql hg17 < ~/kent/src/hg/lib/ensPep.sql
     hgsql hg17 -e 'load data local infile "ensPep.tab" into table ensPep'
 
 # kept the following, just in case Ensembl fixed the problem in the future
     # Load Ensembl peptides:
     # Get them from ensembl as above in the gene section except for
     # Page 3) Choose the "Sequences" box. 
     # Page 4) Transcripts/Proteins.  Peptide.  Format = FASTA.
     # Save file as ensemblPep.fa.gz
 # gunzip ensemblPep.fa.gz
 # hgPepPred hg17 ensembl ensemblPep.fa
 
 # UPDATE GENE SORTER TABLES (AKA: FAMILY BROWSER) (STARTED - 2005-05-21, DONE 2005-05-23 - Fan)
 #	This should be done after knownGene tables are complete from known gene
 #	process.
 #
 # Cluster together various alt-splicing isoforms.
 #	Creates the knownIsoforms and knownCanonical tables
 ssh hgwdev
 mkdir /cluster/data/hg17/bed/geneSorter.2005-05-21
 # remove old symbolic link
 rm /cluster/data/hg17/bed/geneSorter
 ln -s /cluster/data/hg17/bed/geneSorter.2005-05-21 \
 	/cluster/data/hg17/bed/geneSorter
 cd /cluster/data/hg17/bed/geneSorter
 hgClusterGenes hg17 knownGene knownIsoforms knownCanonical
 
 # Extract peptides from knownGenes into fasta file
 # and create a blast database out of them.
 mkdir /cluster/data/hg17/bed/geneSorter/blastp
 cd /cluster/data/hg17/bed/geneSorter/blastp
 pepPredToFa hg17 knownGenePep known.faa
 #	You may need to build this binary in src/hg/near/pepPredToFa
 /scratch/blast/formatdb -i known.faa -t known -n known
 #	This command is in /projects/compbio/bin/$MACH/formatdb
 
 # Copy over database to bluearc
 rm -fr /cluster/bluearc/hg17/blastp
 mkdir -p /cluster/bluearc/hg17/blastp
 cp -p /cluster/data/hg17/bed/geneSorter/blastp/known.* \
 	/cluster/bluearc/hg17/blastp
 
 # Split up fasta file into bite sized chunks for cluster
 cd /cluster/data/hg17/bed/geneSorter/blastp
 mkdir split
 faSplit sequence known.faa 8000 split/kg
 
 # Make parasol run directory
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/self
 cd /cluster/data/hg17/bed/geneSorter/blastp/self
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 -p blastp -d /cluster/bluearc/hg17/blastp/known -i $1 -o $2 \
 -e 0.01 -m 8 -b 1000
 '_EOF_'
     # << keep emacs happy
 chmod +x blastSome
 
 # Make gensub2 file
 cat  << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #	'ls ../../split/*.fa' is too much, hence the echo
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 # Wait a couple of minutes, and do a para check,  if all is good
 # then do a
 para push
 # This should finish in ~15 minutes if the cluster is free.
 # Completed: 7739 of 7739 jobs
 # CPU time in finished jobs:     150459s    2507.64m    41.79h    1.74d  0.005 y
 # IO & Wait Time:                 22325s     372.09m     6.20h    0.26d  0.001 y
 # Average job time:                  22s       0.37m     0.01h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:             198s       3.30m     0.06h    0.00d
 # Submission to last job:          2019s      33.65m     0.56h    0.02d
 
 # Load into database.  This takes about 30 minutes
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/self/run/out
 time hgLoadBlastTab hg17 knownBlastTab *.tab
 # Scanning through 7739 files
 # Loading database with 9836439 rows
 # 232.300u 42.580s 23:13.41 19.7% 0+0k 0+0io 205pf+0w
 
 cd /cluster/data/hg17/bed/geneSorter
 # Create table that maps between known genes and RefSeq
 hgMapToGene hg17 refGene knownGene knownToRefSeq
 #	may need to build this command in src/hg/near/hgMapToGene
 #	hgsql -e "select count(*) from knownToRefSeq;" hg17
 #	row count changed 34667
 
 # Create table that maps between known genes and LocusLink
 hgsql --skip-column-names -e "select mrnaAcc,locusLinkId from refLink" hg17 \
 	> refToLl.txt
 hgMapToGene hg17 refGene knownGene knownToLocusLink -lookup=refToLl.txt
 #	hgsql -e "select count(*) from knownToLocusLink;" hg17
 #	row count changed to 34667
 
 # Create table that maps between known genes and Pfam domains
 hgMapViaSwissProt hg17 knownGene name proteinID Pfam knownToPfam
 #	hgsql -e "select count(*) from knownToPfam;" hg17
 #	row count changed to 36010
 
 # Create table to map between known genes and GNF Atlas2
 # expression data.
     hgMapToGene hg17 gnfAtlas2 knownGene knownToGnfAtlas2 '-type=bed 12'
 #	hgsql -e "select count(*) from knownToGnfAtlas2;" hg17
 #	row count changed to 32381
 
 # Create expression distance table - takes about an hour
     hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
     	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
 	-lookup=knownToGnfAtlas2 &
 # Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
 # Got 32381 unique elements in hgFixed.gnfHumanAtlas2MedianRatio
 
 #	hgsql -e "select count(*) from gnfAtlas2Distance;" hg17
 #	row count changed to 32381000
 
 # Create a table that maps between known genes and 
 # the nice affy expression data.
 hgMapToGene "-type=bed 12" hg17 affyUclaNorm knownGene knownToU133
 #	hgsql -e "select count(*) from knownToU133;" hg17
 #	row count changed to 32886
 
 # Create expression distance table.  This will take about 2.5 hours
 cd /tmp
 cp -p ~/kent/src/hg/near/hgExpDistance/affyUcla.weight .
 time hgExpDistance hg17 affyUclaNorm affyUclaExp knownExpDistance \
 	-weights=affyUcla.weight -lookup=knownToU133 &
 # Have 43039 elements in affyUclaNorm
 # 211 genes, 42 weights, 26.500000 total wieght
 # Got 32886 unique elements in affyUclaNorm
 
 # Create table that maps between known genes and 
 # the GNF data.
 cd /tmp
 hgMapToGene hg17 affyU95 knownGene knownToU95
 #	row count changed to 17501
 #	hgFixed.gnfHumanU95Exps argument is unused, no need to exist
 hgExpDistance hg17 hgFixed.gnfHumanU95MedianRatio \
 	hgFixed.gnfHumanU95Exps gnfU95Distance  -lookup=knownToU95 &
 # Have 11545 elements in hgFixed.gnfHumanU95MedianRatio
 # Got 16450 unique elements in hgFixed.gnfHumanU95MedianRatio
 #	row count changed to 16450000  
 
 # Create known gene mapping table and expression distance tables
 # for GNF Atlas 2.  (The hgExpDistance takes only 10 minutes.)
 
 hgMapToGene hg17 affyGnf1h knownGene knownToGnf1h
 hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
 	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
 	-lookup=knownToGnf1h &
 # Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
 # Got 8814 unique elements in hgFixed.gnfHumanAtlas2MedianRatio
 
 cd /cluster/data/hg17/bed/geneSorter
 hgMapToGene hg17 affyU133Plus2 knownGene knownToU133Plus2
 #	row count changed to 35055 
 
 #### UPDATE GO DATABASE (DONE 5/21/05 Fan)
 
 # Download the terms and make the database.
 ssh hgwdev
 mkdir /cluster/store1/geneOntology/20050521
 cd /cluster/store1/geneOntology/20050521
 
 wget --timestamping http://www.godatabase.org/dev/database/archive/latest/go_200504-assocdb-data.gz
 
 hgsql mysql <<end
 create database go050521;
 end
 zcat go_*data.gz | sed -e 's/ENGINE=MyISAM DEFAULT CHARSET=latin1/TYPE=MyISAM/g' >j.tmp
 hgsql go050521 <j.tmp
 rm j.tmp
 
 wget --timestamping ftp://ftp.geneontology.org/pub/go/gene-associations/gene_association.goa_uniprot.gz
 
 zcat gene_association.goa_uniprot.gz | hgGoAssociation go050521 goaPart stdin 
 # Passed 5589891 of 6584507 of 6584507, 84.89%
 
 # Ask sys-admin to switch the database pointer go to point to go050521.
 
 cd /cluster/data/hg17/bed/geneSorter
 
 # Rebuilt Ensembl Gene tables.  See documentation (5/23/05 Fan) above.
 
 # Create knownToEnsembl column
 hgMapToGene hg17 ensGene knownGene knownToEnsembl
 #	table row count went from previous version: 38251 to 35436
 
 # Make knownToCdsSnp table 
   ssh hgwdev
   nice hgMapToGene hg17 snp knownGene knownToCdsSnp -all -cds
 # row count 94394
 # approx. 5 minutes running time
 
 # C.ELEGANS BLASTP FOR GENE SORTER 
     # Make C. elegans ortholog column using blastp on wormpep.
     # First make C. elegans protein database and copy it to iscratch/i
     # if it doesn't exist already:
 
 # The following section is done during mm6 build already.
 #    ssh eieio
 #    mkdir /cluster/data/ce2/bed/blastp
 #    cd /cluster/data/ce2/bed/blastp
 #    # Point a web browser at ftp://ftp.sanger.ac.uk/pub/databases/wormpep/
 #    # to find out the latest version.  Then use that in place of 142 below.
 #    wget -O wormPep142.faa ftp://ftp.sanger.ac.uk/pub/databases/wormpep/wormpep142/wormpep142
 #    formatdb -i wormPep142.faa -t wormPep142 -n wormPep142
 #    ssh kkr1u00
 #    if (-e /iscratch/i/ce2/blastp) then
 #      rm -r /iscratch/i/ce2/blastp
 #    endif
 #    mkdir -p /iscratch/i/ce2/blastp
 #    cp /cluster/data/ce2/bed/blastp/wormPep142.p?? /iscratch/i/ce2/blastp
 #    iSync
 
     # Make parasol run directory 
     ssh kk
     mkdir -p /cluster/data/hg17/bed/blastp/ce2/run/out
     cd /cluster/data/hg17/bed/blastp/ce2/run
     # Make blast script
     cat > blastSome <<end
 #!/bin/csh
 setenv BLASTMAT /iscratch/i/blast/data
 /iscratch/i/blast/blastall -p blastp -d /iscratch/i/ce2/blastp/wormPep142 -i \$1 -o \$2 -e 0.01 -m 8 -b 1
 end
     chmod a+x blastSome
     # Make gensub2 file
     cat > gsub <<end
 #LOOP
 blastSome {check in line+ \$(path1)} {check out line out/\$(root1).tab}
 #ENDLOOP
 end
     # Create parasol batch
     ls -1S /cluster/data/hg17/bed/geneSorter/blastp/split >split.lst
     #EDIT split.lst to add "../../../geneSorter/blastp/split/" in front of "kg"
     gensub2 split.lst single gsub spec
     para create spec
     para try, check, push, check, ...
 # Completed: 7739 of 7739 jobs
 # CPU time in finished jobs:      60973s    1016.22m    16.94h    0.71d  0.002 y
 # IO & Wait Time:                 21292s     354.86m     5.91h    0.25d  0.001 y
 # Average job time:                  11s       0.18m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              50s       0.83m     0.01h    0.00d
 # Submission to last job:           570s       9.50m     0.16h    0.01d
 
 # Load into database.  
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastp/ce2/run/out
     hgLoadBlastTab hg17 ceBlastTab -maxPer=1 *.tab
 # Scanning through 7739 files
 # Loading database with 25706 rows
 
 # Make mouse ortholog column using blastp on mouse known genes.
 # First make mouse protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This already exists.  See makeMm6.doc for procedure
 #	the directory: /cluster/bluearc/scratch/mus/mm6/blastp should have data
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/mm6
 cd /cluster/data/hg17/bed/geneSorter/blastp/mm6
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 -p blastp -d /cluster/panasas/home/store/mm6/blastp/known \
 -i $1 -o $2 -e 0.001 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #	this echo trick is used because otherwise the command line is
 #	too long and you can not do a simple ls
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 
 # Completed: 7739 of 7739 jobs
 # CPU time in finished jobs:      65337s    1088.95m    18.15h    0.76d  0.002 y
 # IO & Wait Time:                 20794s     346.56m     5.78h    0.24d  0.001 y
 # Average job time:                  11s       0.19m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              80s       1.33m     0.02h    0.00d
 # Submission to last job:           598s       9.97m     0.17h    0.01d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/mm6/run/out
 hgLoadBlastTab hg17 mmBlastTab -maxPer=1 *.tab
 # Scanning through 7739 files 
 #	row count changed to 32880  
 
 # Make rat ortholog column using blastp on rat known genes.
 # First make rat protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This already exists.  See makeRn3.doc for procedure.
 #	Files were put in this directory: /cluster/bluearc/rn3/blastp/
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/rn3
 cd /cluster/data/hg17/bed/geneSorter/blastp/rn3
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 -p blastp -d /cluster/bluearc/rn3/blastp/known \
 -i $1 -o $2 -e 0.001 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #	this echo trick is used because otherwise the command line is
 #	too long and you can not do a simple ls
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7739 of 7739 jobs
 # CPU time in finished jobs:      28325s     472.08m     7.87h    0.33d  0.001 y
 # IO & Wait Time:                 20416s     340.27m     5.67h    0.24d  0.001 y
 # Average job time:                   6s       0.10m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              24s       0.40m     0.01h    0.00d
 # Submission to last job:           617s      10.28m     0.17h    0.01d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/rn3/run/out
 hgLoadBlastTab hg17 rnBlastTab -maxPer=1 *.tab
 # Scanning through 7739 files
 # Loading database with 24140 rows
 
 # ZEBRAFISH BLASTP FOR GENE SORTER 
     # Make Danio rerio (zebrafish) ortholog column using blastp on Ensembl.
     # First make protein database and copy it to iscratch/i
     # if it doesn't exist already:
     ssh kkstore
     mkdir /cluster/data/danRer2/bed/blastp
     cd /cluster/data/danRer2/bed/blastp
     wget ftp://ftp.ensembl.org/pub/current_zebrafish/data/fasta/pep/Danio_rerio.ZFISH4.may.pep.fa.gz 
     zcat Dan*.pep.fa.gz > ensembl.faa
     /scratch/blast/formatdb -i ensembl.faa -t ensembl -n ensembl
     ssh kkr1u00
     if (-e /iscratch/i/danRer2/blastp) then
       rm -r /iscratch/i/danRer2/blastp
     endif
     mkdir -p /iscratch/i/danRer2/blastp
     cp /cluster/data/danRer2/bed/blastp/ensembl.p?? /iscratch/i/danRer2/blastp
     iSync
 
     # Make parasol run directory 
     ssh kk
     mkdir -p /cluster/data/hg17/bed/blastp/danRer2/run/out
     cd /cluster/data/hg17/bed/blastp/danRer2/run
     # Make blast script
     cat > blastSome <<end
 #!/bin/csh
 setenv BLASTMAT /iscratch/i/blast/data
 /iscratch/i/blast/blastall -p blastp -d /iscratch/i/danRer2/blastp/ensembl -i \$1 -o \$2 -e 0.005 -m 8 -b 1
 end
     chmod a+x blastSome
     # Make gensub2 file
     cat > gsub <<end
 #LOOP
 blastSome {check in line+ \$(path1)} {check out line out/\$(root1).tab}
 #ENDLOOP
 end
     # Create parasol batch
     ls -1S /cluster/data/hg17/bed/geneSorter/blastp/split \
     |sed -e 's=kg=../../../geneSorter/blastp/split/kg=g' > split.lst
     gensub2 split.lst single gsub spec
     para create spec
     para try, check, push, check, ...
 # Completed: 7739 of 7739 jobs
 # CPU time in finished jobs:     113595s    1893.26m    31.55h    1.31d  0.004 y
 # IO & Wait Time:                 26231s     437.18m     7.29h    0.30d  0.001 y
 # Average job time:                  18s       0.30m     0.01h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              99s       1.65m     0.03h    0.00d
 # Submission to last job:           445s       7.42m     0.12h    0.01d
 
     # Load into database.  
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastp/danRer2/run/out
     hgLoadBlastTab hg17 drBlastTab -maxPer=1 *.tab
 # Scanning through 7739 files
 # Loading database with 30731 rows
 
 
 # Make Saccharomyces cerevisiae (yeast) ortholog column using blastp on RefSeq.
 # First make protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This is already done, see makeMm3.doc for procedure
 #	the directory: /cluster/bluearc/sc1/blastp should have data
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/sc1
 cd /cluster/data/hg17/bed/geneSorter/blastp/sc1
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 -p blastp -d /cluster/bluearc/sc1/blastp/sgd \
 -i $1 -o $2 -e 0.01 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ... 
 # Completed: 7739 of 7739 jobs
 # CPU time in finished jobs:      18630s     310.50m     5.17h    0.22d  0.001 y
 # IO & Wait Time:                 20776s     346.27m     5.77h    0.24d  0.001 y
 # Average job time:                   5s       0.08m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              15s       0.25m     0.00h    0.00d
 # Submission to last job:           295s       4.92m     0.08h    0.00d
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/sc1/run/out
 hgLoadBlastTab hg17 scBlastTab -maxPer=1 *.tab
 # Loading database with 16540 rows
 
 # Make Drosophila melanagaster ortholog column using blastp on FlyBase.
 # First make SwissProt protein database and copy it to cluster/bluearc
 # The following section was already done.
 # cd /cluster/data/dm1/bed
 # mkdir blastp
 # cd blastp
 #wget ftp://ftp.fruitfly.org/pub/download/dmel_RELEASE3-1/FASTA/whole_genome_translation_dmel_RELEASE3-1.FASTA.gz
 # zcat whole_ge*.gz | faFlyBaseToUcsc stdin flyBase.faa
 # formatdb -i flyBase.faa -t flyBase -n flyBase
 # if (-e /cluster/bluearc/dm1/blastp) then
 #    rm -r /cluster/bluearc/dm1/blastp
 # endif
 # mkdir -p /cluster/bluearc/dm1/blastp
 # cp /cluster/data/dm1/bed/blastp/flyBase.p?? /cluster/bluearc/dm1/blastp
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/dm1
 cd /cluster/data/hg17/bed/geneSorter/blastp/dm1
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 -p blastp -d /cluster/bluearc/dm1/blastp/flyBase \
 -i $1 -o $2 -e 0.01 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7739 of 7739 jobs
 # CPU time in finished jobs:      73518s    1225.30m    20.42h    0.85d  0.002 y
 # IO & Wait Time:                 45038s     750.63m    12.51h    0.52d  0.001 y
 # Average job time:                  15s       0.26m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              69s       1.15m     0.02h    0.00d
 # Submission to last job:           762s      12.70m     0.21h    0.01d 
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/dm1/run/out
 hgLoadBlastTab hg17 dmBlastTab -maxPer=1 *.tab
 # Loading database with 27212 rows
 
 # update knownToHInv table
 # Verified that there is now new release of HInv data.
 hgMapToGene hg17 HInvGeneMrna knownGene knownToHInv
 # count changed to 28851
 
 # The new KG process no longer need entries in knownGeneLink (used to store
 # info for DNA based RefSeqs.  So clean out the old data in knownGeneLink.
 
 hgsql hg17 -e "delete from knownGeneLink"
 
 #### RE-BUILD SUPERFAMILY RELATED TABLES (DONE - 2005-05-27 - Fan)
 
 # Download latest Superfamily data files and build the Superfamily DB
 # from supfam.mrc-lmb.cam.ac.uk
 
     mkdir /cluster/store10/superfamily/050524
     ln -s /cluster/store10/superfamily/050524 /cluster/data/superfamily/050524
     cd /cluster/data/superfamily/050524
 
 # ftp over the following two files:
 ass_22-May-2005.tab.gz
 supfam_22-May-2005.sql.gz
     gzip -d *.gz
 
 # Load the Superfamily database
     hgsql hg17 -e "create database superfam050524"
     nice hgsql superfam050524 < supfam_22-May-2005.sql &
 
 # This may take about an hour.
 
 # Make sure to add an index on id of the des table of superfam050524.
     hgsql superfam050524 -e "create index id on des(id);"
 
     hgsql superfam050524 < ~/src/hg/lib/sfAssign.sql
     hgsql superfam050524 -e 'load data local infile "ass_22-May-2005.tab" into table 
 superfam050524.sfAssign;'
 
 # Build or rebuild Superfamily track and create sf tables needed for PB
 
    hgsql hg17 < ~/src/hg/lib/sfAssign.sql
 
    cd /cluster/data/superfamily/050524  
    hgsql hg17 -e 'load data local infile "ass_22-May-2005.tab" into table hg17.sfAssign;'
 
 # If hg17.sfDes already exists, drop it.
 
    hgsql superfam050524 -N -e "select * from des" >sfDes.tab
    hgsql hg17 < ~/src/hg/lib/sfDes.sql
    hgsql hg17 -e 'load data local infile "sfDes.tab" into table sfDes'
 
 # If hg17.superfamily already exists, drop it.
    cd /cluster/data/hg17/bed
    mkdir /cluster/data/hg17/sf.2004-1128
    ln -s sf.2004-1128 sf
    hgSuperfam hg17 > sf.log
 
 # It is normal that many proteins does not have corresponding Superfamily entries.
 
 # If hg17.sfDescription exists, drop it.
 
    hgsql hg17 < ~/src/hg/lib/sfDescription.sql
    hgsql hg17 -e 'LOAD DATA local INFILE "sfDescription.tab"  into table hg17.sfDescription;'
 
 # Finally, load the superfamily table.
 
    hgLoadBed hg17 superfamily superfamily.tab -tab
 
 # Create knownToSuperfamily table
 # Note hs is changed into ht for this Superfamily release.
    
    cat /cluster/data/superfamily/050524/ass_22-May-2005.tab \
    | hgKnownToSuper hg17 hs stdin
 # created 25287 rows in knownToSuper
 
 # Build tables needed by pbGlobal in proteins050415
 
    cd /cluster/data/superfamily/050524  
    hgsql proteins050415 -e 'load data local infile "ass_22-May-2005.tab" into table sfAssign'
 
    hgsql proteins050415 -e 'load data local infile "sfDes.tab" into table sfDes'
 
    cd /cluster/store10/kg/kgHg17F
    hgsql proteins050415 -e 'load data local infile "ensemblXref.tab" into table ensemblXref'
 
 # These sf tables and ensemblXref3 are needed for non-HMR KG proteins.
 # Should add content of ensemblXref3 of mm6 after it is done.
 # And similarly for rn4 and possibly for other non-HMR species.
 # CCDS <-> knownGene mapping need to be updated (markd 2005-05-29)
 # this should be part of the known gene build
     /cluster/data/genbank/bin/i386/mkCcdsGeneMap -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap
 
 # AUGUSTUS GENES (DONE 6/1/2005 Andy)
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mkdir augustus
     cd augustus/
     wget http://augustus.gobics.de/predictions/hg17/hg17.allchr.augustus.gtf.gz
     cp /cluster/data/dm2/bed/augustus/cleanAugustus.awk .
     zcat hg17.allchr.augustus.gtf.gz | awk -f cleanAugustus.awk | gzip > hg17.allchr.augustus.clean.gtf.gz
     ldHgGene -gtf hg17 augustus hg17.allchr.augustus.clean.gtf.gz
     rm hg17.allchr.augustus.gtf.gz
 
 # MAKE Mouse Proteins track (DONE for chr13 braney ~5/25/05)
     ssh kkstore01
     mkdir -p /cluster/data/hg17/blastDb
     cd /cluster/data/hg17/blastDb
     awk "{print \$2}" ../*/chr*/*.lft > subChr.lst
     for i in `cat subChr.lst`
     do
 	ln -s ../*/chr*/$i.fa 
 	echo formatdb -i $i.fa -p F
 	formatdb -i $i.fa -p F
     done
     rm *.log *.fa list
     cd ..
     for i in `cat chrom.lst`; do cat  $i/chr*/*.lft  ; done > jkStuff/subChr.lft
 
     ssh kkr1u00
     rm -rf /iscratch/i/hg17/blastDb
     mkdir -p /iscratch/i/hg17/blastDb
     cd /cluster/data/hg17/blastDb
     for i in nhr nin nsq; do cp *.$i /iscratch/i/hg17/blastDb     ; echo $i; done
 
     cd
     iSync > sync.out
 
     mkdir -p /cluster/data/hg17/bed/tblastn.mm6KG
     cd /cluster/data/hg17/bed/tblastn.mm6KG
     echo  /panasas/store/hg17/blastDb/*.nsq  | xargs ls -S | sed "s/\.nsq//"  > query.lst    
     # back to kkstore01
     exit
 
     cd /cluster/data/hg17/bed/tblastn.mm6KG
     rm -rf /cluster/bluearc/hg17/bed/tblastn.mm6KG/kgfa
     mkdir -p /cluster/bluearc/hg17/bed/tblastn.mm6KG/kgfa
     split -l 560 /cluster/data/mm6/bed/blat.mm6KG/mm6KG.psl /cluster/bluearc/hg17/bed/tblastn.mm6KG/kgfa/kg
     ln -s /cluster/bluearc/hg17/bed/tblastn.mm6KG/kgfa kgfa
     cd kgfa
     for i in *; do pslxToFa $i $i.fa; rm $i; done
     cd ..
     ls -1S kgfa/*.fa > kg.lst
     rm -rf /cluster/bluearc/hg17/bed/tblastn.mm6KG/blastOut
     mkdir -p /cluster/bluearc/hg17/bed/tblastn.mm6KG/blastOut
     ln -s /cluster/bluearc/hg17/bed/tblastn.mm6KG/blastOut
     for i in `cat kg.lst`; do  mkdir blastOut/`basename $i .fa`; done
     tcsh
     cat << '_EOF_' > blastGsub
 #LOOP
 blastSome $(path1) {check in line $(path2)} {check out exists blastOut/$(root2)/q.$(root1).psl } 
 #ENDLOOP
 '_EOF_'
     cat << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/iscratch/i/blast/data
 export BLASTMAT
 g=`basename $2`
 f=/tmp/`basename $3`.$g
 for eVal in 0.01 0.001 0.0001 0.00001 0.000001 1E-09 1E-11
 do
 if /scratch/blast/blastall -M BLOSUM80 -m 0 -F no -e $eVal -p tblastn -d $1 -i $2 -o $f.8
 then
         mv $f.8 $f.1
         break;
 fi
 done
 if test -f  $f.1
 then
     if /cluster/bin/i386/blastToPsl $f.1 $f.2
     then
 	liftUp -nosort -type=".psl" -nohead $f.3 ../../jkStuff/subLiftAll.lft carry $f.2                     
 	liftUp -nosort -type=".psl" -nohead $f.4 ../../jkStuff/liftAll.lft carry $f.3                     
 	liftUp -nosort -type=".psl" -pslQ -nohead $3.tmp /cluster/data/mm6/bed/blat.mm6KG/protein.lft warn $f.4       
 
         if pslCheck -prot $3.tmp                                                                          
         then                                                                                              
             mv $3.tmp $3                                                                                  
             rm -f $f.1 $f.2 $f.3  $f.4
         fi
         exit 0                                                                                            
     fi                                                                                                    
 fi                                                                                                        
 rm -f $f.1 $f.2 $3.tmp $f.8 $f.3 $f.4
 exit 1
 '_EOF_'
 
     chmod +x blastSome
     gensub2 query.lst kg.lst blastGsub blastSpec
 
     ssh kk
     cd /cluster/data/hg17/bed/tblastn.mm6KG
     para create blastSpec
     para push
 # Completed: 214524 of 214524 jobs
 # CPU time in finished jobs:   44907411s  748456.85m 12474.28h  519.76d  1.424 y
 # IO & Wait Time:                712709s   11878.48m   197.97h    8.25d  0.023 y
 # Average job time:                 213s       3.54m     0.06h    0.00d
 # Longest finished job:            1363s      22.72m     0.38h    0.02d
 # Submission to last job:         75910s    1265.17m    21.09h    0.88d
 
 # just for chr13
 # completed: 55290 of 55290 jobs
 # cCPU time in finished jobs:    1487547s   24792.46m   413.21h   17.22d  0.047 y
 # cIO & Wait Time:                148854s    2480.89m    41.35h    1.72d  0.005 y
 # cAverage job time:                  30s       0.49m     0.01h    0.00d
 # cLongest running job:                0s       0.00m     0.00h    0.00d
 # cLongest finished job:              98s       1.63m     0.03h    0.00d
 # cSubmission to last job:          3904s      65.07m     1.08h    0.05d
 
 
     cat << '_EOF_' > chainGsub
 #LOOP
 chainSome $(path1)
 #ENDLOOP
 '_EOF_'
 
     ssh kki
     cd /cluster/data/hg17/bed/tblastn.mm6KG
     tcsh
     cat << '_EOF_' > chainOne
 (cd $1; cat q."$2"* | simpleChain -prot -outPsl -maxGap=200000 stdin ../c.`basename $1`.$2.psl)
 '_EOF_'
     chmod +x chainOne
 
     for j in blastOut/kg??; do for i in `cat ../../chrom.lst`; do echo chainOne $j chr"$i"; done ; done > chainSpec
 
     para create chainSpec
     para push
 # CPU time in finished jobs:         90s       1.50m     0.03h    0.00d  0.000 y
 # IO & Wait Time:                 19151s     319.18m     5.32h    0.22d  0.001 y
 # Average job time:                   3s       0.04m     0.00h    0.00d
 # Longest finished job:               5s       0.08m     0.00h    0.00d
 # Submission to last job:          1642s      27.37m     0.46h    0.02d
 
 # Completed: 7695 of 7695 jobs
 # CPU time in finished jobs:         48s       0.80m     0.01h    0.00d  0.000 y
 # IO & Wait Time:                 18931s     315.51m     5.26h    0.22d  0.001 y
 # Average job time:                   2s       0.04m     0.00h    0.00d
 # Longest finished job:               6s       0.10m     0.00h    0.00d
 # Submission to last job:          1618s      26.97m     0.45h    0.02d
 
     exit
     # back to kkstore01
     cd /cluster/data/hg17/bed/tblastn.mm6KG/blastOut
     for i in kg??
     do 
 	cat c.$i.*.psl | awk "(\$13 - \$12)/\$11 > 0.6 {print}" > c60.$i.psl
 	sort -rn c60.$i.psl | pslUniq stdin u.$i.psl
 	awk "((\$1 / \$11) ) > 0.90 { print   }" c60.$i.psl > m60.$i.psl
 	echo $i
     done
 
     cat u.*.psl m60* | sort -T /tmp -k 14,14 -k 17,17n -k 17,17n  | uniq  > /cluster/data/hg17/bed/tblastn.mm6KG/blastMm6KG.psl
     cd ..
     ssh hgwdev
     cd /cluster/data/hg17/bed/tblastn.mm6KG
     hgLoadPsl hg17 blastHg17KG.psl
     # 1425966 bases of 64944656 (2.196%) 
 
     # back to kkstore01
     rm -rf blastOut
 # End tblastn of mouse proteins
 
 
 ####################################################################################
 # RE-BUILD KNOWN GENES TABLES, 3ND TRIAL WITH CORRECTED kgCheck and kgGetCds (DONE 6/5/05 Fan)
 
 # Start from the step that gene-check is run and kgCandidate0.gp is produced.
 
    cd
    cd /cluster/store10/kg/kgHg17F
 
    mkdir try3
    cd try3
    hgsql kgHg17FTempTry3 -e 'drop table kgCandidate0'
 
    hgsql kgHg17FTempTry3 -e 'drop table kgCandidate0'
    hgsql kgHg17FTempTry3 < ~/src/hg/lib/kgCandidate0.sql 
    hgsql kgHg17FTempTry3 -e  'load data local infile "../kgCandidate0.gp" into table kgCandidate0'
 
    hgsql kgHg17FTempTry3 -e 'drop table geneCheck'
    hgsql kgHg17FTempTry3 < ~/src/hg/lib/geneCheck.sql
    hgsql kgHg17FTempTry3 -e  'load data local infile "../kgCandidate0.check" into table geneCheck ignore 2 lines'
 
 # Run kgCheck to get all KG candidates that pass the KG gene check criteria
 
    kgCheck kgHg17FTempTry3 hg17 kgCandidate0 geneCheck kgCandidate.tab
    hgsql kgHg17FTempTry3 -e  'drop table kgCandidate'
    hgsql kgHg17FTempTry3 < ~/src/hg/lib/kgCandidate.sql
    hgsql kgHg17FTempTry3 -e  'load data local infile "kgCandidate.tab" into table kgCandidate'
    hgsql kgHg17FTempTry3 -e 'create index alignID on kgCandidate(alignID)'
 
 # Construct the kgCandidateX table that has alignID in the name field. 
    cut -f 2-10 kgCandidate.tab >j2.tmp
    cut -f 11 kgCandidate.tab >j1.tmp
    paste j1.tmp j2.tmp >kgCandidateX.tab
 
    hgsql kgHg17FTempTry3 -e  'drop table kgCandidateX'
    hgsql kgHg17FTempTry3 < ~/src/hg/lib/kgCandidateX.sql
    hgsql kgHg17FTempTry3 -e  'load data local infile "kgCandidateX.tab" into table kgCandidateX'
 
 # Score protein/mRna and protein/RefSeq alignments
 
 #   kgResultBestMrna2 050415 kgHg17FTempTry3 hg17|sort -u >protMrnaBlatScore.tab
 #   kgResultBestRef2  050415 kgHg17FTempTry3 hg17|sort -u >protRefScore.tab
 
 # Combine scoring results and load them into temp DB.
    cat protMrnaBlatScore.tab protRefScore.tab >protMrnaScore.tab
    hgsql kgHg17FTempTry3 -e 'drop table protMrnaScore'
    hgsql kgHg17FTempTry3 < ~/src/hg/lib/protMrnaScore.sql
    hgsql kgHg17FTempTry3 -e 'load data local infile "../protMrnaScore.tab" into table protMrnaScore'
    hgsql kgHg17FTempTry3 -e 'create index mrnaAcc on protMrnaScore(mrnaAcc)'
 
 
 # Run kgGetCds to get CDS structure of each gene
 
    kgGetCds kgHg17FTempTry3 kgCandidateX jY.tmp1
    cat jY.tmp1 |sort -u >kgCandidateY.tab
    rm jY.tmp1
    hgsql kgHg17FTempTry3 -e  'drop table kgCandidateY'
    hgsql kgHg17FTempTry3 < ~/src/hg/lib/kgCandidateY.sql
    hgsql kgHg17FTempTry3 -e  'load data local infile "kgCandidateY.tab" into table kgCandidateY'
 
 # Run kgPickPrep to replace long cds structure string with cdsId.
    kgPickPrep kgHg17FTempTry3 kgCandidateZ.tab
    hgsql kgHg17FTempTry3 -e  'drop table kgCandidateZ'
    hgsql kgHg17FTempTry3 < ~/src/hg/lib/kgCandidateZ.sql
    hgsql kgHg17FTempTry3 -e  'load data local infile "kgCandidateZ.tab" into table kgCandidateZ'
    hgsql kgHg17FTempTry3 -e 'create index cdsId on kgCandidateZ(cdsId)'
 
 # Run kgPick to pick the representative a mrna/protein pair for each unique CDS structure.
 
    kgPick kgHg17FTempTry3 hg17 proteins050415 kg3Try3.tmp dupSpMrna.tmp
 
    cat kg3Try3.tmp | grep NM_ > jNM
    cat kg3Try3.tmp | grep -v NM_ >jnoNM
    cut -f 1 jnoNM | sed -e "s/_/_\n/" |grep -v _ >jnoNM1
    cut -f 2-12  jnoNM >jnoNM2
    paste jnoNM1 jnoNM2 > kg3Try3B.tmp
    cat jNM >> kg3Try3B.tmp
 
    sort -u dupSpMrna.tmp >dupSpMrna.tab
 
    hgsql hg17 -e 'drop table dupSpMrna'
    hgsql hg17 < ~/src/hg/lib/dupSpMrna.sql
    hgsql hg17 -e  'load data local infile "dupSpMrna.tab" into table dupSpMrna'
 
 
 # Add entries in the put back list
 # Obtain from Mark the put back list, kgPutBack.lis, for human RefSeq.
 
    hgsql kgHg17FTempTry3 -e 'drop table kgPutBack'
    hgsql kgHg17FTempTry3 < ~/src/hg/lib/kgPutBack.sql
    hgsql kgHg17FTempTry3 -e  'load data local infile "kgPutBack.lis" into table kgPutBack'
 
    kgPutBack kgHg17FTempTry3 hg17 proteins050415 kgPutBack kgPutBack.gp
 
 # Sort KG genes to make the kg3Try3.gp table file.
 
    cat kg3Try3B.tmp kgPutBack.gp >kg3Try3C.tmp
    ~/kent/src/hg/protein/sortKg.pl kg3Try3C.tmp >kg3Try3.gp
 
 # Manually edit to correct one line problem of O75438_BC009691
 
    hgsql kgHg17FTempTry3 -e  'drop table knownGene'
    hgsql kgHg17FTempTry3 < ~/src/hg/lib/knownGene.sql
    hgsql kgHg17FTempTry3 -e  'load data local infile "kg3Try3.gp" into table knownGene'
 
    hgsql hg17 -e  'drop table kg3Try3'
    hgsql hg17 < ~/src/hg/lib/kg3Try3.sql
    hgsql hg17 -e  'load data local infile "kg3Try3.gp" into table kg3Try3'
 
 # Perform analysis before renaming the kg3Try3 table to knownGene.
 
 # Load data into hg17 knownGene table.
    hgsql hg17 -e  'drop table knownGene'
    hgsql hg17 < ~/src/hg/lib/knownGene.sql
    hgsql hg17 -e  'load data local infile "kg3Try3.gp" into table knownGene'
 
 # Build knownGeneMrna and knownGenePep tables.
    hgsql kgHg17FTempTry3 -e  'drop table mrnaSeq'
    hgsql kgHg17FTempTry3 < ~/src/hg/lib/mrnaSeq.sql
    hgsql kgHg17FTempTry3 -e  'load data local infile "../mrnaSeq.tab" into table mrnaSeq'
 
    kgPepMrna kgHg17FTempTry3 hg17 050415
 
    hgsql hg17 -e  'drop table knownGeneMrna'
    hgsql hg17 < ~/src/hg/lib/knownGeneMrna.sql
    hgsql hg17 -e  'load data local infile "knownGeneMrna.tab" into table knownGeneMrna'
    hgsql hg17 -e  'drop table knownGenePep'
    hgsql hg17 < ~/src/hg/lib/knownGenePep.sql
    hgsql hg17 -e  'load data local infile "knownGenePep.tab" into table knownGenePep'
 
 # Build spMrna table
 
    hgsql hg17 -N -e 'select proteinID, name from knownGene' |sort -u >kgSpMrna.tab
 
    hgsql hg17 -e  'drop table spMrna'
    hgsql hg17 <~/src/hg/lib/spMrna.sql
    hgsql hg17 -e 'load data local infile "kgSpMrna.tab" into table spMrna'
 
 # Build kgXref table
 
    kgXref2 kgHg17FTempTry3 050415 hg17
 
    hgsql hg17 -e  'drop table kgXref'
    hgsql hg17 < ~/src/hg/lib/kgXref.sql
    hgsql hg17 -e  'load data local infile "kgXref.tab" into table kgXref'
  
 # MAKE FOLDUTR TABLES 
 # First set up directory structure and extract UTR sequence on hgwdev
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mkdir rnaStruct.2005-06-05
     rm rnaStruct
     ln -s rnaStruct.2005-06-05 rnaStruct
     cd rnaStruct
     mkdir -p utr3/split utr5/split utr3/fold utr5/fold
     utrFa hg17 knownGene utr3 utr3/utr.fa
     utrFa hg17 knownGene utr5 utr5/utr.fa
 
 # Split up files and make files that define job.
     ssh kk
     cd /cluster/data/hg17/bed/rnaStruct
     faSplit sequence utr3/utr.fa 50000 utr3/split/s
     faSplit sequence utr5/utr.fa 50000 utr5/split/s
     ls -1 utr3/split > utr3/in.lst
     ls -1 utr5/split > utr5/in.lst
     cd utr3
     cat > gsub <<end
 #LOOP
 rnaFoldBig split/\$(path1) fold
 #ENDLOOP
 end
     cp gsub ../utr5
 
 # Do cluster run for 3' UTRs
     gensub2 in.lst single gsub spec
     para create spec
     para try
     para push
 # Completed: 35774 of 35774 jobs
 # CPU time in finished jobs:    1174534s   19575.57m   326.26h   13.59d  0.037 y
 # IO & Wait Time:                 98071s    1634.51m    27.24h    1.14d  0.003 y
 # Average job time:                  36s       0.59m     0.01h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:            5409s      90.15m     1.50h    0.06d
 # Submission to last job:          6712s     111.87m     1.86h    0.08d
 
 # Do cluster run for 5' UTRs 
     cd ../utr5
     gensub2 in.lst single gsub spec
     para create spec
     para try
     para push
 # Completed: 33765 of 33765 jobs
 # CPU time in finished jobs:     341000s    5683.33m    94.72h    3.95d  0.011 y
 # IO & Wait Time:                106605s    1776.75m    29.61h    1.23d  0.003 y
 # Average job time:                  13s       0.22m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:           30479s     507.98m     8.47h    0.35d
 # Submission to last job:         30622s     510.37m     8.51h    0.35d 
 
 # Load database
     ssh hgwdev
     cd /cluster/data/hg17/bed/rnaStruct/utr5
     hgLoadRnaFold hg17 foldUtr5 fold
     cd ../utr3
     hgLoadRnaFold hg17 foldUtr3 fold
 
 # Clean up
     rm -r split fold err batch.bak
     cd ../utr5
     rm -r split fold err batch.bak
 
 # Build kgProtMap table
 
 # move all files under hg17Kg to old and copy try3/kgXref.tab up.
 
 # Note: it is important that tight_mrna.psl is here!
     cp old/tight_mrna.psl . -p
 
     ~/src/hg/protein/kgProtMap2.sh kgHg17F hg17 050415
 
 # Update and clean up kgResultBestMrna2.c and then check it in.
 
 # Build alias tables
 
 #	kgAliasM reads from proteins050415.hugo.symbol, proteins050415.hugo.aliases
 #	proteins050415.hugo.withdraws, hg17.kgXref.kgID
 #	to create kgAliasM.tab and geneAlias.tab
 #	by picking out those kgID items from kgXref where
 #	kgXref.geneSymbol == hugo.symbol
 
    kgAliasM hg17 proteins050415
 
 #	kgAliasKgXref reads from hg17.knownGene.proteinID,
 #	hg17.knownGene.name, hg17.kgXref.geneSymbol
 #	to create kgAliasKgXref.tab
 
    kgAliasKgXref hg17
 
 
 #	kgAliasRefseq reads from hg17.knownGene.name,
 #	hg17.knownGene.proteinID, hg17.kgXref.refseq
 #	to create kgAliasRefseq.tab
 
    kgAliasRefseq hg17
 
    hgsql sp050415 -N -e 'select name,gene.val from hg17.knownGene,displayId,gene where displayId.val=proteinID and displayId.acc=gene.acc' \
    | sort -u  > kgAliasP.tab
 
    hgsql hg17 -N -e 'select name, name from knownGene' >kgAliasDup.tab
    hgsql hg17 -N -e 'select mrnaID, dupMrnaID from dupSpMrna' >>kgAliasDup.tab
    
    cat kgAliasM.tab kgAliasRefseq.tab kgAliasKgXref.tab kgAliasP.tab kgAliasDup.tab| \
    sort |uniq > kgAlias.tab
 
    hgsql -e "drop table kgAlias;" hg17 
    hgsql hg17 < ~/kent/src/hg/lib/kgAlias.sql
    hgsql hg17 -e 'LOAD DATA local INFILE "kgAlias.tab" into table kgAlias' 
 
 #	kgProtAlias reads from hg17.knownGene.name,
 #	hg17.knownGene.proteinID, hg17.knownGene.alignID,
 #	proteins050415.spXref3.accession, proteins050415.spSecondaryID, proteins050415.pdbSP.pdb
 #	to create kgProtAlias.tab
 #
 
    kgProtAlias hg17 050415
 
    hgsql hg17 -N -e \
    'select kgID, spDisplayID, protAcc from kgXref where protAcc != ""'\
    | sort -u >kgProtAliasNCBI.tab
 
 # include variant splice protein IDs
    
    hgsql hg17 -N -e \
    'select name, proteinID, parAcc from knownGene,sp050415.varAcc where varAcc=proteinID'\
    |sort -u >kgProtAliasDup.tab
 
 # include duplicate protein IDs from dupSpMrna table
    hgsql hg17 -N -e \
    'select name, knownGene.proteinID, dupProteinID from knownGene, dupSpMrna where name=mrnaID'\
    |sort -u >>kgProtAliasDup.tab
 
 # catch parent acc from dupProteinID too
 
    hgsql hg17 -N -e\
    'select name, knownGene.proteinID, parAcc from knownGene,dupSpMrna,sp050415.varAcc where name=mrnaID and dupProteinID=varAcc.varAcc'\
    |sort -u >>kgProtAliasDup.tab
     cat kgProtAliasNCBI.tab kgProtAlias.tab kgProtAliasDup.tab | sort -u > kgProtAliasAll.tab
 
     echo "`date` creating table kgProtAlias"
     hgsql hg17 -e "drop table kgProtAlias;"
     hgsql hg17 <~/src/hg/lib/kgProtAlias.sql; 
     hgsql hg17 -e 'LOAD DATA local INFILE "kgProtAliasAll.tab" into table kgProtAlias;'  
 
 # Build kgSpAlias table
 
     hgsql hg17 -e \
     'select kgXref.kgID, spID, alias from kgXref, kgAlias where kgXref.kgID=kgAlias.kgID' >j.tmp
     hgsql hg17 -e \
     'select kgXref.kgID, spID, alias from kgXref, kgProtAlias where kgXref.kgID=kgProtAlias.kgID'\
     >>j.tmp
     cat j.tmp|sort -u |grep -v 'kgID' >hg17.kgSpAlias.tab
     rm j.tmp
 
     hgsql hg17 -e 'drop table kgSpAlias';
     hgsql hg17 < ~/src/hg/lib/kgSpAlias.sql
     hgsql hg17 -e 'load data local infile "hg17.kgSpAlias.tab" into table kgSpAlias'
     
 
 # Build KEGG pathway tables
 
    ssh hgwdev
    cd /cluster/store10/kg/kgHg17F
    md kegg
    cd kegg
 
    ~/src/hg/protein/KGpath.sh kgHg17F hg17 050415
 
    hgsql hg17 -e "drop table keggMapDesc"
    hgsql hg17 -e "drop table keggPathway"
    hgsql hg17 <~/src/hg/lib/keggMapDesc.sql
    hgsql hg17 <~/src/hg/lib/keggPathway.sql
    hgsql hg17 -e 'load data local infile "keggMapDesc.tab" into table keggMapDesc'
    hgsql hg17 -e 'load data local infile "keggPathway.tab" into table keggPathway'
 
 # Build CGAP pathway tables
 
    cd ..
    ~/src/hg/protein/KGcgap.sh kgHg17F hg17 050415
    hgsql hg17 -e "drop table cgapAlias"
    hgsql hg17 -e "drop table cgapBiocDesc"
    hgsql hg17 -e "drop table cgapBiocPathway"
    hgsql hg17 <~/src/hg/lib/cgapAlias.sql
    hgsql hg17 <~/src/hg/lib/cgapBiocDesc.sql
    hgsql hg17 <~/src/hg/lib/cgapBiocPathway.sql
    hgsql hg17 -e 'load data local infile "cgapAlias.tab" into table cgapAlias'
    hgsql hg17 -e 'load data local infile "cgapBIOCARTAdescSorted.tab" into table cgapBiocDesc'
    hgsql hg17 -e 'load data local infile "cgapBIOCARTA.tab" into table cgapBiocPathway'
 
 # Build BioCyc pathway tables
 
 # Download BioCyc DB, create and load bioCyc DB
 # See makeBioCycDB.doc for details.
    
    hgsql hg17 -e "drop table bioCycMapDesc"
    hgsql hg17 <~/src/hg/lib/bioCycMapDesc.sql
    hgsql hg17 -e 'load data local infile "bioCycMapDesc.tab" into table bioCycMapDesc'
 
    kgBioCyc |sort -u > bioCycPathway.tab
 
    hgsql hg17 -e "drop table bioCycPathway"
    hgsql hg17 <~/src/hg/lib/bioCycPathway.sql
    hgsql hg17 -e 'load data local infile "bioCycPathway.tab" into table bioCycPathway'
 
 # CCDS <-> knownGene mapping need to be updated (Fan redone 2005-06-05)
 # this should be part of the known gene build
     /cluster/data/genbank/bin/i386/mkCcdsGeneMap -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap
 
 
 ### HG17 PROTEOME BROWSER TABLES RE-BUILD ####  (DONE - 2005-06-05 - Fan)
 # These are instructions for rebuilding tables 
 # needed for the Proteome Browser.  
 # DON'T START THESE UNTIL TABLES FOR KNOWN GENES AND kgProtMap table
 # ARE REBUILT.  
 # This update is based on proteins DBs dated 050415.
 
 # Create the working directory
 
     ssh hgwdev
     mkdir /cluster/store10/kg/kgHg17F/pb-2005-06-05
     cd /cluster/data/hg17/bed
     rm pb
     ln -s /cluster/store10/kg/kgHg17F/pb-2005-06-05 pb
     cd pb
 
 # Move the existing PB tables by:
 
 hgsql hg17
 create database hg17Sav4;
 
 alter table hg17.pepCCntDist rename as hg17Sav4.pepCCntDist;
 alter table hg17.pepExonCntDist rename as hg17Sav4.pepExonCntDist;
 alter table hg17.pepHydroDist rename as hg17Sav4.pepHydroDist;
 alter table hg17.pepIPCntDist rename as hg17Sav4.pepIPCntDist;
 alter table hg17.pepMolWtDist rename as hg17Sav4.pepMolWtDist;
 alter table hg17.pepMwAa rename as hg17Sav4.pepMwAa;
 alter table hg17.pepPi rename as hg17Sav4.pepPi;
 alter table hg17.pepPiDist rename as hg17Sav4.pepPiDist;
 alter table hg17.pepResDist rename as hg17Sav4.pepResDist;
 
 alter table hg17.pbAnomLimit rename as hg17Sav4.pbAnomLimit;
 alter table hg17.pbResAvgStd rename as hg17Sav4.pbResAvgStd;
 alter table hg17.pbStamp rename as hg17Sav4.pbStamp;
 
 quit
 
 # Define pep* tables in hg17 DB
 
 	cat ~/kent/src/hg/lib/pep*.sql > pepAll.sql
 
 #  First edit out pepPred table definition, then
 
 	hgsql hg17 < pepAll.sql
 
 # Build the pepMwAa table
 
   hgsql proteins050415 -N -e \
 "select info.acc, molWeight, aaSize from sp050415.info, sp050415.accToTaxon where accToTaxon.taxon=9606 and accToTaxon.acc = info.acc" > pepMwAa.tab
 
 hgsql hg17 -e 'load data local infile "pepMwAa.tab" into table pepMwAa'
 
 o Build the pepPi table
 
     hgsql proteins050415 -e \
     "select info.acc from sp050415.info, sp050415.accToTaxon where accToTaxon.taxon=9606 and accToTaxon.acc = info.acc" > protAcc.lis
 
     hgsql hg17 -N -e 'select proteinID from knownGene where proteinID like "%-%"' | sort -u >> protAcc.lis
 
     pbCalPi protAcc.lis sp050415 pepPi.tab
     hgsql hg17 -e 'delete from pepPi'
     hgsql hg17 -e 'load data local infile "pepPi.tab" into table hg17.pepPi'
 
 # Calculate and load pep distributions
 
     pbCalDist sp050415 proteins050415 9606 hg17 >pbCalDist.out
     wc  pbCalDist.out
 
     hgsql hg17
     load data local infile "pepExonCntDist.tab" into table hg17.pepExonCntDist;
     load data local infile "pepCCntDist.tab" into table hg17.pepCCntDist;
     load data local infile "pepHydroDist.tab" into table hg17.pepHydroDist;
     load data local infile "pepMolWtDist.tab" into table hg17.pepMolWtDist;
     load data local infile "pepResDist.tab" into table hg17.pepResDist;
     load data local infile "pepIPCntDist.tab" into table hg17.pepIPCntDist;
     load data local infile "pepPiDist.tab" into table hg17.pepPiDist;
     quit
 
 # Calculate frequency distributions
 
     pbCalResStd sp050415 9606 hg17
 
 # Create pbAnomLimit and pbResAvgStd tables
 
    hgsql hg17 -e "drop table pbAnomLimit"
    hgsql hg17 -e "drop table pbResAvgStd"
    hgsql hg17 < ~/src/hg/lib/pbAnomLimit.sql
    hgsql hg17 < ~/src/hg/lib/pbResAvgStd.sql
 
    hgsql hg17 -e 'load data local infile "pbResAvgStd.tab" into table hg17.pbResAvgStd;'
    hgsql hg17 -e 'load data local infile "pbAnomLimit.tab" into table hg17.pbAnomLimit;'
 
 # Create pbStamp table for PB
   hgsql hg17 -e "drop table pbStamp"
   hgsql hg17 < ~/src/hg/lib/pbStamp.sql
   hgsql hg17Sav4 -N -e 'select * from pbStamp' > pbStamp.tab
   hgsql hg17 -e 'load data local infile "pbStamp.tab" into table hg17.pbStamp'
 
 # Adjust drawing parameters for Proteome Browser stamps
 
   Now invoke Proteome Browser and adjust various drawing parameters
   (mostly the ymax of each stamp) if necessary, by updating the 
   pbStamp.tab file and then delete and reload the pbStamp table. 
 
 # Perform preliminary review of Proteome Browser for hg17, then
   notify QA for formal review.
 
 
 # RE-BUILD GENE SORTER TABLES (AKA: FAMILY BROWSER) (DONE - 2005-06-04 - Fan)
 #	This should be done after KG tables are complete from known genes build
 #	process.
 #
 # Cluster together various alt-splicing isoforms.
 #	Creates the knownIsoforms and knownCanonical tables
 ssh hgwdev
 mkdir /cluster/data/hg17/bed/geneSorter.2005-06-04
 # remove old symbolic link
 rm /cluster/data/hg17/bed/geneSorter
 ln -s /cluster/data/hg17/bed/geneSorter.2005-06-04 /cluster/data/hg17/bed/geneSorter
 cd /cluster/data/hg17/bed/geneSorter
 hgClusterGenes hg17 knownGene knownIsoforms knownCanonical
 
 # Extract peptides from knownGenes into fasta file
 # and create a blast database out of them.
 mkdir /cluster/data/hg17/bed/geneSorter/blastp
 cd /cluster/data/hg17/bed/geneSorter/blastp
 pepPredToFa hg17 knownGenePep known.faa
 #	You may need to build this binary in src/hg/near/pepPredToFa
 /scratch/blast/formatdb -i known.faa -t known -n known
 #	This command is in /projects/compbio/bin/$MACH/formatdb
 
 # Copy over database to bluearc
 rm -fr /cluster/bluearc/hg17/blastp
 mkdir -p /cluster/bluearc/hg17/blastp
 cp -p /cluster/data/hg17/bed/geneSorter/blastp/known.* /cluster/bluearc/hg17/blastp
 
 # Split up fasta file into bite sized chunks for cluster
 cd /cluster/data/hg17/bed/geneSorter/blastp
 mkdir split
 faSplit sequence known.faa 8000 split/kg
 
 # Make parasol run directory
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/self
 cd /cluster/data/hg17/bed/geneSorter/blastp/self
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 -p blastp -d /cluster/bluearc/hg17/blastp/known -i $1 -o $2 \
 -e 0.01 -m 8 -b 1000
 '_EOF_'
     # << keep emacs happy
 chmod +x blastSome
 
 # Make gensub2 file
 cat  << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #	'ls ../../split/*.fa' is too much, hence the echo
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para push
 para check
 # Completed: 7735 of 7735 jobs
 # CPU time in finished jobs:     142764s    2379.39m    39.66h    1.65d  0.005 y
 # IO & Wait Time:                 67623s    1127.06m    18.78h    0.78d  0.002 y
 # Average job time:                  27s       0.45m     0.01h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:             144s       2.40m     0.04h    0.00d
 # Submission to last job:           392s       6.53m     0.11h    0.00d
 
 # Load into database.  This takes about 30 minutes
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/self/run/out
 time hgLoadBlastTab hg17 knownBlastTab *.tab
 # Scanning through 7735 files
 # Loading database with 9757382 rows
 # 255.200u 50.520s 25:19.66 20.1% 0+0k 0+0io 247pf+0w
 
 cd /cluster/data/hg17/bed/geneSorter
 # Create table that maps between known genes and RefSeq
 hgMapToGene hg17 refGene knownGene knownToRefSeq
 #	may need to build this command in src/hg/near/hgMapToGene
 #	hgsql -e "select count(*) from knownToRefSeq;" hg17
 #	row count changed 34667
 
 # Create table that maps between known genes and LocusLink
 hgsql --skip-column-names -e "select mrnaAcc,locusLinkId from refLink" hg17 > refToLl.txt
 hgMapToGene hg17 refGene knownGene knownToLocusLink -lookup=refToLl.txt
 #	hgsql -e "select count(*) from knownToLocusLink;" hg17
 #	row count changed to 34773
 
 # Create table that maps between known genes and Pfam domains
 hgMapViaSwissProt hg17 knownGene name proteinID Pfam knownToPfam
 #	hgsql -e "select count(*) from knownToPfam;" hg17
 #	row count changed to 29171 
 
 # Create table to map between known genes and GNF Atlas2
 # expression data.
     hgMapToGene hg17 gnfAtlas2 knownGene knownToGnfAtlas2 '-type=bed 12'
 #	hgsql -e "select count(*) from knownToGnfAtlas2;" hg17
 #	row count changed to 32458
 
 # Create expression distance table - takes about an hour
     hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
     	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
 	-lookup=knownToGnfAtlas2 &
 # Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
 # Got 32458 unique elements in hgFixed.gnfHumanAtlas2MedianRatio
 #	hgsql -e "select count(*) from gnfAtlas2Distance;" hg17
 #	row count changed to 32381000
 
 # Create a table that maps between known genes and 
 # the nice affy expression data.
 hgMapToGene "-type=bed 12" hg17 affyUclaNorm knownGene knownToU133
 #	hgsql -e "select count(*) from knownToU133;" hg17
 #	row count changed to 32965
 
 # Create expression distance table.  This will take about 2.5 hours
 cd /tmp
 cp -p ~/kent/src/hg/near/hgExpDistance/affyUcla.weight .
 time hgExpDistance hg17 affyUclaNorm affyUclaExp knownExpDistance \
 	-weights=affyUcla.weight -lookup=knownToU133 &
 # Have 43039 elements in affyUclaNorm
 # 211 genes, 42 weights, 26.500000 total wieght
 # Got 32965 unique elements in affyUclaNorm
 
 # Create table that maps between known genes and 
 # the GNF data.
 cd /tmp
 hgMapToGene hg17 affyU95 knownGene knownToU95
 #	row count changed to 17555
 #	hgFixed.gnfHumanU95Exps argument is unused, no need to exist
 hgExpDistance hg17 hgFixed.gnfHumanU95MedianRatio \
 	hgFixed.gnfHumanU95Exps gnfU95Distance  -lookup=knownToU95 &
 # Have 11545 elements in hgFixed.gnfHumanU95MedianRatio
 # Got 16501 unique elements in hgFixed.gnfHumanU95MedianRatio
 #	row count changed to 16450000  
 
 # Create known gene mapping table and expression distance tables
 # for GNF Atlas 2.  (The hgExpDistance takes only 10 minutes.)
 
 hgMapToGene hg17 affyGnf1h knownGene knownToGnf1h
 hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
 	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
 	-lookup=knownToGnf1h &
 # Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
 # Got 8827 unique elements in hgFixed.gnfHumanAtlas2MedianRatio
 
 cd /cluster/data/hg17/bed/geneSorter
 hgMapToGene hg17 affyU133Plus2 knownGene knownToU133Plus2
 #	row count changed to 35139  
 
 #### UPDATE GO DATABASE (THIS PART WAS DONE 5/21/05 Fan)
 
 # Download the terms and make the database.
 ssh hgwdev
 mkdir /cluster/store1/geneOntology/20050521
 cd /cluster/store1/geneOntology/20050521
 
 wget --timestamping http://www.godatabase.org/dev/database/archive/latest/go_200504-assocdb-data.gz
 
 hgsql mysql <<end
 create database go050521;
 end
 zcat go_*data.gz | sed -e 's/ENGINE=MyISAM DEFAULT CHARSET=latin1/TYPE=MyISAM/g' >j.tmp
 hgsql go050521 <j.tmp
 rm j.tmp
 
 wget --timestamping ftp://ftp.geneontology.org/pub/go/gene-associations/gene_association.goa_uniprot.gz
 
 zcat gene_association.goa_uniprot.gz | hgGoAssociation go050521 goaPart stdin 
 # Passed 5589891 of 6584507 of 6584507, 84.89%
 
 # Ask sys-admin to switch the database pointer go to point to go050521.
 
 cd /cluster/data/hg17/bed/geneSorter
 
 # Rebuilt Ensembl Gene tables.  See documentation (5/23/05 Fan) above.
 
 # Create knownToEnsembl column
 hgMapToGene hg17 ensGene knownGene knownToEnsembl
 # hgsql hg17 -e "select count(*) from knownToEnsembl" 
 #	table row count 35521
 
 # Make knownToCdsSnp table 
   ssh hgwdev
   nice hgMapToGene hg17 snp knownGene knownToCdsSnp -all -cds
 # hgsql hg17 -e "select count(*) from knownToCdsSnp" 
 # row count 94633
 # approx. 5 minutes running time
 
 # C.ELEGANS BLASTP FOR GENE SORTER 
     # Make C. elegans ortholog column using blastp on wormpep.
     # First make C. elegans protein database and copy it to iscratch/i
     # if it doesn't exist already:
 
 # The following section is done during mm6 build already.
 #    ssh eieio
 #    mkdir /cluster/data/ce2/bed/blastp
 #    cd /cluster/data/ce2/bed/blastp
 #    # Point a web browser at ftp://ftp.sanger.ac.uk/pub/databases/wormpep/
 #    # to find out the latest version.  Then use that in place of 142 below.
 #    wget -O wormPep142.faa ftp://ftp.sanger.ac.uk/pub/databases/wormpep/wormpep142/wormpep142
 #    formatdb -i wormPep142.faa -t wormPep142 -n wormPep142
 #    ssh kkr1u00
 #    if (-e /iscratch/i/ce2/blastp) then
 #      rm -r /iscratch/i/ce2/blastp
 #    endif
 #    mkdir -p /iscratch/i/ce2/blastp
 #    cp /cluster/data/ce2/bed/blastp/wormPep142.p?? /iscratch/i/ce2/blastp
 #    iSync
 
     # Make parasol run directory 
     ssh kk
     mkdir -p /cluster/data/hg17/bed/blastp/ce2/run/out
     cd /cluster/data/hg17/bed/blastp/ce2/run
     # Make blast script
     cat > blastSome <<end
 #!/bin/csh
 setenv BLASTMAT /iscratch/i/blast/data
 /iscratch/i/blast/blastall -p blastp -d /iscratch/i/ce2/blastp/wormPep142 -i \$1 -o \$2 -e 0.01 -m 8 -b 1
 end
     chmod a+x blastSome
     # Make gensub2 file
     cat > gsub <<end
 #LOOP
 blastSome {check in line+ \$(path1)} {check out line out/\$(root1).tab}
 #ENDLOOP
 end
     # Create parasol batch
     ls -1S /cluster/data/hg17/bed/geneSorter/blastp/split \
     |sed -e 's=kg=../../../geneSorter/blastp/split/kg=g' >split.lst
     gensub2 split.lst single gsub spec
     para create spec
     para try, check, push, check, ...  
 
 # Initial run has 13 jobs crashed.
 # crashed: 13
 # ranOk: 7722
 
 # para problems show the following typical message:
 
 # total jobs in batch: 7735
 # job: blastSome ../../../geneSorter/blastp/split/kg5911.fa out/kg5911.tab
 # id: 209522384
 # failure type: crash
 # host: kkr2u28.kilokluster.ucsc.edu
 # start time: Sat Jun  4 11:45:51 2005
 # return: 0
 # stderr:
 # [blastall] FATAL ERROR: blast: Unable to open input file ../../../geneSorter/blastp/split/kg5911.fa
 
 # para push again and these 13 ran fine.
 # Completed: 7735 of 7735 jobs
 # CPU time in finished jobs:      60319s    1005.32m    16.76h    0.70d  0.002 y
 # IO & Wait Time:                 31239s     520.65m     8.68h    0.36d  0.001 y
 # Average job time:                  12s       0.20m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              72s       1.20m     0.02h    0.00d
 # Submission to last job:           199s       3.32m     0.06h    0.00d
 
 # Load into database.  
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastp/ce2/run/out
     hgLoadBlastTab hg17 ceBlastTab -maxPer=1 *.tab
 # Scanning through 7735 files
 # Loading database with 25574 rows
 
 # Make mouse ortholog column using blastp on mouse known genes.
 # First make mouse protein database and copy it to /cluster/panasas
 # if it doesn't exist already
 #	This already exists.  See makeMm6.doc for procedure
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/mm6
 cd /cluster/data/hg17/bed/geneSorter/blastp/mm6
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 -p blastp -d /cluster/panasas/home/store/mm6/blastp/known \
 -i $1 -o $2 -e 0.001 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #	this echo trick is used because otherwise the command line is
 #	too long and you can not do a simple ls
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7735 of 7735 jobs
 # CPU time in finished jobs:      85769s    1429.49m    23.82h    0.99d  0.003 y
 # IO & Wait Time:                 20587s     343.11m     5.72h    0.24d  0.001 y
 # Average job time:                  14s       0.23m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              78s       1.30m     0.02h    0.00d
 # Submission to last job:           206s       3.43m     0.06h    0.00d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/mm6/run/out
 hgLoadBlastTab hg17 mmBlastTab -maxPer=1 *.tab
 # Scanning through 7735 files
 # Loading database with 32951 rows
 
 # Make rat ortholog column using blastp on rat known genes.
 # First make rat protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This already exists.  See makeRn3.doc for procedure.
 #	Files were put in this directory: /cluster/bluearc/rn3/blastp/
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/rn3
 cd /cluster/data/hg17/bed/geneSorter/blastp/rn3
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 -p blastp -d /cluster/bluearc/rn3/blastp/known \
 -i $1 -o $2 -e 0.001 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #	this echo trick is used because otherwise the command line is
 #	too long and you can not do a simple ls
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc 
 # Completed: 7735 of 7735 jobs
 # CPU time in finished jobs:      27804s     463.40m     7.72h    0.32d  0.001 y
 # IO & Wait Time:                 30334s     505.56m     8.43h    0.35d  0.001 y
 # Average job time:                   8s       0.13m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              26s       0.43m     0.01h    0.00d
 # Submission to last job:           119s       1.98m     0.03h    0.00d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/rn3/run/out
 hgLoadBlastTab hg17 rnBlastTab -maxPer=1 *.tab
 # Scanning through 7735 files
 # Loading database with 24030 rows
 
 # ZEBRAFISH BLASTP FOR GENE SORTER 
     # Make Danio rerio (zebrafish) ortholog column using blastp on Ensembl.
     # First make protein database and copy it to iscratch/i
     # if it doesn't exist already:
     ssh kkstore
     mkdir /cluster/data/danRer2/bed/blastp
     cd /cluster/data/danRer2/bed/blastp
     wget ftp://ftp.ensembl.org/pub/current_zebrafish/data/fasta/pep/Danio_rerio.ZFISH4.may.pep.fa.gz 
     zcat Dan*.pep.fa.gz > ensembl.faa
     /scratch/blast/formatdb -i ensembl.faa -t ensembl -n ensembl
     ssh kkr1u00
     if (-e /iscratch/i/danRer2/blastp) then
       rm -r /iscratch/i/danRer2/blastp
     endif
     mkdir -p /iscratch/i/danRer2/blastp
     cp /cluster/data/danRer2/bed/blastp/ensembl.p?? /iscratch/i/danRer2/blastp
     iSync
 
     # Make parasol run directory 
     ssh kk
     mkdir -p /cluster/data/hg17/bed/blastp/danRer2/run/out
     cd /cluster/data/hg17/bed/blastp/danRer2/run
     # Make blast script
     cat > blastSome <<end
 #!/bin/csh
 setenv BLASTMAT /iscratch/i/blast/data
 /iscratch/i/blast/blastall -p blastp -d /iscratch/i/danRer2/blastp/ensembl -i \$1 -o \$2 -e 0.005 -m 8 -b 1
 end
     chmod a+x blastSome
     # Make gensub2 file
     cat > gsub <<end
 #LOOP
 blastSome {check in line+ \$(path1)} {check out line out/\$(root1).tab}
 #ENDLOOP
 end
     # Create parasol batch
     ls -1S /cluster/data/hg17/bed/geneSorter/blastp/split \
     |sed -e 's=kg=../../../geneSorter/blastp/split/kg=g' > split.lst
     gensub2 split.lst single gsub spec
     para create spec
     para try, check, push, check, ...
 # Completed: 7735 of 7735 jobs
 # CPU time in finished jobs:     111467s    1857.78m    30.96h    1.29d  0.004 y
 # IO & Wait Time:                 21159s     352.65m     5.88h    0.24d  0.001 y
 # Average job time:                  17s       0.29m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              95s       1.58m     0.03h    0.00d
 # Submission to last job:           223s       3.72m     0.06h    0.00d
 
     # Load into database.  
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastp/danRer2/run/out
     hgLoadBlastTab hg17 drBlastTab -maxPer=1 *.tab
 # Scanning through 7735 files
 # Loading database with 30651 rows
 
 # Make Saccharomyces cerevisiae (yeast) ortholog column using blastp on RefSeq.
 # First make protein database and copy it to cluster/bluearc
 # if it doesn't exist already
 #	This is already done, see makeMm3.doc for procedure
 #	the directory: /cluster/bluearc/sc1/blastp should have data
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/sc1
 cd /cluster/data/hg17/bed/geneSorter/blastp/sc1
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 -p blastp -d /cluster/bluearc/sc1/blastp/sgd \
 -i $1 -o $2 -e 0.01 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7735 of 7735 jobs
 # CPU time in finished jobs:      18194s     303.23m     5.05h    0.21d  0.001 y
 # IO & Wait Time:                 24452s     407.53m     6.79h    0.28d  0.001 y
 # Average job time:                   6s       0.09m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              16s       0.27m     0.00h    0.00d
 # Submission to last job:           120s       2.00m     0.03h    0.00d
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/sc1/run/out
 hgLoadBlastTab hg17 scBlastTab -maxPer=1 *.tab
 # Scanning through 7735 files
 # Loading database with 16395 rows 
 
 # Make Drosophila melanagaster ortholog column using blastp on FlyBase.
 # First make SwissProt protein database and copy it to cluster/bluearc
 # The following section was already done.
 # cd /cluster/data/dm1/bed
 # mkdir blastp
 # cd blastp
 #wget ftp://ftp.fruitfly.org/pub/download/dmel_RELEASE3-1/FASTA/whole_genome_translation_dmel_RELEASE3-1.FASTA.gz
 # zcat whole_ge*.gz | faFlyBaseToUcsc stdin flyBase.faa
 # formatdb -i flyBase.faa -t flyBase -n flyBase
 # if (-e /cluster/bluearc/dm1/blastp) then
 #    rm -r /cluster/bluearc/dm1/blastp
 # endif
 # mkdir -p /cluster/bluearc/dm1/blastp
 # cp /cluster/data/dm1/bed/blastp/flyBase.p?? /cluster/bluearc/dm1/blastp
 
 # Make parasol run directory 
 ssh kk
 mkdir /cluster/data/hg17/bed/geneSorter/blastp/dm1
 cd /cluster/data/hg17/bed/geneSorter/blastp/dm1
 mkdir run
 cd run
 mkdir out
 
 # Make blast script
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 -p blastp -d /cluster/bluearc/dm1/blastp/flyBase \
 -i $1 -o $2 -e 0.01 -m 8 -b 1
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 # Create parasol batch
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 # Completed: 7735 of 7735 jobs
 # CPU time in finished jobs:      72141s    1202.35m    20.04h    0.83d  0.002 y
 # IO & Wait Time:                 41717s     695.28m    11.59h    0.48d  0.001 y
 # Average job time:                  15s       0.25m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              57s       0.95m     0.02h    0.00d
 # Submission to last job:           204s       3.40m     0.06h    0.00d
 
 # Load into database.  
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/dm1/run/out
 hgLoadBlastTab hg17 dmBlastTab -maxPer=1 *.tab
 # Scanning through 7735 files
 # Loading database with 27109 rows
 
 # update knownToHInv table
 # Verified that there is now new release of HInv data.
 hgMapToGene hg17 HInvGeneMrna knownGene knownToHInv
 # count changed to 28851
 
 # The new KG process no longer need entries in knownGeneLink (used to store
 # info for DNA based RefSeqs.  So clean out the old data in knownGeneLink.
 
 hgsql hg17 -e "delete from knownGeneLink"
 
 #### RE-BUILD SUPERFAMILY RELATED TABLES (DONE - 2005-05-27 - Fan)
 
 # Download latest Superfamily data files and build the Superfamily DB
 # from supfam.mrc-lmb.cam.ac.uk
 
     mkdir /cluster/store10/superfamily/050524
     ln -s /cluster/store10/superfamily/050524 /cluster/data/superfamily/050524
     cd /cluster/data/superfamily/050524
 
 # ftp over the following two files:
 ass_22-May-2005.tab.gz
 supfam_22-May-2005.sql.gz
     gzip -d *.gz
 
 # Load the Superfamily database
     hgsql hg17 -e "create database superfam050524"
     nice hgsql superfam050524 < supfam_22-May-2005.sql &
 
 # This may take about an hour.
 
 # Make sure to add an index on id of the des table of superfam050524.
     hgsql superfam050524 -e "create index id on des(id);"
 
     hgsql superfam050524 < ~/src/hg/lib/sfAssign.sql
     hgsql superfam050524 -e 'load data local infile "ass_22-May-2005.tab" into table 
 superfam050524.sfAssign;'
 
 # Build or rebuild Superfamily track and create sf tables needed for PB
 
    hgsql hg17 < ~/src/hg/lib/sfAssign.sql
 
    cd /cluster/data/superfamily/050524  
    hgsql hg17 -e 'load data local infile "ass_22-May-2005.tab" into table hg17.sfAssign;'
 
 # If hg17.sfDes already exists, drop it.
 
    hgsql superfam050524 -N -e "select * from des" >sfDes.tab
    hgsql hg17 < ~/src/hg/lib/sfDes.sql
    hgsql hg17 -e 'load data local infile "sfDes.tab" into table sfDes'
 
 # If hg17.superfamily already exists, drop it.
    cd /cluster/data/hg17/bed
    mkdir /cluster/data/hg17/sf.2004-1128
    ln -s sf.2004-1128 sf
    hgSuperfam hg17 > sf.log
 
 # It is normal that many proteins does not have corresponding Superfamily entries.
 
 # If hg17.sfDescription exists, drop it.
 
    hgsql hg17 < ~/src/hg/lib/sfDescription.sql
    hgsql hg17 -e 'LOAD DATA local INFILE "sfDescription.tab"  into table hg17.sfDescription;'
 
 # Finally, load the superfamily table.
 
    hgLoadBed hg17 superfamily superfamily.tab -tab
 
 # Create knownToSuperfamily table
 # Note hs is changed into ht for this Superfamily release.
    
    cat /cluster/data/superfamily/050524/ass_22-May-2005.tab \
    | hgKnownToSuper hg17 hs stdin
 # created 32906 rows in knownToSuper
 
 # Build tables needed by pbGlobal in proteins050415
 
    cd /cluster/data/superfamily/050524  
    hgsql proteins050415 -e 'load data local infile "ass_22-May-2005.tab" into table sfAssign'
 
    hgsql proteins050415 -e 'load data local infile "sfDes.tab" into table sfDes'
 
    cd /cluster/store10/kg/kgHg17F
    hgsql proteins050415 -e 'load data local infile "ensemblXref.tab" into table ensemblXref'
 
 # These sf tables and ensemblXref3 are needed for non-HMR KG proteins.
 # Should add content of ensemblXref3 of mm6 after it is done.
 # And similarly for rn4 and possibly for other non-HMR species.
 # CCDS <-> knownGene mapping need to be updated (markd 2005-05-29)
 # this should be part of the known gene build
     /cluster/data/genbank/bin/i386/mkCcdsGeneMap -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap
 
 # Build targetScanS track - (DONE - 2005-06-22 Fan)
 #       requested by: Dudekula, Dawood (NIH/NIA/IRP) DudekulaDB@grc.nia.nih.gov
     ssh hgwdev 
     mkdir -p /cluster/data/hg17/bed/targetScanS
     cd /cluster/data/hg17/bed/targetScanS
     wget --timestamp http://genes.mit.edu/targetscan/tracks/targetscan.bed
 
 # Remove the first description line of targetscan.bed
 
     hgLoadBed -tab hg17 targetScanS targetscan.bed
 
 # Create/edit/check in targetScans.html and trackDb.ra under
 # kent/src/hg/makeDb/trackDb/human/hg17
 
 # Update mrnaRefseq table (DONE - Fan 6/22/05)
 # The old table contains non-human mrna/RefSeqs.
 # The new table contains only human mrna/RefSeq and RefSeq/RefSeq.
 
 # First build entrez DB tables, see makeMm6.doc for details.
 
     hgsql entrez -N -e \
     'select mrna, refseq from entrezRefseq, entrezMrna, hg17.all_mrna where entrezRefseq.geneID=entrezMrna.geneID and mrna=all_mrna.qName' \
     >mrnaRefseq1.tab
 
 # Include RefSeq as valid mRNA too.
     hgsql hg17 -N -e 'select name, name from refGene' >mrnaRefseq2.tab
 
     cat mrnaRefseq1.tab mrnaRefseq2.tab |sort -u >mrnaRefseq.tab
 
     hgsql hg17 -e 'drop table mrnaRefseq'
     hgsql hg17 < ~/src/hg/lib/mrnaRefseq.sql
     hgsql hg17 -e 'load data local infile "mrnaRefseq.tab" into table mrnaRefseq'
 
 # BUILD KNOWN GENE LIST FOR GOOGLE.  DONE 6/27/05 Fan.
 # make knownGeneLists.html hg17GeneList.html mm5GeneList.html rm3GeneList.html
 
     cd /cluster/data/hg17/bed
     rm -rf knownGeneList/hg17
 
 # Run hgKnownGeneList to generate the tree of HTML pages
 # under ./knownGeneList/hg17
 
     hgKnownGeneList hg17
 
 # copy over to /usr/local/apache/htdocs
 
     rm -rf /usr/local/apache/htdocs/knownGeneList/hg17
     mkdir -p /usr/local/apache/htdocs/knownGeneList/hg17
     cp -Rfp knownGeneList/hg17/* /usr/local/apache/htdocs/knownGeneList/hg17
 
 ####  Blat knownGene proteins to determine exons (DONE braney 06-30-05)
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mkdir blat.hg17KG.2005-06-17
     rm blat.hg17KG
     ln -s  blat.hg17KG.2005-06-17 blat.hg17KG
     cd blat.hg17KG
     pepPredToFa hg17 knownGenePep known.fa
     hgPepPred hg17 generic blastKGPep02 known.fa
     grep ">" known.fa | sed "s/>//" > kgName.lst
     kgName hg17 kgName.lst blastKGRef02
     hgsql hg17 < ~/kent/src/hg/lib/blastRef.sql
     echo "rename table blastRef to blastKGRef02" | hgsql hg17
     echo "load data local infile 'blastKGRef02' into table blastKGRef02" | hgsql hg17
     ssh kk
     cd /cluster/data/hg17/bed/blat.hg17KG
     cat << '_EOF_' > blatSome
 #!/bin/csh -fe
 /cluster/bin/i386/blat -t=dnax -q=prot -out=pslx $1 $2 $3
 '_EOF_'
     # << keep emacs happy
     chmod +x blatSome
     ls -1S /scratch/hg/gs.18/build35/bothMaskedNibs/*.nib > human.lst
     mkdir kgfa
     cd kgfa
     faSplit sequence ../known.fa 3020 kg
     cd ..
     ls -1S kgfa/*.fa > kg.lst
     cat << '_EOF_' > blatGsub
 #LOOP
 blatSome $(path1) {check in line $(path2)} {check out line psl/$(root1)/$(root2).psl}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
     gensub2 human.lst kg.lst blatGsub blatSpec
     mkdir psl
     cd psl
     foreach i (`cat ../human.lst`)
 	mkdir `basename $i .nib`
     end
     cd ..
     para create blatSpec
     para push
 # Completed: 134320 of 134320 jobs
 # CPU time in finished jobs:   22196680s  369944.67m  6165.74h  256.91d  0.704 y
 # IO & Wait Time:               1712586s   28543.10m   475.72h   19.82d  0.054 y
 # Average job time:                 178s       2.97m     0.05h    0.00d
 # Longest finished job:            7691s     128.18m     2.14h    0.09d
 # Submission to last job:        608750s   10145.83m   169.10h    7.05d
 
 # Completed: 133676 of 133676 jobs
 # CPU time in finished jobs:   29661130s  494352.16m  8239.20h  343.30d  0.941 y
 # IO & Wait Time:               2181179s   36352.99m   605.88h   25.25d  0.069 y
 # Average job time:                 238s       3.97m     0.07h    0.02d
 # Longest job:                   105972s    1766.20m    29.44h    1.23d
 
     ssh eieio
     cd /cluster/data/hg17/bed/blat.hg17KG
     pslSort dirs raw.psl /tmp psl/*
     pslReps -nohead -minCover=0.9 -minAli=0.9 raw.psl cooked.psl /dev/null
     pslUniq cooked.psl hg17KG.psl
     pslxToFa hg17KG.psl hg17KG_ex.fa -liftTarget=genome.lft -liftQuery=protein.lft
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/blat.hg17KG
     kgName hg17 hg17KG.psl blastKGRef02
     cut -f 10 hg17KG.psl > kgName.lst
     faSomeRecords known.fa kgName.lst hg17KG.fa
     hgPepPred hg17 generic blastKGPep02 hg17KG.fa
 #end blat proteins
     
 # MAKE Drosophila Proteins track (DONE 07-05-05 braney)
     ssh kk
     mkdir -p /cluster/data/hg17/bed/tblastn.dm2FB
     cd /cluster/data/hg17/bed/tblastn.dm2FB
     echo  /panasas/store/hg17/blastDb/*.nsq  | xargs ls -S | sed "s/\.nsq//"  > target.lst    
     mkdir fbfa
     # calculate a reasonable number of jobs 
     calc `wc /cluster/data/dm2/bed/blat.dm2FB/dm2FB.psl|awk "{print \\\$1}"`/\(264630/`wc target.lst| awk "{print \\\$1}"`\)
 # 18929/(350000/5959) = 322.279746
     split -l 322 /cluster/data/dm2/bed/blat.dm2FB/dm2FB.psl fbfa/fb
     cd fbfa
     for i in *; do pslxToFa $i $i.fa; rm $i; done
     cd ..
     ls -1S fbfa/*.fa > fb.lst
     mkdir -p /cluster/bluearc/hg17/bed/tblastn.dm2FB/blastOut  
     ln -s /cluster/bluearc/hg17/bed/tblastn.dm2FB/blastOut  
     for i in `cat fb.lst`; do  mkdir blastOut/`basename $i .fa`; done
     tcsh
     cat << '_EOF_' > blastGsub
 #LOOP
 blastSome $(path1) {check in line $(path2)} {check out exists blastOut/$(root2)/q.$(root1).psl } 
 #ENDLOOP
 '_EOF_'
     cat << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/iscratch/i/blast/data
 export BLASTMAT
 g=`basename $2`
 f=/tmp/`basename $3`.$g
 for eVal in 0.01 0.001 0.0001 0.00001 0.000001 1E-09 1E-11
 do
 if /scratch/blast/blastall -M BLOSUM80 -m 0 -F no -e $eVal -p tblastn -d $1 -i $2 -o $f.8
 then
         mv $f.8 $f.1
         break;
 fi
 done
 if test -f  $f.1
 then
 if /cluster/bin/i386/blastToPsl $f.1 $f.2
 then
 	liftUp -nosort -type=".psl" -nohead $f.3 /cluster/data/hg17/jkStuff/subLiftAll.lft warn $f.2  
 	liftUp -nosort -type=".psl" -nohead $f.4 /cluster/data/hg17/jkStuff/liftAll.lft warn $f.3  
         liftUp -isPtoG -nosort -type=".psl" -pslQ -nohead $3.tmp /cluster/data/dm2/bed/blat.dm2FB/protein.lft warn $f.4
         mv $3.tmp $3
         rm -f $f.1 $f.2 $f.3
         exit 0
     fi
 fi
 rm -f $f.1 $f.2 $3.tmp $f.3 $f.8
 exit 1
 '_EOF_'
 
     chmod +x blastSome
     gensub2 target.lst fb.lst blastGsub blastSpec
 
     ssh kk
     cd /cluster/data/hg17/bed/tblastn.dm2FB
     para create blastSpec
     para push
 
 # Completed: 351581 of 351581 jobs
 # CPU time in finished jobs:   30733031s  512217.19m  8536.95h  355.71d  0.975 y
 # IO & Wait Time:               1035790s   17263.16m   287.72h   11.99d  0.033 y
 # Average job time:                  90s       1.51m     0.03h    0.00d
 # Longest finished job:             816s      13.60m     0.23h    0.01d
 # Submission to last job:        135367s    2256.12m    37.60h    1.57d
 
     ssh kki
     cd /cluster/data/hg17/bed/tblastn.dm2FB
     tcsh
     
     cat << '_EOF_' > chainGsub
 #LOOP
 chainSome $(path1) $(path2)
 #ENDLOOP
 '_EOF_'
 
     cat << '_EOF_' > chainSome
 (cd $1; cat $2.psl | simpleChain -prot -outPsl -maxGap=150000 stdin ../c.`basename $1`.psl)
 '_EOF_'
     chmod +x chainSome
 
     ls -1dS `pwd`/blastOut/fb?? > chain.lst
     gensub2 chain.lst single chainGsub chainSpec
 
     para create chainSpec
     para push
  
 # Completed: 2714 of 2714 jobs
 # CPU time in finished jobs:     222508s    3708.46m    61.81h    2.58d  0.007 y
 # IO & Wait Time:                 10577s     176.29m     2.94h    0.12d  0.000 y
 # Average job time:                  86s       1.43m     0.02h    0.00d
 # Longest finished job:            9787s     163.12m     2.72h    0.11d
 
     cd /cluster/data/hg17/bed/tblastn.dm2FB/blastOut
     for i in fb??
     do 
 	awk "(\$13 - \$12)/\$11 > 0.6 {print}" c.$i.*.psl > c60.$i.psl
 	sort -rn c60.$i.psl | pslUniq stdin u.$i.psl
 	awk "((\$1 / \$11) ) > 0.60 { print   }" c60.$i.psl > m60.$i.psl
 	echo $i
     done
 
     sort -u -T /tmp -k 14,14 -k 16,16n -k 17,17n u.*.psl m60* > /cluster/data/hg17/bed/tblastn.dm2FB/blastDm2FB.psl
     cd ..
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/tblastn.dm2FB
     hgLoadPsl hg17 blastDm2FB.psl
     exit
 
     # back to kksilo
     rm -rf blastOut
 
 # End tblastn
 
 # Build kgReactome table for KG to Reactome xref.  Done 6/28/05 Fan.
 
     ssh hgwdev
     mkdir -p /cluster/store10/reactome/reactome050613
     rm    /cluster/data/reactome
     ln -s /cluster/store10/reactome/reactome050613 /cluster/data/reactome
 
     cd /cluster/data/reactome
     
     wget --timestamp http://www.reactome.org/download/current/sql.gz
     
     hgsql hg17 -e 'drop database reactome'
     hgsql hg17 -e 'create database reactome'
     zcat sql.gz| hgsql reactome 
 
     hgsql reactome -N -e 'select kgId, spID, DB_ID from ReferenceEntity, hg17.kgXref where identifier=spID' >kgReactome.tab;
 
     hgsql hg17 -e 'drop table kgReactome'
     hgsql hg17 < ~/src/hg/lib/kgReactome.sql
     hgsql hg17 -e 'load data local infile "kgReactome.tab" into table kgReactome'
 
 # UPDATE WGRNA TRACK (DONE, 2005-07-05, Fan)
 
   ssh hgwdev
   cd /cluster/data/hg17/bed
 
   mv wgRna wgRna-2005-06-16
   mkdir wgRna-2005-07-05
   cd wgRna-2005-07-05
 
 # Received the data file, wgtrack_july2005.txt, from Michel Weber's email (Michel.Weber@ibcg.biotoul.fr)
 # and place it under cd /cluster/data/hg17/bed/wgRna-2005-07-05.
 
   cat wgtrack_july2005.txt|sed -e 's/ /\t/g' >wgRna.tab
   
 # edit wgRna.tab to take out the first 5 lines of data field labels.
 
     hgLoadBed -sqlTable=/cluster/home/fanhsu/src/hg/lib/wgRna.sql hg17 wgRna wgRna.tab
 
 # REBUILT knownToPfam TABLE TO ALLOW KG REPRESENTED BY VARIANT SPLICE PROTEINS MAPPED TO PFAM (DONE 7/14/05, Fan)
 # hgMapViaSwissProt.c was updated to support this.
 
 # Create table that maps between known genes and Pfam domains
 hgMapViaSwissProt hg17 knownGene name proteinID Pfam knownToPfam
 
 
 ## EVOFOLD (DONE, 2005-07-15, Jakob (jsp) ) 
 # EvoFold is a new comparative method for predicting functional RNA
 # secondary structures based on multiple sequence alignnments. The
 # predictions generated for the EvoFold track are based on the most
 # conserved elements of the 8-way alignment (multiz8way). The current
 # data is the result of a pilot study (ongoing research of mine), the
 # procedure used to generate the data will therefore be simplified
 # when forthcoming evofold tracks for other organism are made. The
 # documentation therefore skips the actual data generation, and
 # instead starts with a data file I provide.
 
   ssh -C hg17
   mkdir -p /cluster/data/hg17/bed/evofold
   cd /cluster/data/hg17/bed/evofold
   cp /cluster/home/jsp/data/rnass/genome-scan/vertebrate/folds_hg17.bed foldsHg17.bed
   # The folds_hg17.bed is a 9-column bed file: column 1-6 provide
   # standard bed information, column 7 is element length, column 8 is
   # the RNA secondary structure in parentheses format, and column nine
   # is a commaseparated list of position specific confidence scores
   # (floats).
   hgLoadBed -notItemRgb -sqlTable=/cluster/home/jsp/prog/kent/src/hg/lib/evofold.sql hg17 evofold foldsHg17.bed
 
 ##########################################################################
 # TRANSFRAG PHASE 2 TABLES  - lifted from hg15 (Jakob Skou Pedersen)
 # Done: July 21, 2005
 #
 # These tables were lifted for use in my own research, but may be used
 # for the 'Affymetrix Transcriptome Project Phase 2' tracks.
   ssh -C hgwdev
   mkdir -p /cluster/data/hg17/bed/transfrag
   cd /cluster/data/hg17/bed/transfrag
   # lifting transfrag tables from hg15 via hg16 to hg17
   for name in A375CytosolicPolyAPlusTnFg FHs738LuCytosolicPolyAPlusTnFg HepG2CytosolicPolyAMinusTnFg HepG2CytosolicPolyAPlusTnFg HepG2NuclearPolyAMinusTnFg HepG2NuclearPolyAPlusTnFg JurkatCytosolicPolyAPlusTnFg NCCITCytosolicPolyAPlusTnFg PC3CytosolicPolyAPlusTnFg SKNASCytosolicPolyAPlusTnFg U87CytosolicPolyAPlusTnFg; do
     echo "select chrom, chromStart, chromEnd, name from ${name};" | hgsql hg15 | sed -e 1d > ${name}Hg15.bed
     liftOver ${name}Hg15.bed /cluster/data/hg15/bed/liftOver/hg15ToHg16.over.chain ${name}Hg16.bed unmappedHg16.bed
     liftOver ${name}Hg16.bed /cluster/data/hg16/bed/liftOver/hg16ToHg17.over.chain ${name}Hg17.bed unmappedHg17.bed
     echo "hg16 unmapped count for ${name}: " `grep "#" unmappedHg16.bed | wc -l | awk '{print $1}'`
     echo "hg17 unmapped count for ${name}: " `grep "#" unmappedHg17.bed | wc -l | awk '{print $1}'`
     hgLoadBed hg17 ${name} ${name}Hg17.bed
     # clean up
     rm ${name}Hg15.bed ${name}Hg16.bed unmappedHg16.bed unmappedHg17.bed
   done
 
 # GLADSTONE ARRAY TRACK (DONE 7/19/2005 Andy)
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mkdir gladHumES
     cd gladHumES/
     cp /cluster/data/hg16/bed/geneAtlas2/geneAtlas2.bed .
     cut -f1-12 geneAtlas2.bed > bed.hg16
     liftOver bed.hg16 /gbdb/hg16/liftOver/hg16ToHg17.over.chain bed.hg17 /dev/null
     hgMapMicroarray bed.hg17.data hgFixed.gladHumESRatio \
        -bedIn bed.hg17
 #Loaded 11087 rows of expression data from hgFixed.gladHumESRatio
 #Mapped 10925,  multiply-mapped 382, missed 23266, unmapped 162
     hgLoadBed hg17 gladHumES bed.hg17.data
 
 # PHASTODDS GENESORTER COLUMN (DONE 7/28/2005 Andy)
     ssh kolossus
     cd /panasas/store/andy
     mkdir phastOdds
     cd phastOdds/
     export PATH=${PATH}:/cluster/bin/phast/x86_64
     mkdir sso beds
     cat > runChrom.sh << "_EOF_"
 #!/bin/bash
 c=$1
 numDir=`echo ${c#chr} | sed 's/_random//'`
 ALNDIR=/cluster/data/hg17/bed/multiz10way
 
 echo msa_view $c
 /cluster/bin/phast/x86_64/msa_view --in-format MAF ${ALNDIR}/maf/${c}.maf --refseq /cluster/data/hg17/${numDir}/${c}.fa > /tmp/${c}.sso
 
 echo phastCons $c
 /cluster/bin/phast/x86_64/phastOdds -f ${ALNDIR}/cons/run.elements/ave.cons.mod -b ${ALNDIR}/cons/run.elements/ave.noncons.mod -g kg/${c}.bed /tmp/${c}.sso > /tmp/${c}.phastOdds.gtf
 cp /tmp/${c}.sso sso/
 rm /tmp/${c}.sso
 cp /tmp/${c}.phastOdds.gtf gtfs/
 rm /tmp/${c}.phastOdds.gtf
 echo $c done
 _EOF_
     ssh hgwdev
     cd /panasas/store/andy/phastOdds
     genePredToGtf hg17 knownGene kg.gtf
     exit 
     for c in `cut -f1 kg.gtf | sort | uniq`; do 
        grep "\b${c}\b" kg.gtf > kg/${c}.gtf; 
     done
     for f in kg/*.bed; do
        c=`basename $f .bed`;
        echo $c;
        ./runChrom.sh $c;
        addPhastOddsExons $f gtfs/$c.phastOdds.gtf beds/$c.bed
     done
     cat beds/* | sort -k4,4 -k1,1 -k2,2n -k3,3n > phastOdds.kg.bed
     cat > phastOdds.sql << "EOF"
 CREATE TABLE phastOdds (
     bin smallint not null, # Speedup.
     chrom varchar(255) not null,        # Human chromosome or FPC contig
     chromStart int unsigned not null,   # Start position in chromosome
     chromEnd int unsigned not null,     # End position in chromosome
     name varchar(255) not null, # Name of item
               #Indices
     score float not null, # phastOdds score.
     index(chrom(8),bin),
     index(name(10))
 );
 EOF
 # <<
     hgLoadBed -sqlTable=phastOdds.sql hg17 phastOdds phastOdds.kg.bed
     # Actually I probably don't need that hg17 table.
     echo create table phastOdds select name, score from hg17.phastOdds | hgsql hgFixed
     echo create index nameIndex on phastOdds (name(10)) | hgsql hgFixed    
 
 ##########################################################################
 # Illumina SNPs (Heather, July 2005)
 # Source: Jeff Ohmen, PhD, johmen@illumina.com, 858/232-2702
 
 # using /cluster/store11 because /cluster/data/hg17 is on store5,
 # which is currently being restored
 
   cd /cluster/store11/heather/illumina
   fix.pl < LinkageIVbSNP.txt > illumina.bed
   hgLoadBed hg17 snpIllumina -tab -strict -sqlTable=snpIllumina.sql illumina.bed
   # Reading illumina.bed
   # Loaded 6008 elements of size 4
   # Sorted
   # Saving bed.tab
   # Loading hg17
 
   # note: 28 rows where chrom = "chrXY"
 
 # reload rankProp and psiBlast gene sorter tables to link with new
 # known genes (markd 2005-07-15)
    (spLoadRankProp -noMapFile=max1k.nomap hg17 rankProp --  /cluster/bluearc/markd/rankprop/results/hs.sw+tr/max1k.rankp.gz)   >&max1k.hg17.out
    (spLoadPsiBlast hg17 spPsiBlast  /cluster/bluearc/markd/rankprop/results/hs.sw+tr.eval.gz) >&pslBlast.hg17.out 
 
 
 # BLASTZ/CHAIN/NET CANFAM2 (DONE 8/2/05 angie - REDONE 12/12/05 angie - REDONE 2/6/06 angie)
 # Unfortunately, this was done with a corrupted 
 # /san/sanvol1/scratch/hg17/nib/chr5.nib the first time around; 
 # also, a linSpecRep bug in blastz-run-ucsc has been fixed since then.
 # Doh, then Kate pointed out that linSpecReps were not snipped properly --
 # I had omitted the BLASTZ_ABRIDGE_REPEATS line from the DEF!!!  
 # Added an error message to doBlastzChainNet.pl to catch that next time.
 # Therefore I'm moving aside the previous run:
     mv /usr/local/apache/htdocs/goldenPath/hg17/vsCanFam2{,.bak}
 # And rerunning...
     ssh kkstore02
     mkdir /cluster/data/hg17/bed/blastz.canFam2.2006-02-06
     cd /cluster/data/hg17/bed/blastz.canFam2.2006-02-06
     cat << '_EOF_' > DEF
 # human vs. dog
 
 BLASTZ_ABRIDGE_REPEATS=1
 
 # TARGET: Human
 SEQ1_DIR=/san/sanvol1/scratch/hg17/nib
 SEQ1_SMSK=/san/sanvol1/scratch/hg17/linSpecRep.notInDog
 SEQ1_LEN=/cluster/data/hg17/chrom.sizes
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY: Dog
 SEQ2_DIR=/san/sanvol1/scratch/canFam2/nib
 SEQ2_SMSK=/san/sanvol1/scratch/canFam2/linSpecRep.notInHuman
 SEQ2_LEN=/cluster/data/canFam2/chrom.sizes
 SEQ2_CHUNK=10000000
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/blastz.canFam2.2006-02-06
 '_EOF_'
     # << for emacs
     doBlastzChainNet.pl DEF -bigClusterHub pk -smallClusterHub pk \
       -workhorse pk \
       -blastzOutRoot /san/sanvol1/scratch/blastzHg17CanFam2Out >& do.log &
     tail -f do.log
     rm -f /cluster/data/hg17/bed/blastz.canFam2
     ln -s blastz.canFam2.2006-02-06 /cluster/data/hg17/bed/blastz.canFam2
 
 # RE-RUN NETTOAXT, AXTTOMAF FOR CANFAM2 (DONE 10/26/05 angie)
     # Kate fixed netToAxt to avoid duplicated blocks, which is important 
     # for input to multiz.  Regenerate maf using commands from sub-script 
     # netChains.csh generated by doBlastzChainNet.pl above.  
 # Obsoleted by re-run of hg17-canFam2 above 12/12/05 angie...
     ssh kolossus
     cd /cluster/data/hg17/bed/blastz.canFam2.2005-08-01/axtChain
     netSplit hg17.canFam2.net.gz net
     chainSplit chain hg17.canFam2.all.chain.gz
     cd ..
     mv axtNet axtNet.orig
     mkdir axtNet
     foreach f (axtChain/net/*.net)
       netToAxt $f axtChain/chain/$f:t:r.chain \
         /panasas/store/hg17/nib /iscratch/i/canFam2/nib stdout \
       | axtSort stdin stdout \
       | gzip -c > axtNet/$f:t:r.hg17.canFam2.net.axt.gz
     end
     rm -r mafNet
     mkdir mafNet
     foreach f (axtNet/*.hg17.canFam2.net.axt.gz)
       axtToMaf -tPrefix=hg17. -qPrefix=canFam2. $f \
             /cluster/data/hg17/chrom.sizes /cluster/data/canFam2/chrom.sizes \
             stdout \
       | gzip -c > mafNet/$f:t:r:r:r:r:r.maf.gz
     end
     rm -r axtChain/{chain,net}/ axtNet.orig
 
 
 ############
 # Sangamo/EIO DNaseI Hypersensitive Sites (2005-08-15 kate)
 #   (Sangamo Biosciences and European Inst. Oncology)
 #   Contact: Fyodor Umov (fumov@sangamo.com)
 
     cd /cluster/data/hg17/bed
     mkdir sangamo
     cd sangamo
     grep chr 3314_hs_sites_browser.bed | grep -v browser | \
         hgLoadBed -noBin hg17 sangamoDnaseHs stdin
             # Loaded 3314 elements of size 6
     checkTableCoords -table=sangamoDnaseHs hg17
 
     # use "antiword" to create plain text from .doc description file
 
 # UPDATE WGRNA TRACK (DONE, 2005-08-24, Fan)
 
   ssh hgwdev
   cd /cluster/data/hg17/bed
 
   mkdir wgRna-2005-08-24
   cd wgRna-2005-08-24
 
 # Received the data file, wgtrack_aug2005.txt, from Michel Weber's email 
 # (Michel.Weber@ibcg.biotoul.fr)
 # and place it under cd /cluster/data/hg17/bed/wgRna-2005-08-24.
 
   cut -f 2-10 wgtrack_aug2005.txt >wgRna.tab
   vi wgRna.tab
   
 # edit wgRna.tab to take out the first line of data field labels.
 
     hgLoadBed -sqlTable=/cluster/home/fanhsu/src/hg/lib/wgRna.sql hg17 wgRna wgRna.tab
 
 # Compared to 7/5 data, one record updated, one record dropped, one record added, out of 741 records.
 
 # Generate snpMask files (Done Heather Sept. 1, 2005)
 # Takes about 10-15 minutes
 # Consumes about 1 gig of disk
   ssh hgwdev
   cd /usr/local/apache/htdocs/goldenPath/hg17
   mkdir snpMask
   cd snpMask
   foreach chrom ( `cat /cluster/data/hg17/chrom.lst` )
       snpMaskChrom hg17 ${chrom} /gbdb/hg17/nib/${chrom}.nib ${chrom}.ambigs.fa
       gzip ${chrom}.ambigs.fa
   end
 
 #############################################################################
 # BLASTZ Mm7 (WORKING - 2005-09-06 - Hiram)
 #	Experiment, try the alignments without the linage specific
 #	repeats
     ssh pk
     mkdir /cluster/data/hg17/bed/blastzMm7.2005-09-06
     cd /cluster/data/hg17/bed
     ln -s blastzMm7.2005-09-06 blastz.mm7
     cd blastzMm7.2005-09-06
     cat << '_EOF_' > DEF
 # human vs mouse
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/x86_64:/cluster/home/angie/schwartzbin
 
 BLASTZ=blastz.v7.x86_64
 BLASTZ_H=2000
 BLASTZ_M=50
 BLASTZ_ABRIDGE_REPEATS=0
 
 # TARGET: Human Hg17
 SEQ1_DIR=/cluster/bluearc/hg17/bothMaskedNibs
 SEQ1_LEN=/cluster/bluearc/hg17/chrom.sizes
 SEQ1_CTGDIR=/cluster/bluearc/hg17/bothMaskedNibs
 SEQ1_CTGLEN=/cluster/bluearc/hg17/chrom.sizes
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=500000
 SEQ1_LAP=50
 
 # QUERY: Mouse Mm7
 SEQ2_DIR=/cluster/bluearc/mm7/mm7.2bit
 SEQ2_LEN=/cluster/bluearc/mm7/chrom.sizes
 SEQ2_CTGDIR=/cluster/bluearc/mm7/mm7Chroms_RandomContigs.2bit
 SEQ2_CTGLEN=/cluster/bluearc/mm7/mm7Chroms_RandomContigs.sizes
 SEQ2_LIFT=/cluster/bluearc/mm7/Chroms_RandomContigs.lft
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=3000000000
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/blastzMm7.2005-09-06
 TMPDIR=/scratch/tmp
 '_EOF_'
     # happy emacs
 
     cp -p /cluster/data/hg17/chrom.sizes ./S1.len
     twoBitInfo /cluster/bluearc/mm7/mm7Chroms_RandomContigs.2bit S2.len
 
     #	establish a screen to control this job
     screen
     time ./doBlastzChainNet.pl -stop chainMerge \
 	-bigClusterHub=pk \
 	`pwd`/DEF > toChainMerge.run.out 2>&1 &
     #	STARTED - 2005-09-06 - 11:00
     #	detach from screen session: Ctrl-a Ctrl-d
     #	to reattach to this screen session:
     time $HOME/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk \
 	-continue=cat -stop=cat \
 	`pwd`/DEF > catStep.out 2>&1 &
 
     time $HOME/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk \
 	-continue=chainRun \
 	`pwd`/DEF > continueChainRun.out 2>&1 &
 
     time $HOME/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk \
 	-continue=chainMerge -stop=chainMerge \
 	`pwd`/DEF > chainMerge.out 2>&1 &
     time $HOME/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk \
 	-continue=net -stop=net \
 	`pwd`/DEF > net.out 2>&1 &
 
     time $HOME/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk \
 	-continue=load -stop=load \
 	`pwd`/DEF > load.out 2>&1 &
 
     time $HOME/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk \
 	-swap -stop=load \
 	`pwd`/DEF > swap.out 2>&1 &
 
     #	Create plain pslChrom files to load a simple blastz track
     ssh kkstore02
     cd /cluster/data/hg17/bed/blastzMm7.2005-09-06
     mkdir -p pslChrom
 (cd pslParts; ls | awk -F"." '{print $1}' | sort -u) | while read C
 do
     echo -n "working ${C} ... "
     zcat pslParts/${C}.nib*.gz | gzip -c > pslChrom/${C}.psl.gz
     echo "done"
 done
     #	Load those alignments
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastzMm7.2005-09-06
 ls pslChrom | sed -e "s/.psl.gz//" | while read T
 do
 echo "hgLoadPsl -fastLoad -noTNameIx hg17 -table=${T}_blastzMm7 pslChrom/${T}.psl.gz"
 hgLoadPsl -fastLoad -noTNameIx hg17 -table=${T}_blastzMm7 pslChrom/${T}.psl.gz
 done
 
 
     #	After this same alignment was done with Hg17 query and Mm7
     #	target, came back to these swapped results in mm7 and manually loaded
     #	the swapped tables as: chainMm7LSR, chainMm7LSRLink and
     #	netMm7LSR
     #	41,223,632 total rows in the chainMm7Link split tables
     #	58,458,613 total rows in the chainMm7LSRLink table
 
     time featureBits hg17 chainMm7LSRLink
     #	959444893 bases of 2866216770 (33.474%) in intersection
     #	real    36m30.822s
     #	user    14m19.620s
     #	sys     5m13.910s
     time featureBits hg17 chainMm7Link
     #	955168137 bases of 2866216770 (33.325%) in intersection
     #	real    16m13.902s
     #	user    10m20.780s
     #	sys     3m42.810s
     #	And, their intersection:
     ssh kolossus
     time HGDB_CONF=~/.hg.conf.read-only featureBits hg17 \
 	chainMm7LSRLink chainMm7Link
     #	952667399 bases of 2866216770 (33.238%) in intersection
     #	real    38m53.448s
     #	user    8m38.853s
     #	sys     2m23.362s
 
 # LOAD ACEMBLY TRACK (DONE, 2005-09-12, Fan)
     mv /cluster/data/hg17/bed/acembly /cluster/data/hg17/bed/acembly_050217
     mkdir -p /cluster/data/hg17/bed/acembly
     cd /cluster/data/hg17/bed/acembly
     # Data is obtained from Jean Thierry-Mieg     mieg@ncbi.nlm.nih.gov
  
     wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35g.human.genes/acembly.ncbi_35g.genes.proteins.fasta.tar.gz
     wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35g.human.genes/acembly.ncbi_35g.genes.gff.tar.gz
     wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35g.human.genes/acembly.ncbi_35g.mrnas.fasta.tar.gz
     wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35g.human.genes/acembly.ncbi_35g.genes.pfamhits.tar.gz
     tar xvzf acembly.ncbi_35g.genes.gff.tar.gz
     tar xvzf acembly.ncbi_35g.genes.proteins.fasta.tar.gz
     cd acembly.ncbi_35.genes.gff
     # the acembly dataset for hg16 had problems with reverse blocks so
     # check for these
 cat << '_EOF_' > checkReversedBlocks
 for i in x1*.gff
 do
         echo -n "$i working ..."
         awk -F"\t" '
 {
 if ($4 > $5) {
     printf "reverse blocks problem for $1"
         printf "\n"
 }
 }
 ' $i > $i.fixed
         echo " done"
 done
 '_EOF_'
     # << this line makes emacs coloring happy
     chmod +x checkReversedBlocks
     ./checkReversedBlocks  
     ls -l *.fixed
     # all *.fixed files are empty so remove - there is no reversing of blocks
     rm *.fixed
 
     foreach f (x1.acemblygenes.*.gff)
       set c=$f:r:e
       egrep '^[a-zA-Z0-9]+\|NT_[0-9][0-9][0-9][0-9][0-9][0-9]' $f | \
         perl -wpe 's/^(\w+)\|(\w+)/$1\/$2/' > ctg-chr${c}_random.gff
       if (-e ../../../$c/lift/random.lft) then
         liftUp chr${c}_random.gff ../../../$c/lift/random.lft warn \
           ctg-chr${c}_random.gff
       endif
       grep -v ^$c\| $f | grep -v ^Hs | perl -wpe 's/^/chr/;' | \
         grep -v "^chr//" > chr$c.gff
       echo "done $c"
     end
     
     #- Load into database - use extended genePred
     ssh hgwdev
     cd /cluster/data/hg17/bed/acembly
     # Reloaded without -genePredExt 1/6/05:
     ldHgGene -gtf hg17 acembly acembly.ncbi_35.genes.gff/chr*.gff
     # for entry with 28212470 from chr6.gff, change to chr6 
     # and for 29124352 in chr6.gff, change to chr6 (1/13/05)
     echo 'update acembly set chrom = "chr6" where chrom = "chr28212470";' \
          | hgsql hg17 
     echo 'update acembly set chrom = "chr6" where chrom = "chr29124352";' \
          | hgsql hg17 
     
     # checkTableCoords and runGeneCheck to check data
     checkTableCoords hg17 acembly
 
 hgPepPred hg17 generic acemblyPep \
         acembly.ncbi_35.genes.proteins.fasta/*.fasta
 
 # create table of Acembly gene classifications
 cd /cluster/data/hg17/bed/acembly/acembly.ncbi_35.genes.gff
 rm acemblyClass.tab
 foreach f (x1.acemblygenes.*.gff)
 
 cut -f 9 $f |sed -e 's/;/\t/g' |sed -e 's/transcript_id //' >j.tmp
 cut -f 2 j.tmp >j2.tmp
 cut -f 3 j.tmp >j3.tmp
 paste j3.tmp j2.tmp|sed -e 's/Main_gene/main/g' |sed -e 's/Putative_gene/putative/g' |sed -e 's/ //g' >>acemblyClass.tab
 end
 
 rm *.tmp
 hgsql hg17 -e 'drop table acemblyClass'
 hgsql hg17 < ~src/hg/lib/acemblyClass.sql
 hgsql hg17 -e 'load data local infile "acemblyClass.tab" into table acemblClass.tab'
 hgsql hg17 -e 'delete from acemblyClass where class!="main" and class!="putative"'
 
 # build acemblyPep table
 
 hgPepPred hg17 generic acemblyPep \
         acembly.ncbi_35.genes.proteins.fasta/*.fasta
 
 # Please note, per email from Jean Thierry-Mieg on 9/9/05,
 # there are AceView genes (~10,000) without corresponding 
 # protein sequences.  They will fix it next time.
 
 ###########################################################################
 # LOADING AFFYTXNPHASE2 TRACK (sugnet)
 
 # cd to where data is downloaded.
 cd /cluster/store10/sugnet/affyTranscription/graphs/transcriptome.affymetrix.com/download/publication/polyA_minus/graphs
 
 # lift data from hg16 to hg17. This takes a long time.
 ./liftWigFilesHg16ToHg17.sh
 
 # make the .wib and .wig files. This takes a long time.
 ./makeWibWigHg17.sh
 
 # Copy .wib files to /cluster/data/hg17 files
 mkdir /cluster/data/hg17/bed/affyTxnPhase2/wigData/
 cp `find ./ -name "*.hg17.wib"` /cluster/data/hg17/bed/affyTxnPhase2/wigData/
 chmod 775 /cluster/data/hg17/bed/affyTxnPhase2/wigData/
 chmod 664 /cluster/data/hg17/bed/affyTxnPhase2/wigData/*
 
 # Make gbdb entry
 mkdir /gbdb/hg17/wib/affyTxnPhase2
 chmod 775 /gbdb/hg17/wib/affyTxnPhase2
 cd /gbdb/hg17/wib/affyTxnPhase2
 ln -s /cluster/data/hg17/bed/affyTxnPhase2/wigData/* .
 cd -
 
 # Load the database tables (using bash) this takes a while
 for file in `find ./ -name "*hg17.wig"`; do
   base=`basename $file .hg17.wig`
   echo "Doing ${base}Txn"
   hgLoadWiggle -pathPrefix=/gbdb/hg17/wib/affyTxnPhase2  hg17 ${base}Txn $file
 done
 
 # Do the transfrags
 cd ../transfrags
 ./liftHg16ToHg17.sh
 ./loadHg17Tnfg.sh
 
 # End of affyTxnPhase2
 
 ###########################################################################
 #  Creating download files for the affyTxnPhase2 data
 #		(DONE - 2006-11-20 - Hiram)
     #	Copy all of the data above to a temporary /san/sanvol1/scratch/
     #	location, and run the following script:
 #!/bin/sh
 
 mkdir -p rawData/hg17
 
 TOP=`pwd`
 export TOP
 for dir in `find ./ -type d | grep '_' | grep -v A375_cytosolic_polyAPlus | grep -v FHs738Lu_cytosolic_polyAPlus | grep -v HepG2_CytosolVsNucleusDifferenceGraphs | grep -v HepG2_cytosolic_polyAPlus | grep -v HepG2_cytosolic_polyAMinus | sed -e "s#^./##"`; do
 base=`echo $dir | sed -e 's/\.\///; s/\//_/g' | sed -e 's/polyA-/polyAMinus/g' | sed -e 's/-/_/g' | sed -e 's/\+/Plus/g' | $TOP/changeName.pl`
 RAW=$TOP/rawData/hg17/$base.data
 echo $RAW
    cd $dir;
    zcat `ls -1 *hg17.bed.gz` | bedSort stdin stdout | cut -f 1,2,3,4 | grep chr | $TOP/avgerizeBed.pl > $RAW
    cd $TOP;
 done
 
     #	Then copy the rawData/hg17/ results directory back to:
     /cluster/data/hg17/bed/affyTxnPhase2/rawResults/
     #	And deliver to hgdownloads via symlinks on hgwdev:
     cd /usr/local/apache/htdocs/goldenPath/hg17/affyTxnPhase2/
     # to:
     ln -s /cluster/data/hg17/bed/affyTxnPhase2/rawData/*.data.gz .
     #	Remove the san scratch data
 
 ###########################################################################
 # ALTGRAPHX TRACK (sugnet)
 
 /cluster/store1/sugnet/altSplice
 mkdir hg17-2005.03.28
 
 # First get the RNA clusters
 cd hg17-2005.03.28
 
 # Don't use RAGE libraries for clone bounds.
 ~/kent/src/hg/geneBounds/clusterRna/generateRageAccList.csh  hg17 rage.libs
 
 # Make spec file to run.
 foreach c (`echo 'select chrom from chromInfo' | hgsql hg17 | grep -v chrom`)
     set out = chrom/$c.bed
     echo "clusterRna -mrnaExclude=hg17.rage.libs hg17 /dev/null $out -chrom=$c" >> clusterRna.spec
 end
 
 # Tried running it on the minicluster, but can't connect to the 
 # cluster accounts so run it from here on hgwdev.
 ./clusterRna.spec >& clusterRna.log
 
 cd ..
 
 # Make script to setup parasol job file for raw altGraphX files on human
 cat << '_EOF_' > makeRun.sh
 #!/bin/sh
 
 for chrom in `echo "select chrom from chromInfo" | hgsql hg17 | grep -v chrom`; do
 echo 'echo "Doing $chrom"'
 echo "/cluster/home/sugnet/bin/i386/altSplice -db=hg17   -beds=rnaCluster/chrom/$chrom.bed -agxOut=agxs/hg17.$chrom.agx -consensus -weightMrna -localMem -minAli=.95 -minCover=.5 -chromNib=/cluster/data/hg17/nib/$chrom.nib"
 done
 '_EOF_'
     # << this line makes emacs coloring happy
 mkdir agxs
 chmod 755 makeRun.sh
 
 # Minicluster down, have to run on hgwdev.
 ./makeRun.sh > toRun.sh
 chmod 755 toRun.sh
 ./toRun.sh >& toRun.log &
 
 cat agxs/*.agx > hg17.agx
 
 # make raw altGraphX files for mouse
 mkdir ../mm5-2005.03.28/
 cd ../mm5-2005.03.28/
 
 # make the rnaClusters
 mkdir rnaCluster
 cd rnaCluster/
 mkdir chrom
 
 # Don't use RAGE libraries for clone bounds.
 ~/jk/hg/geneBounds/clusterRna/generateRageAccList.csh mm5 rage.libs
 # Doing select on mm5 into mm5.rage.libs
 # Done.
 
 # Make spec file to run.
 foreach c (`echo 'select chrom from chromInfo' | hgsql mm5 | grep -v chrom`)
     set out = chrom/$c.bed
     echo "clusterRna -mrnaExclude=mm5.rage.libs mm5 /dev/null $out -chrom=$c" >> clusterRna.spec
 end
 
 # Tried running it on the minicluster, but can't connect to the 
 # cluster accounts so run it from here on hgwdev.
 chmod 755 clusterRna.spec
 ./clusterRna.spec >& clusterRna.log &
 
 
 # Make the gene bounds in rnaCluster.
 mkdir agxs
 
 # This script generates the jobs, one per chromosome.
 echo << '_EOF_' > makeRun.sh
 #!/bin/sh
 
 for chrom in `echo "select chrom from chromInfo" | hgsql mm5 | grep -v chrom`; do
 echo 'echo "Doing $chrom"'
 echo "/cluster/home/sugnet/bin/i386/altSplice -db=mm5   -beds=rnaCluster/chrom/$chrom.bed -agxOut=agxs/mm5.$chrom.agx -consensus -weightMrna -localMem -minAli=.95 -minCover=.5 -chromNib=/cluster/data/mm5/nib/$chrom.nib"
 done
 '_EOF_'
 chmod 755 makeRun.sh
 ./makeRun.sh > toRun.sh
 chmod 755 toRun.sh
 ./toRun.sh >& toRun.log & # Takes an hour or so...
 
 # Consolodiate all of the records in a single file.
 cat agxs/*.agx > mm5.agx
 
 
 # Make the orthologous splicing graphs.
 mkdir orthoSpliceExoniphy
 cd orthoSpliceExoniphy/
 
 # Get the exoniphy exons...
 echo "select chrom, txStart, txEnd, name, id, strand from exoniphy order by chrom, txStart;" | hgsql hg17 | grep -v txStart > hg17.exoniphy.bed
 
 # Set up the commands for the orthosplice run.
 echo 'select chrom, size from chromInfo' | hgsql hg17 | grep -v chrom > chromSizes.tab
 ln -s /cluster/data/hg17/bed/blastz.mm5/axtChain/mouseNet/ nets
 ln -s /cluster/data/hg17/bed/blastz.mm5/axtChain/chain/ chains
 mkdir agx report logs
 
 cat << '_EOF_' > makeRun.sh
 
 #!/usr/bin/perl -w
 
 open(IN, 'chromSizes.tab') or die "Can't open chromSizes.tab\n";
 while(<IN>) {
     chomp;
     @w = split;
     print "/cluster/home/sugnet/bin/i386/orthoSplice -chromSize=$w[1] -exonFile=hg17.exoniphy.bed -trumpNum=3 -chrom=$w[0] -altInFile=../agxs/hg17.$w[0].agx -orthoAgxFile=../../mm5-2005.03.28/mm5.agx -db=hg17 -orthoDb=mm5 -netFile=nets/$w[0].net -chainFile=chains/$w[0].chain -commonFile=agx/$w[0].hg17.mm5.cons.t3.agx -reportFile=report/$w[0].hg17.report -edgeFile=report/$w[0].hg17.edge.report >& logs/$w[0].test.log\n";
 }
 '_EOF_'
 # << emacs
 
 ./makeRun.sh > orthoSplice.para.spec
 ssh kki
 cd /cluster/store1/sugnet/altSplice/hg17-2005.03.28/orthoSpliceExoniphy
 para create orthoSplice.para.spec
 para push
 
 cat agx/*.agx > hg17.mm5.t3.exoniphy.agx
 
 # Make bed file
 agxToBed hg17.mm5.t3.exoniphy.agx hg17.mm5.t3.exoniphy.bed
 
 # Load up files
 hgLoadBed hg17 agxBed hg17.mm5.t3.exoniphy.bed
 hgLoadBed -notItemRgb -sqlTable=/cluster/home/sugnet/kent/src/hg/lib/altGraphX.sql hg17 altGraphX hg17.mm5.t3.exoniphy.agx
 
 # end altGraphX track
 
 # EXONWALK TRACK (sugnet)
 
 # make altGraphX track (see above)
 
 cd /cluster/store1/sugnet/altSplice/hg17-2005.03.28/orthoSpliceExoniphy
 
 cd exonWalk
 mkdir beds
 
 # Make parasol script.
 foreach file (`ls ../agx/*.agx`)
   set base=`basename $file .agx`
   echo "/cluster/home/sugnet/bin/i386/exonWalk db=hg17 minPercent=0 trumpSize=100000 $file beds/$base.bed" >> exonWalk.para.spec
 end
 para create exonWalk.para.spec
 para push
 cat beds/*.bed > hg17.mm5.cons.t3.exoniphy.bed
 
 # Predict orfs
 mkdir orfs
 cd orfs
 mkdir bedOrf beds fa borf
 cp ~/store1/altSplice/hg17-2005.01.09/orthoSpliceExonify/exonWalk/orfs.mrna2/*.sh ./
 splitFile ../../hg17.mm5.cons.t3.exoniphy.bed 500 exonWalk.
 cat < < '_EOF_' > makeFa.sh
 #!/bin/sh
 
 for file in "$@"
 do
  base=`basename $file`
  echo "Doing $file"
  echo "sequenceForBed -db=hg17 -bedIn=$file -fastaOut=fa/$base.fa "
  sequenceForBed -db=hg17 -bedIn=$file -fastaOut=fa/$base.fa 
 done
 '_EOF_'
 chmod 755 makeFa.sh
 makeFa.sh beds/*
 
 # Run borf lots of times...
 makeSpec.sh beds/* > para.spec
 para create para.spec
 para push
 mkdir genePred
 cat << '_EOF_' > makeGenePred.sh
 #!/bin/sh
 
 for file in "$@"
 do
   base=`basename $file`
   /cluster/home/sugnet/bin/i386/borfMatcher -keepNmd beds/$base borf/$base.borf bedOrf/$base.bed genePred/$base.gp 
 done
 '_EOF_'
     # << this line makes emacs coloring happy
 chmod 755 makeGenePred.sh 
 makeGenePred.sh beds/*
 
 cat beds/* > hg17.mm5.exonWalk.bed
 cat genePred/*.gp > hg17.mm5.exonWalk.gp
 
 wc *.bed *.gp
 # 155470 1865640 29956585 hg17.mm5.exonWalk.bed
 #  98433  984330 32495119 hg17.mm5.exonWalk.gp
 
 # Load it into the database.
 ldHgGene -predTab hg17 exonWalk hg17.mm5.exonWalk.gp 
 
 # end exonWalk
 
 ####################################################################
 ### hapmapRecombRate (Daryl; September 19, 2005)
     Lifted from hg16; see makeHg16.doc for details
     # Update (Jen; October 25, 2005)
     Data points that lifted to chroms other than 1-22 + X removed
     before release to RR (confirmed with Daryl).
     chr4_random: 11 data points
     chr6_hla_hap1: 25 data points
 ### hapmapRecombHotspot (Daryl; September 19, 2005)
     Lifted from hg16; see makeHg16.doc for details
 
 ### HapMap SNPs (Daryl; February 4, 2006)
   # most of this work was done in October and November 2005 for the ENCODE workshop
     cd /cluster/store4/gs.17/build34/bed/hapmap/frequencies/2005-10/non-redundant/hapmapSnps
     ln -sf ../hg17.daf.all/daf.txt.gz .
     ln -sf ../hg17.panTro1.rheMac1.txt.gz .
     zcat hg17.panTro1.rheMac1.txt | grep -v chrom | sort >! hg17.panTro1.rheMac1.sort.txt
     zcat daf.txt                  | grep -v chrom | sort >! daf.sort.txt
     # check that order matches; should be empty
     paste hg17.panTro1.rheMac1.sort.txt daf.sort.txt | awk '$1!=$17||$2!=$18||$3!=$19||$4!=$20||$5!=$21||$6!=$22||$7!=$23||$8!=$24||$11!=$25||$12!=$27||$15!=$26||$16!=$28{print $0;}'
     paste hg17.panTro1.rheMac1.sort.txt daf.sort.txt | awk '{printf "%s\t%d\t%d\t%s\t0\t%c\t%c\t%c\t%c\t%c\t%d\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n",$1,$2,$3,$4,$6,$7,$8,$11,$15,$12,$16,$29,$30,$31,$32,$33}' >! hapmapSnps.bed
     hgLoadBed hg17 hapmapSnps -sqlTable=hapmapSnps.sql hapmapSnps.bed
 
 ############################################################################################
 # HapMap SNPs rel21a (Feb. 2007, Heather)
 # June 2007 [partial fix of hapmapAllelesSummary released 6/25/07:
 # using hg17 instead of hg18 liftOver files... for most but not all
 # chroms! :( not documented below; error found by user]
 # 1/11/08, 1/24/08 (angie): regenerated hapmapAllelesSummary with corrected
 # hapmapAllelesChimp.
 
 # get files for each chrom, for each population
 # these contain data for all individuals
 # not using the JPT+CHB files
 ssh kkstore05
 cd /cluster/store12/snp/hapmap/rel21a/genotypes/2007-01/rs_strand/non-redundant/zips
 wget http://www.hapmap.org/downloads/genotypes/2007-01/rs_strand/non-redundant/* 
 
 # get population data (needed to define children in CEU and YRI trios)
 cd /cluster/store12/snp/hapmap
 wget http://www.hapmap.org/downloads/samples_individuals/*gz
 gunzip pedinfo2sample_CEU.txt.gz
 filterPedigree.pl < pedinfo2sample_CEU.txt > CEU.filtered
 cp CEU.filtered rel21a/genotypes/2007-01/rs_strand/non-redundant/CEU.list
 gunzip pedinfo2sample_YRI.txt.gz
 filterPedigree.pl < pedinfo2sample_YRI.txt > YRI.filtered
 cp YRI.filtered rel21a/genotypes/2007-01/rs_strand/non-redundant/YRI.list
 
 #!/usr/bin/env perl
 while (<STDIN>) {
     my @fields = split;
     if ($fields[2] == 0 && $fields[3] == 0) {
         @subfields = split /:/, $fields[6];
         print $subfields[4];
         print "\n";
     }
 }
 
 cd /cluster/store12/snp/hapmap/rel21a/genotypes/2007-01/rs_strand/non-redundant
 zcat zips/*chr22_CEU* | head -1 > header.CEU
 zcat zips/*chr22_YRI* | head -1 > header.YRI
 # add carriage returns to header.CEU and header.YRI
 grep -n -f CEU.list header.CEU > offsets.CEU
 grep -n -f YRI.list header.YRI > offsets.YRI
 # delete ids in offsets.CEU and offsets.YRI so just column numbers remain
 
 # for each population, combine all chroms, and combine all individuals
 # for CEU and YRI, filter out children from trios
 # This creates CEU.merge, CHB.merge, JPT.merge, YRI.merge
 
 ./merge.csh
 
 #!/bin/tcsh 
 rm -f CEU.merge
 rm -f CHB.merge
 rm -f JPT.merge
 rm -f YRI.merge
 
 foreach chrom (`cat chrom.list`)
 
     echo $chrom
 
     # CEU
     echo "CEU"
     set fileName=`echo $chrom | awk '{printf "zips/genotypes_%s_CEU_r21a_nr.txt.gz", $1}'`
     zcat $fileName | filterCEU.pl >> CEU.merge
 
     # CHB
     echo "CHB"
     set fileName=`echo $chrom | awk '{printf "zips/genotypes_%s_CHB_r21a_nr.txt.gz", $1}'`
     zcat $fileName | filterCHB.pl >> CHB.merge
 
     # JPT
     echo "JPT"
     set fileName=`echo $chrom | awk '{printf "zips/genotypes_%s_JPT_r21a_nr.txt.gz", $1}'`
     zcat $fileName | filterJPT.pl >> JPT.merge
 
     # YRI
     echo "YRI"
     set fileName=`echo $chrom | awk '{printf "zips/genotypes_%s_YRI_r21a_nr.txt.gz", $1}'`
     zcat $fileName | filterYRI.pl >> YRI.merge
 
 end
 
 
 # Below is filterCEU.pl
 # The others are very similar: YRI uses "offsets.YRI"
 # CHB and JPT just read the input directly
 
 #!/usr/bin/env perl 
 
 # read in a list of the columns that we are keeping
 sub initList {
 open LIST, "offsets.CEU";
 chomp(@list = <LIST>);
 close LIST;
 $listSize = @list;
 }
 
 &initList;
 
 while (<STDIN>) {
     my @fields = split;
     # skip header
     if ($fields[0] eq "rs#") { next; }
     # chrom
     print $fields[2];
     print " ";
     # position: add zero-based start coord
     print $fields[3] - 1;
     print " ";
     print $fields[3];
     print " ";
     # rsId
     print $fields[0];
     print " ";
     # score
     print "0 ";
     # strand
     print $fields[4];
     print " ";
     # observed
     print $fields[1];
     print " ";
     @alleles = ();
     for ( my $loop = 0; $loop < $listSize; $loop++ ) {
 	push (@alleles, $fields[@list[$loop]-1]);
     }
     # N is used for missing data
     $nCount = 0;
     # counts
     $aCountHomo = 0;
     $cCountHomo = 0;
     $gCountHomo = 0;
     $tCountHomo = 0;
     $aCountHetero = 0;
     $cCountHetero = 0;
     $gCountHetero = 0;
     $tCountHetero = 0;
     foreach $allele (@alleles) {
         $parent1 = substr($allele, 0, 1);
         $parent2 = substr($allele, 1, 1);
 	# Ns must be together
 	if ($parent1 eq "N" && $parent2 ne "N") { die "Unexpected input"; }
 	if ($parent2 eq "N" && $parent1 ne "N") { die "Unexpected input"; }
 	if ($parent1 eq "N" && $parent2 eq "N") { $nCount++; next; }
 
 	if ($parent1 eq "A" && $parent2 eq "A") { 
 	    $aCountHomo = $aCountHomo + 2;
 	    next; 
 	}
 	if ($parent1 eq "C" && $parent2 eq "C") { 
 	    $cCountHomo = $cCountHomo + 2;
 	    next; 
 	}
 	if ($parent1 eq "G" && $parent2 eq "G") { 
 	    $gCountHomo = $gCountHomo + 2;
 	    next; 
 	}
 	if ($parent1 eq "T" && $parent2 eq "T") { 
 	    $tCountHomo = $tCountHomo + 2;
 	    next; 
 	}
 
 	if ($parent1 eq "A") { $aCountHetero++; }
 	if ($parent1 eq "C") { $cCountHetero++; }
 	if ($parent1 eq "G") { $gCountHetero++; }
 	if ($parent1 eq "T") { $tCountHetero++; }
 	if ($parent2 eq "A") { $aCountHetero++; }
 	if ($parent2 eq "C") { $cCountHetero++; }
 	if ($parent2 eq "G") { $gCountHetero++; }
 	if ($parent2 eq "T") { $tCountHetero++; }
     }
     print "A ";
     print $aCountHomo;
     print " ";
     print $aCountHetero;
     print " ";
     print "C ";
     print $cCountHomo;
     print " ";
     print $cCountHetero;
     print " ";
     print "G ";
     print $gCountHomo;
     print " ";
     print $gCountHetero;
     print " ";
     print "T ";
     print $tCountHomo;
     print " ";
     print $tCountHetero;
     print " ";
 
     print "\n";
 
 }
 
 # << emacs
 # Switch to C programs from kent/src/hg/snp/snpLoad.
 
 # Determine allele1 and allele2 (set allele2 to "none" if monomorphic)
 # Alleles are in alphabetical order
 # Calculate score (minor allele frequency)
 # Log and skip if wrong number of elements in row
 # Log and skip if triallelic or quadallelic
 # Log and skip degenerate case (no alleles) 
 # No errors this run
 # Still running on kkstore05
 # Could rename "hapmap1" to "hapmapCondense"
 /cluster/home/heather/kent/src/hg/snp/snpLoad/hapmap1 CEU.merge CEU.condense
 wc -l hapmap1.log
 /cluster/home/heather/kent/src/hg/snp/snpLoad/hapmap1 CHB.merge CHB.condense
 wc -l hapmap1.log
 /cluster/home/heather/kent/src/hg/snp/snpLoad/hapmap1 JPT.merge JPT.condense
 wc -l hapmap1.log
 /cluster/home/heather/kent/src/hg/snp/snpLoad/hapmap1 YRI.merge YRI.condense
 wc -l hapmap1.log
 
 # save some space
 gzip *merge
 
 # load
 ssh hgwdev
 cd /cluster/store12/snp/hapmap/rel21a/genotypes/2007-01/rs_strand/non-redundant
 cp /cluster/home/heather/kent/src/hg/lib/hapmapSnps.sql .
 # modify hapmapSnps for 4 populations
 hgLoadBed hg17 hapmapSnpsCEU -sqlTable=hapmapSnpsCEU.sql CEU.condense
 hgLoadBed hg17 hapmapSnpsCHB -sqlTable=hapmapSnpsCHB.sql CHB.condense
 hgLoadBed hg17 hapmapSnpsJPT -sqlTable=hapmapSnpsJPT.sql JPT.condense
 hgLoadBed hg17 hapmapSnpsYRI -sqlTable=hapmapSnpsYRI.sql YRI.condense
 
 # save some more space
 ssh kkstore05
 cd /cluster/store12/snp/hapmap/rel21a/genotypes/2007-01/rs_strand/non-redundant
 gzip *condense
 
 # sanity check
 mysql> select count(*) from hapmapSnpsCEU where homoCount1 + homoCount2 + heteroCount = 0;
 +----------+
 | count(*) |
 +----------+
 |        0 |
 +----------+
 
 mysql> select count(*) from hapmapSnpsCHB where homoCount1 + homoCount2 + heteroCount = 0;
 +----------+
 | count(*) |
 +----------+
 |        0 |
 +----------+
 
 mysql> select count(*) from hapmapSnpsJPT where homoCount1 + homoCount2 + heteroCount = 0;
 +----------+
 | count(*) |
 +----------+
 |        0 |
 +----------+
 
 mysql> select count(*) from hapmapSnpsYRI where homoCount1 + homoCount2 + heteroCount = 0;
 +----------+
 | count(*) |
 +----------+
 |        0 |
 +----------+
 
 mysql> select max(score) from hapmapSnpsCEU;  
 +------------+
 | max(score) |
 +------------+
 |        500 |
 +------------+
 
 
 # create indexes
 mysql> alter table hapmapSnpsCEU add index name (name);
 mysql> alter table hapmapSnpsCEU add index chrom (chrom, bin);
 mysql> alter table hapmapSnpsCHB add index name (name);
 mysql> alter table hapmapSnpsCHB add index chrom (chrom, bin);
 mysql> alter table hapmapSnpsJPT add index name (name);
 mysql> alter table hapmapSnpsJPT add index chrom (chrom, bin);
 mysql> alter table hapmapSnpsYRI add index name (name);
 mysql> alter table hapmapSnpsYRI add index chrom (chrom, bin);
 
 # 2nd step in processing: create hapmapSnpsCombined
 ssh hgwdev
 cd /cluster/data/hg17/bed/hapmap/rel21a
 /cluster/home/heather/kent/src/hg/snp/snpLoad/hapmap2 hg17
 hgLoadBed hg17 hapmapSnpsCombined -sqlTable=/cluster/home/heather/kent/src/hg/lib/hapmapSnpsCombined.sql hapmapSnpsCombined.tab
 
 # create indexes (not used by browser)
 mysql> alter table hapmapSnpsCombined add index name (name);
 mysql> alter table hapmapSnpsCombined add index chrom (chrom, bin);
 
 # errors
 # nothing that isn't biallelic
 # nothing with mixed positions
 # over 500K that were not available in all 4 populations
 # YRI: 187,485
 # CEU: 129,359
 # CHB and JPT: 97,095
 
 # Also, 2 strand corrections done
 grep -v missing hapmap2.errors
 # different strands for rs1621378
 # different strands for rs5768
 
 # cleanup to save space
 rm hapmapSnpsCombined.tab
 
 # monomorphism
 
 YRI        867,835 
 CEU      1,252,743 
 CHB      1,496,438 
 JPT      1,539,094 
 combined   607,393 
 
 # observed strings
 # why is A/T different from other transversions?
 
 A/G      1,344,043
 C/T      1,344,542
 A/C        352,875
 A/T        275,670
 C/G        354,299
 G/T        354,149
 triallelic   1,370
 quadallelic    403
 other        1,226
 
 # some details on the others:
 125     -/A/T
 124     -/A/G
 107     -/C/T
 85      -/A/C
 79      -/G/T
 25      -/C/G
 
 18      -/A/C/T
 13      -/A/G/T
 12      -/A/C/G
 11      -/C/G/T
 
 7       (LARGEINSERTION)
 5       (LARGEDELETION)
 6       microsat
 2       het
 
 # check for collisions (more than one SNP at the same location)
 # none found
 ssh hgwdev
 cd /cluster/data/hg17/bed/hapmap/rel21a
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpCheckCluster2 hg17 hapmapSnpsCombined > snpCheckCluster2.out
 
 # check against hg17.snp125
 ssh hgwdev
 cd /cluster/data/hg17/bed/hapmap/rel21a
 /cluster/home/heather/kent/src/hg/snp/snpLoad/hapmapLookup hg17 hapmapSnpsCombined snp125 snp125Exceptions
 
 # 1817 total that are complex type from dbSNP (hapmapLookup.log)
 # This is not based on observed string, only on size, class and locType
 # 1176 class = mixed
 #  616 class = single but locType != exact
 #   11 class = named
 #    6 class = insertion
 #    4 class = deletion
 #    2 class = microsat
 #    2 class = het
 
 # Generally if class = single the observed string is bi-allelic as expected
 # Exceptions to that:
 #    rs700519 quad-allelic, locType = rangeDeletion
 #    rs1572672 tri-allelic, locType = between
 #    rs2357412 tri-allelic, locType = range
 #    rs2364671 tri-allelic, locType = rangeSubstitution
 #    rs3959788 quad-allelic, locType = between
 
 # 74 items in hapmapLookup.error
 # 59 reverse complement (that's okay)
 #  7 multiple alignment (6 from chrX:154,219,000-154,220,500 which is close to PAR)
 # Also rs6645103 which is PAR 
 mysql> select chrom, chromStart, chromEnd, strand, observed, class, locType, weight from snp125 where name = "rs6645103";
 +-------------+------------+----------+--------+----------+--------+---------+--------+
 | chrom       | chromStart | chromEnd | strand | observed | class  | locType | weight |
 +-------------+------------+----------+--------+----------+--------+---------+--------+
 | chrX_random |     273788 |   273789 | -      | C/T      | single | exact   |      3 |
 | chrX        |     421141 |   421142 | +      | C/T      | single | exact   |      3 |
 | chrY        |     421141 |   421142 | +      | C/T      | single | exact   |      3 |
 +-------------+------------+----------+--------+----------+--------+---------+--------+
 
 
 #  4 observed with dbSNP complex, hapmap biallelic
 #  all positive strand, locType = between
 #  all cluster errors in dbSNP
 #  rs10485830
 #  rs7625205 (intronic)
 #  rs713582
 #  rs11403115 (class = insertion)
 
 # 3 observed mismatch
 # all dbSNP clustering error
 # rs2230624 (tri-allelic)
 # rs3963317 (monomorphic in hapmap, rangeSubstitution in dbSNP)
 # rs5017503 (monomoprhic in hapmap)
 
 # a strange one
 # rs731449
 # dbSNP strand = -, hapmap strand = +
 # dbSNP observed = G/T, hapmap observed = C/T
 # dbSNP clustering error rs2321451, which is C/T
 # hapmap monomorphic for T
 # ortho A
 # no repeats, no genes, no mRNAs, no conservation
 
 # Counts of rows where 3 populations have one major allele, the 4th has the other
 
 hapmapMixed hg17
 # countCEU = 162931
 # countCHB = 46543
 # countJPT = 48791
 # countYRI = 309105
 
 # Generate summary table (used by filters)
 # Summary table includes ortho allele and ortho qual score
 # Summary table score is heterozygosity
 # Individual zygosity is *not* preserved
 ssh hgwdev
 # 6/25/08: regenerated with mostly-corrected hapmapAllelesChimp
 # 1/11/08 angie: regenerated with finally-corrected (I hope) hapmapAllelesChimp
 # 1/24/08 angie: regenerated with finally-corrected (I hope!) hapmapAllelesChimp
 cd /cluster/data/hg17/bed/hapmap/rel21a
 /cluster/home/heather/kent/src/hg/snp/snpLoad/hapmapSummary hg17 hapmapSnpsCombined hapmapAllelesChimp hapmapAllelesMacaque
 hgLoadBed hg17 hapmapAllelesSummary -sqlTable=/cluster/home/heather/kent/src/hg/lib/hapmapAllelesSummary.sql hapmapSummary.tab -tab
 
 # sanity check
 mysql> select count(*) from hapmapAllelesSummary where majorAlleleCountCEU > totalAlleleCountCEU;
 +----------+
 | count(*) |
 +----------+
 |        0 |
 +----------+
 
 mysql> select count(*) from hapmapAllelesSummary where majorAlleleCountCHB > totalAlleleCountCHB;
 +----------+
 | count(*) |
 +----------+
 |        0 |
 +----------+
 
 mysql> select max(score) from hapmapAllelesSummary;
 +------------+
 | max(score) |
 +------------+
 |        500 |
 +------------+
 
 mysql> select count(*), popCount from hapmapAllelesSummary group by popCount;
 +----------+----------+
 | count(*) | popCount |
 +----------+----------+
 |    52479 |        1 |
 |    72977 |        2 |
 |   207643 |        3 |
 |  3700478 |        4 |
 +----------+----------+
 
 mysql> select count(*), isMixed from hapmapAllelesSummary group by isMixed; 
 
 +----------+---------+
 | count(*) | isMixed |
 +----------+---------+
 |  3192896 | NO      |
 |   840681 | YES     |
 +----------+---------+
 
 
 # histogram of heterozygosity:
 
   0 ************************************************************ 883400
  25 ************** 204000
  50 ************* 188703
  75 *********** 157404
 100 ********** 143119
 125 ********* 131575
 150 ********* 126916
 175 ********* 128585
 200 ******** 123440
 225 ******** 119815
 250 ******** 120646
 275 ******** 120239
 300 ******** 122654
 325 ********* 128233
 350 ********* 130069
 375 ********** 144699
 400 ********** 152829
 425 ************ 172513
 450 *************** 225645
 475 ********************************** 503166
 500  5927
 
 ############################################################################################
 
 ### HapMap LD (Daryl; February 11, 2006)
     ## start from the genotypes files, run Haploview, reformat, and load
     mkdir -p /san/sanvol1/hg17/bed/hapmap/genotypes/2006-01/non-redundant/para
     cd /san/sanvol1/hg17/bed/hapmap/genotypes/2006-01/non-redundant
     # wget all genotype data:
     # ftp://www.hapmap.org/genotypes/2006-01/non-redundant/genotypes_chr*_*.b35.txt.gz
 
     # Haploview had to be recompiled because there was a missing JPT sample in the ped file
     ##runHaploview.csh
     #!/bin/csh
     if ( $#argv < 2 ) then
 	echo "usage: $0 <absolutePath> <genotypeFileName.gz> [<javaMaxMem>]"
 	echo "       $0 /cluster/bin/foo bar.gz 2G"
 	exit 1
     endif
     set path = $1
     set file = $2
     set root = $file:r
     set memFlag = ""
     if ( $#argv >= 3  ) then
 	set memFlag = "-Xmx$3"
     endif
     cd /scratch
     /bin/cp     -f $path/$file .
     /bin/gunzip -f $file
     /usr/java/jre1.5.0_06/bin/java -d64 $memFlag -jar /cluster/home/daryl/haploview/haploview/Haploview.jar -c -d -n -maxDistance 250 -a $root >&! $root.log
     /bin/gzip   -f $root.LD    $root.CHECK >>& $root.log
     /bin/mv     -f $root.LD.gz $root.CHECK.gz  $root.log $path/
     /bin/rm     -f $root*
     ###
 
     cd para
     set hv = /cluster/home/daryl/scripts/runHaploview.csh
     set ldDir = /cluster/store5/gs.18/build35/bed/hapmap/genotypes/2006-01/non-redundant
     
     foreach pop (YRI CEU CHB JPT JPT+CHB)
 	foreach chrom (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X)
 	    echo $hv $ldDir genotypes_chr${chrom}_{$pop}.b35.txt.gz 4G >> jobList
 	end
     end
     ssh pk
     # para create, para try, para push -maxNode=25 ...
     #Completed: 120 of 120 jobs
     #CPU time in finished jobs:    1564915s   26081.91m   434.70h   18.11d  0.050 y
     #IO & Wait Time:                 21862s     364.37m     6.07h    0.25d  0.001 y
     #Average job time:               13223s     220.39m     3.67h    0.15d
     #Longest running job:                0s       0.00m     0.00h    0.00d
     #Longest finished job:           40742s     679.03m    11.32h    0.47d
     #Submission to last job:        104809s    1746.82m    29.11h    1.21d
 
     #### makeDcc.pl 
     #!/usr/bin/perl -W
     $pop   = shift || die "usage: makeDcc.pl <pop> <chr>\n";
     $chrom = shift || die "usage: makeDcc.pl <pop> <chr>\n";
     $geno  = "geno/genotypes_${chrom}_${pop}.b35.txt.gz";
     $ld    = "ld/genotypes_${chrom}_${pop}.b35.txt.LD.gz";
     $txt   = "dcc/ld_${chrom}_${pop}.b35.txt";
     open(GENO,"zcat $geno | " ) || die "can't open $geno";
     open(LD,  "zcat $ld   | " ) || die "can't open $ld";
     open(TXT, " > $txt      " ) || die "can't open $txt";
     <GENO>;#ignore header
     while (<GENO>) { @fields = split / /; $pos{$fields[0]} = $fields[3]; }
     close(GENO);
     <LD>;#ignore header;
     while (<LD>) { @fields     = split /\t/; $chromStart = $pos{$fields[0]}; $chromEnd   = $pos{$fields[1]};
 	print TXT "$chromStart $chromEnd $pop $fields[0] $fields[1] $fields[2] $fields[4] $fields[3]\n"; }
     close(LD);
     close(TXT);
     system("gzip $txt");
     ####
 
     #### makeDcc.csh
     #!/bin/csh
     #set path = "/cluster/home/daryl/scripts";
     set path = ".";
     foreach pop (CEU CHB JPT YRI JPT+CHB)
 	foreach chr (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y)
 	    echo $path/makeDcc.pl $pop chr$chr
 	end
     end
     ####
 
     #### makeLdBed.pl
     #!/usr/bin/perl -W
     sub min ($$)
     {
 	my $a = shift @_;
 	my $b = shift @_;
 	if ($a<$b) {return $a;}
 	return $b;
     }
     sub encodeDprime($)
     {
 	my $val = shift @_;
 	if    ( ($val > 1) || ($val < -1) ) { die "Dprime value ($val) is out of range [-1,1]";}
 	elsif ($val>=0) { $ret = ord('a') + $val*9;}
 	else         { $ret = ord('A') - $val*9;}
 	return chr($ret);
     }
     sub encodeRsquared($)
     {
 	my $val = shift @_;
 	if ( ($val > 1) || ($val < 0) ) { die "R^2 value ($val) is out of range [0,1]";}
 	return encodeDprime($val);
     }
     sub encodeLod($$)
     {
 	my $lod    = shift @_;
 	my $dPrime = shift @_;
 	$ret = ord('a');
 	if ($lod>=2) # high LOD
 	{
 	if (abs($dPrime)<0.5) { $ret = ord('y'); } # high LOD, low D'  -> pink
 	else { $ret += min((int($lod-abs($dPrime)-1.5)), 9) ;}
 	}
 	elsif (abs($dPrime)>0.99) { $ret = ord('z'); } # high D', low LOD  -> blue
 	return chr($ret); 
     }
     $inDir  = shift||"data"; 
     $outDir = shift||"bed";
     $foo    = "";
     $bar    = ""; 
     @rest   = (); 
     @pops   = ("CEU", "CHB", "JPT", "YRI", "JPT+CHB");
     printf("> Starting        \t" . `date` . "\n");
     foreach $pop (@pops)
     {
 	opendir(DIR, $inDir) || die "can't open $inDir";
 	if ($pop eq "JPT+CHB") { @hmFiles = grep {/^ld_/ && /_JPT/ && /CHB.b35.txt.gz$/} readdir(DIR); }
 	else { @hmFiles = grep {/^ld_/ && /_${pop}.b35.txt.gz$/} readdir(DIR); }
 	closedir(DIR);
 	printf "POP:\t$pop\t$#hmFiles\n";
 	foreach $hmFile (sort @hmFiles)
 	{
 	    ($foo, $chrom, $bar) = split /_/, $hmFile;
 	    $chrom =~ s/chrx/chrX/;
 	    $chrom =~ s/chry/chrY/;
 	    $outfile = "$outDir/${pop}_${chrom}.hg17.bed";
 	    if ((-e $outfile)||(-e "$outfile.gz")) { next; }
 	    $tmpFile = "/tmp/${pop}_${chrom}.hg17.bed";
 	    printf("$inDir/$hmFile => $outfile.gz\t" . `date`);
 	    open(OUT, "> $tmpFile" ) || die "can't open $tmpFile";
 	    open(IN, "zcat $inDir/$hmFile | " ) || die "can't open $inDir/$hmFile";
 	    $line = <IN>;
 	    if (!defined $line){next;}
 	    chomp($line);
 	    ($chromStart, $chromEnd, $pop, $name, $marker2, $dprime, $rsquared, $lod, @rest) = split / /, $line;
 	    $ldCount = 1;
 	    while (<IN>)
 	    {
 	        chomp();
 		($chromStartNew, $chromEndNew, $pop, $nameNew, $marker2, $dprime, $rsquared, $lod, @rest) = split / /;
 		if ($chromStart ne $chromStartNew)
 		{
 		    $chromStart--;
 		    printf(OUT "$chrom\t$chromStart\t$chromEnd\t$name\t$ldCount\t$dprimeList\t$rsquaredList\t$lodList\n");
 		    $chromStart   = $chromStartNew;
 		    $chromEnd     = $chromEndNew;
 		    $name         = $nameNew;
 		    $ldCount      = 1;
 		    $dprimeList   = encodeDprime($dprime);
 		    $rsquaredList = encodeRsquared($rsquared);
 		    $lodList      = encodeLod($lod, $dprime);
 		}
 		elsif ($chromEndNew-$chromStartNew<250000)
 		{
 		    $chromEnd     = $chromEndNew;
 		    $ldCount++;
 		    $dprimeList   .= encodeDprime($dprime);
 		    $rsquaredList .= encodeRsquared($rsquared);
 		    $lodList      .= encodeLod($lod, $dprime);
 		}
 	    }
 	    close(IN);
 	    $chromStart--;
 	    printf(OUT "$chrom\t$chromStart\t$chromEnd\t$name\t$ldCount\t$dprimeList\t$rsquaredList\t$lodList\n");
 	    close(OUT);
 	    system("gzip $tmpFile");
 	    system("mv $tmpFile.gz $outDir");
 	}
     }
     printf("> Finished        \t" . `date` . "\n");
     ####
 
     #### getMax.csh -- check for consistency by chrom and population
     #!/bin/csh
     set out = maxDist.txt
     rm -f $out
     touch $out
     echo this takes about 4 hours to run completely >> $out
     foreach f (dcc/ld_*.b35.txt.gz)
 	echo -n "$f    " >> $out
 	zcat $f | awk '{if ($2-$1>max) max=$2-$1} END {print max}' >> $out
     end
 
     #### getSizes.csh -- should all be 249999
     #!/bin/csh
     set out = wcList.txt
     rm -f $out
     touch $out
     echo "this takes about 2 hours to run completely"
     foreach f (dcc/*.txt.gz) 
 	echo -n $f:r:r "    " | sed 's/ld_//;s/chr//;s/_/\t/' >> $out
 	zcat $f | cut -f1 -d " " | uniq | wc -l >> $out
     end
 
     #### load.csh
     #!/bin/csh
     set db = hg17
     sed 's/hapmapLd/hapmapLdCeu/'    ${HOME}/kent/src/hg/lib/hapmapLd.sql | hgsql ${db}
     sed 's/hapmapLd/hapmapLdChb/'    ${HOME}/kent/src/hg/lib/hapmapLd.sql | hgsql ${db}
     sed 's/hapmapLd/hapmapLdJpt/'    ${HOME}/kent/src/hg/lib/hapmapLd.sql | hgsql ${db}
     sed 's/hapmapLd/hapmapLdYri/'    ${HOME}/kent/src/hg/lib/hapmapLd.sql | hgsql ${db}
     sed 's/hapmapLd/hapmapLdChbJpt/' ${HOME}/kent/src/hg/lib/hapmapLd.sql | hgsql ${db}
     # about half an hour to an hour per population
     foreach c (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X)
 	hgLoadBed -noSort -oldTable -strict ${db} hapmapLdCeu CEU_chr${c}.${db}.bed.gz
 	hgLoadBed -noSort -oldTable -strict ${db} hapmapLdChb CHB_chr${c}.${db}.bed.gz
 	hgLoadBed -noSort -oldTable -strict ${db} hapmapLdJpt JPT_chr${c}.${db}.bed.gz
 	hgLoadBed -noSort -oldTable -strict ${db} hapmapLdYri YRI_chr${c}.${db}.bed.gz
 	hgLoadBed -noSort -oldTable -strict ${db} hapmapLdChbJpt JPT+CHB_chr${c}.${db}.bed.gz
     end
     rm -f bed.tab
     ###
 
 
 # AFFYHUEX1 TRACK (sugnet Wed Oct  5 12:16:42 PDT 2005)
 
 mkdir hg17
 cd hg17
 pwd
 # /cluster/store1/sugnet/affymetrixHumanAllExon/hg17
 mkdir gff beds annot
 cd gff
 # download gff design files
 
 # parse gff script...
 #!/usr/bin/perl -w
 if(scalar(@ARGV) == 0) {
     print STDERR "parseGff.pl - Parse out affymetrixes gff annotation 
 probesets for human all exon design.
 usage:
    parseGff.pl file1.design.gff file2.design.gff ... fileN.design.gff
 ";
     exit(1);
 }
 
 sub splitField($) {
     my $l = shift(@_);
     my @w = split / /, $l;
     return $w[1];
 }
 
 while($file = shift(@ARGV)) {
     if(!($file =~ /(.+)\.gff/)) {
 	die "$file doesn't have .gff suffix\n";
     }
     $prefix = $1;
     print STDERR "Doing file $file.\n";
     open(IN, $file) or die "Can't open $file to read.";
     open(BED, ">../beds/$prefix.pset.bed") or die "Can't open ../beds/$prefix.pset.bed to write.";
     open(ANNOT, ">../annot/$prefix.tab") or die "Can't open ../annot/$prefix.tab to write.";
     while($line = <IN>) {
 	# Only want the probeset records.
 	if($line =~ /\tprobeset\t/) {
 	    $score = 0;
 	    $cds = 0;
 	    $bounded = 0;
 	    chomp($line);
 	    # pop off an microsoft line endings.
 	    $line =~ s/\r$//;
 	    @words = split /\t/, $line;
 	    # This makes the evidence be comman separated.
 	    $words[8] =~ s/\" \"/,/g;
 	    # This gets rid of pesky quotes.
 	    $words[8] =~ s/\"//g;
 
 	    # Set the score based on the annotation type
 	    if($words[8] =~ /full/) {
 		$score = 200;
 	    }
 	    elsif($words[8] =~ /extended/) {
 		$score = 500;
 	    }
 	    elsif($words[8] =~ /core/) {
 		$score = 900;
 	    }
 	    if($words[8] =~ /bounded/) {
 		$score -= 200;
 	    }
 	    if($words[8] =~ /cds/) {
 		$score += 100;
 	    }
 	    if($score <= 0) {
 		$score = 100;
 	    }
 		
 	    # Print out the annotation fields.
 	    @fields = split /; /,$words[8];
 	    $id = splitField($fields[1]);
 	    $f = shift(@fields);
 	    $f = splitField($f);
 	    print ANNOT "$f";
 	    while($f = shift(@fields)) {
 		if($f =~ /^bounded/) {
 		    $bounded = 1;
 		}
 		if($f =~ /^cds/) {
 		    $cds = 1;
 		}
 		if(!($f =~ /^bounded/ || $f =~ /^cds/)) {
 		    $f = splitField($f);
 		    print ANNOT "\t$f";
 		}
 	    }
 	    print ANNOT "\t$bounded\t$cds";
 	    print ANNOT "\n";
 	    print BED "$words[0]\t$words[3]\t$words[4]\t$id\t$score\t$words[6]\n";
 	}
     }
     close(IN);
     close(BED);
     close(ANNOT);
 }
 
 ./parseGff.pl *.gff
 cat beds/*.bed > affyHuEx1.bed
 hgLoadBed hg17 affyHuEx1 affyHuEx1.bed -strict
 cat annot/*.tab > affyHuEx1.annot.tab 
 
 # Contents of affyHuEx1Annot.sql file
 CREATE TABLE affyHuEx1Annot (
   numIndependentProbes smallint not null,
   probesetId int(11) not null,
   exonClustId int(11) not null,
   numNonOverlapProbes smallint not null,
   probeCount smallint not null,
   transcriptClustId int(11) not null,
   probesetType smallint not null,
   numXHybeProbe smallint not null,
   psrId int(11) not null,
   level varchar(10) not null,
   evidence varchar(255) not null,
   bounded smallint not null,
   cds smallint not null,
   PRIMARY KEY (probesetId)
 );
 hg17S -A < affyHuEx1Annot.sql 
 echo "load data local infile 'affyHuEx1.annot.tab' into table affyHuEx1Annot;" | hg17S -A
 
 # end AFFYHUEX1 track
 
 ##########################################################################
 # AFFY HUEX1 OFF-BY-ONE FIX (Andy 2006-12-14)
    ssh hgwdev
    cd /cluster/data/hg17/bed/affyHumanExon
    echo "select * from affyHuEx1" | hgsql hg17 | \
       tail +2 | awk 'BEGIN{OFS="\t"}{print $2,$3-1,$4,$5,$6,$7}' \
     > affyHuEx1.fixed.bed
    hgLoadBed hg17 affyHuEx1 affyHuEx1.fixed.bed
 
 ##########################################################################
 # NSCAN composite track - (2005-09-29 markd)  loaded proteins 2005-10-13
     cd /cluster/data/hg17/bed/nscan/
     # obtained NSCAN and NSCAN-EST predictions from michael brent's group
     # at WUSTL
     wget http://genome.cse.wustl.edu/predictions/human/hg17_nscan_mm5_9_14_2005/hg17_nscan_mm5_9_14_2005.tar.gz
     tar -zxf hg17_nscan_mm5_9_14_2005.tar.gz 
 
     wget http://genome.cse.wustl.edu/predictions/human/NCBI35_NSCAN_EST_4-16-2005.tar  
     gzip -9 NCBI35_NSCAN_EST_4-16-2005.tar
 
     # change protein fasta file to have transcript id in header
     foreach f (chr_ptx/*.ptx)
         awk '/^>/{$0=$1".a"}{print $0}' $f >$f.fix
     end
     foreach f (NCBI35_NSCAN_EST_4-16-2005/chr_ptx/*.ptx)
         awk '/^>/{$0=$1".a"}{print $0}' $f >$f.fix
     end
 
     # load tracks.  Note that these have *utr features, rather than
     # exon featres.  currently ldHgGene creates separate genePred exons
     # for these.
     ldHgGene -gtf -genePredExt hg17 nscanGene  chr_gtf/chr*.gtf
     hgPepPred hg17 generic nscanPep chr_ptx/chr*.fix
     rm -rf chr_* *.tab
 
     ldHgGene -gtf -genePredExt hg17 nscanEstGene NCBI35_NSCAN_EST_4-16-2005/chr_gtf/chr*.gtf
     hgPepPred hg17 generic nscanEstPep NCBI35_NSCAN_EST_4-16-2005/chr_ptx/chr*.fix
     rm -rf NCBI35_NSCAN_EST_4-16-2005 *.tab
 
     # update trackDb; need a hg17-specific page to describe informants
     human/hg17/nscan.html    
     human/hg17/trackDb.ra
 
 ##########################################################################
 # NHGRI DNASE I HYPERSENSITIVE SITES (2005-10-05 kate)
 #  Submitted by Greg Crawford via web site,
 #  http://research.nhgri.nih.gov/DNaseHS/May2005/
 #  In addition, a file containing the 'randoms' was FTP'ed by Greg
 #  Submitted for hg16 -- lifted to hg17.
 #  Details of hg16 data prep are in makeHg16.doc
 
     mkdir /cluster/data/hg17/bed/nhgri
     cd /cluster/data/hg17/bed/nhgri
     cp /cluster/data/hg16/bed/nhgri/hs.bed hs.hg16.bed
     liftOver hs.hg16.bed /gbdb/hg16/liftOver/hg16ToHg17.over.chain \
                 hs.hg17.bed hs.unmapped
     grep '^chr' hs.unmapped | wc -l
         # 8 unmapped
     hgLoadBed hg17 nhgriDnaseHs hs.hg17.bed
         # Loaded 14216 elements of size 5
     checkTableCoords hg17 nhgriDnaseHs
 
 # UPDATE WGRNA TRACK (DONE, 2005-10-20, Fan)
 
   ssh hgwdev
   cd /cluster/data/hg17/bed
 
   mkdir wgRna-2005-10-20
   cd wgRna-2005-10-20
 
 # Received the data file, wgtrack_no_bin_oct2005.txt, from Michel Weber's email 
 # (Michel.Weber@ibcg.biotoul.fr)
 # and place it under cd /cluster/data/hg17/bed/wgRna-2005-10-20.
 
   cp wgtrack_no_bin_oct2005.txt wgRna.tab
   vi wgRna.tab
   
 # edit wgRna.tab to take out the first line of data field labels.
 
     hgLoadBed -sqlTable=/cluster/home/fanhsu/src/hg/lib/wgRna.sql hg17 wgRna wgRna.tab
 
 # Compared to 8/24/05 data, a few records were changed.
 
 ##########################################################################
 # REBUILD hg17.gnfAtlas2Distance TABLE.  SOMEHOW IT HAD MUCH FEWER RECORDS. (DONE 10/27/05, Fan)
 
 # Create table to map between known genes and GNF Atlas2
 # expression data.
     hgMapToGene hg17 gnfAtlas2 knownGene knownToGnfAtlas2 '-type=bed 12'
 
 # Create expression distance table - takes about an hour
     hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
         hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
         -lookup=knownToGnfAtlas2 &
 
 #       hgsql -e "select count(*) from gnfAtlas2Distance;" hg17
 #       row count changed to 32458000
 			
 ##########################################################################
 # BUILD ALLEN BRAIN TRACK (DONE 10/29/05 JK)
 
 # Make the working directory
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mkdir allenBrain
     cd allenBrain
 
 # Remap the probe alignments from mm7 to hg17
     zcat /cluster/data/mm7/bed/bedOver/mm7.hg17.over.chain.gz \
         |  pslMap -chainMapFile -swapMap \
 	       /cluster/data/mm7/bed/allenBrain/allenBrainAli.psl stdin stdout
 	|  sort -k 14,14 -k 16,16n > unscored.psl
     pslRecalcMatch unscored.psl /cluster/data/hg17/nib \
         /cluster/data/mm7/bed/allenBrain/allProbes.fa allenBrainAli.psl
     
 
 # Load the database
    hgsql hg17 < ~/kent/src/hg/lib/allenBrainUrl.sql
    hgsql hg17 -e 'load data local infile "/cluster/data/mm7/bed/allenBrain/allenBrainUrl.tab" into table allenBrainUrl;'
    hgLoadPsl hg17 allenBrainAli.psl
    mkdir /gbdb/hg17/allenBrain
    ln -s /cluster/data/mm7/bed/allenBrain/allProbes.fa /gbdb/hg17/allenBrain/allProbes.fa
    hgLoadSeq hg17 /gbdb/hg17/allenBrain/allProbes.fa
 
 # Make mapping between known genes and allenBrain	
    hgMapToGene hg17 allenBrainAli -type=psl knownGene knownToAllenBrain 
 
 ##########################################################################
 # BUILD NIBB IMAGE PROGES (DONE 11/07/05 JK)
 
 # Make directory on san for cluster job and copy in sequence
     ssh pk
     mkdir /san/sanvol1/scratch/hg17/nibbPics
     cd /san/sanvol1/scratch/hg17/nibbPics
     cp /cluster/data/xenTro1/bed/nibbPics/nibbImageProbes.fa .
 
 # Make parasol job dir and sequence list files
     mkdir run
     cd run
     mkdir psl
     ls -1 /cluster/sanvol1/scratch/hg17/nib/*.nib > genome.lst
     echo ../nibbImageProbes.fa > rna.lst
 
 # Create parasol gensub file file
 cat << '_EOF_' > gsub
 #LOOP
 blatz -rna -minScore=6000 -out=psl $(path1) $(path2) psl/$(root1)_$(root2).psl
 #ENDLOOP
 '_EOF_'
 
 # Create parasol batch
     gensub2 genome.lst mrna.lst gsub spec
     para create spec
 
 # Do para try/push/time etc.
 #Completed: 46 of 46 jobs
 #CPU time in finished jobs:      11818s     196.97m     3.28h    0.14d  0.000 y
 #IO & Wait Time:                   145s       2.41m     0.04h    0.00d  0.000 y
 #Average job time:                 260s       4.33m     0.07h    0.00d
 #Longest running job:                0s       0.00m     0.00h    0.00d
 #Longest finished job:            1022s      17.03m     0.28h    0.01d
 #Submission to last job:          1060s      17.67m     0.29h    0.01d
 
 # Make sort and filter
     catDir psl | sort -k 10 \
         | pslReps stdin stdout /dev/null -nohead -minAli=0.60 -nearTop=0.001 -minCover=0.10 -minNearTopSize=80 \
 	| sort -k 14,14 -k 16,16n \
 	| sed 's/..\/..\/nib\/chr/chr/' \
 	| sed 's/.nib//' > ../nibbImageProbes.psl
 
 # Make bed file and copy in stuff
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mkdir nibbPics
     cd nibbPics
     cp /cluster/data/xenTro1/bed/nibbPics/nibbImageProbes.fa .
     cp /san/sanvol1/scratch/hg17/nibbPics/nibbImageProbes.psl .
 
 # Load into database
     ln -s /cluster/data/xenTro1/bed/nibbPics/nibbImageProbes.fa /gbdb/hg17/nibbImageProbes.fa
     hgLoadSeq hg17 /gbdb/hg17/nibbImageProbes.fa
     hgLoadPsl hg17 nibbImageProbes.psl
 
 ###########################################################################
 # EXONIPHY WITH DOG (acs, 11/22/05) -- MM7, RN3, CANFAM2, HG17
 
     # first build 4-way multiz alignment from syntenic nets (helps reduce
     # false positive predictions due to paralogous alignments)
 
     # (prepare mafNet files from syntenic nets and copy to
     # /cluster/bluearc/hg17/mafNetSyn; do this for mm7, rn3, canFam2,
     # and galGal2)
 
     # make output dir and run dir
     ssh pk
     cd /cluster/data/hg17/bed/multiz.hg17Mm7Rn3CanFam2
 
     mkdir -p mafSyn runSyn
     cd runSyn
 
     # create scripts to run multiz on cluster
     cat > oneMultiz.csh << 'EOF'
 #!/bin/csh -fe
     set c = $1
     set multi = /scratch/$user/multiz.hg17Mm7Rn3CanFam2.$c
     set pairs = /cluster/bluearc/hg17/mafNetSyn
 
     # special mode --
     # with 1 arg, cleanup
     if ($#argv == 1) then
         rm -fr $multi
         exit
     endif
 
     #  special mode --
     # with 3 args, saves an alignment file
     if ($#argv == 3) then
         cp $multi/$2/$c.maf $3
         exit
     endif 
         
     set s1 = $2
     set s2 = $3
     set flag = $4
 
     # locate input files -- in pairwise dir, or multiple dir
     set d1 = $multi
     set d2 = $multi
     if (-d $pairs/$s1) then
         set d1 = $pairs
     endif
     if (-d $pairs/$s2) then
         set d2 = $pairs
     endif
     set f1 = $d1/$s1/$c.maf
     set f2 = $d2/$s2/$c.maf
     # write to output dir
     set out = $multi/${s1}${s2}
     mkdir -p $out
 
     # check for empty input file
     if (-s $f1 && -s $f2) then
         echo "Aligning $f1 $f2 $flag"
         /cluster/bin/penn/multiz.v10.5 $f1 $f2 $flag > $out/$c.tmp.maf
         echo "Ordering $c.maf"
         /cluster/bin/penn/maf_project $out/$c.tmp.maf hg17.$c > $out/$c.maf
     else if (-s $f1) then
         cp $f1 $out
     else if (-s $f2) then
         cp $f2 $out
     endif
 'EOF'
 # << for emacs
     chmod +x oneMultiz.csh
 
     cat > allMultiz.csh << 'EOF'
 #!/bin/csh -fe
 set c = $1
 oneMultiz.csh $c mm7 rn3 0
 oneMultiz.csh $c mm7rn3 canFam2 1
 # get final alignment file
 oneMultiz.csh $c mm7rn3canFam2 /cluster/data/hg17/bed/multiz.hg17Mm7Rn3CanFam2/mafSyn/$c.maf
 #cleanup
 oneMultiz.csh $c
 'EOF'
 # << for emacs
     chmod +x allMultiz.csh
 
 cat > gsub << 'EOF'
 #LOOP
 allMultiz.csh $(root1) {check out line+ /cluster/data/hg17/bed/multiz.hg17Mm7Rn3CanFam2/mafSyn/$(root1).maf}
 #ENDLOOP
 'EOF'
 # << for emacs
     cut -f 1 /cluster/data/hg17/chrom.sizes > chrom.lst
     set path = (/parasol/bin $path);rehash
     gensub2 chrom.lst single gsub jobList
     para create jobList
         # 46 jobs
     para try; para check
     para push
 
     # build chromosome-by-chromosome SS files
 
     cd /cluster/data/hg17/bed/multiz.hg17Mm7Rn3CanFam2
     mkdir run-ss-syn
     cd run-ss-syn
     mkdir -p /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSyn
     cat > makeSS.csh << 'EOF'
 #!/bin/csh -fe
 set c = $1
 /cluster/bin/phast/msa_view -i MAF -o SS /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/mafSyn/$c.maf --refseq /cluster/bluearc/hg17/chrom/$c.fa | gzip -c >  /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSyn/$c.ss.gz
 'EOF'
 # << for emacs
     chmod +x makeSS.csh
 
     rm -f jobList
     foreach chr (`cut -f 1 /cluster/data/hg17/chrom.sizes`)
 	echo "makeSS.csh $chr" >> jobList
     end
 
     para create jobList
         # 46 jobs
     para try; para check
     para push
 
     # now train hmm, with indel model
     # note: commands below require bash
 
     # first get a clean set of genes for training (with --indel-strict)
     mkdir -p /cluster/data/hg17/bed/exoniphy/train
     cd /cluster/data/hg17/bed/exoniphy/train
     mkdir -p stats genes
     CHROMS="chr1 chr2 chr3 chr4 chr5 chr6 chr7 chr8 chr9 chr10 chr11 chr12 chr13 chr14 chr15 chr16 chr17 chr18 chr19 chr20 chr21 chr22"
     for chr in ${CHROMS} ; do 
 	echo $chr 
 	zcat /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSyn/$chr.ss.gz | clean_genes genes/refGene.$chr.gff - --stats stats/$chr.stats --conserved --indel-strict --groupby exon_id --offset3 4 --offset5 4 > genes/refGene.$chr.clean.gff 
     done
 
     # get conserved noncoding seqs and add to GFFs
     mkdir -p cns
     for chr in ${CHROMS} ; do 
 	    echo $chr
 	    featureBits -bed=cns/$chr.bed -chrom=$chr hg17 phastConsElementsPaper \!knownGene:exon:100 \!refGene:exon:100 \!mrna \!ensGene \!intronEst \!twinscan
 	cp genes/refGene.$chr.clean.gff genes/refGene.$chr.withCNS.gff 
 	awk '{printf "%s\tphastCons\tCNS\t%d\t%d\t.\t.\t.\texon_id \"CNS.%s\"\n", $1, $2+1, $3, $4}' cns/$chr.bed >> genes/refGene.$chr.withCNS.gff 
     done
 
     # now train HMM
     # note: actually have to unzip SS files before this step
     rm -f alns gffs
     for chr in ${CHROMS} ; do 
 	echo /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSyn/$chr.ss >> alns 
 	echo genes/refGene.$chr.withCNS.gff >> gffs 
     done
     hmm_train -m '*alns' -c ~/phast/data/exoniphy/default.cm -g '*gffs' -R exon_id -i SS -I CDS,background,CNS,5\'splice,3\'splice,prestart -t "((hg17,(mm7,rn3)),canFam2)" > indels.hmm
 
     # training complete; now run exoniphy genome-wide
 
     # first need to split up alignments
     
     mkdir -p /cluster/data/hg17/bed/exoniphy/test/run-split
     cd /cluster/data/hg17/bed/exoniphy/test/run-split
     
     cat > doSplit.csh << 'EOF'
 #!/bin/csh -fe
 set c = $1
 mkdir -p /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSynFrags/$c
 /cluster/bin/phast/msa_split /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/mafSyn/$c.maf --refseq /cluster/bluearc/hg17/chrom/$c.fa -i MAF --windows 100000,0 --between-blocks 5000 --min-informative 1000 --out-format SS --out-root /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSynFrags/$c/$c --tuple-size 3
 gzip /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSynFrags/$c/$c*.ss
 'EOF'
 # << for emacs
     chmod +x doSplit.csh
 
     rm -f jobList
     for file in /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/mafSyn/chr*.maf ; do echo doSplit.csh `basename $file .maf` >> jobList ; done
 
     para create jobList
         # 43 jobs
     para try; para check
     para push
 
     # now set up exoniphy run
 
     mkdir -p /cluster/data/hg17/bed/exoniphy/test/run-exoniphy
     cd /cluster/data/hg17/bed/exoniphy/test/run-exoniphy
     cp -p ../../train/indels.hmm /cluster/bluearc/hg17/exoniphy/training
     mkdir -p /cluster/bluearc/hg17/exoniphy/GFF
     
     cat > doExoniphy.sh << 'EOF'
 #!/usr/local/bin/bash
 
 root=`basename $1 .ss.gz`
 chrom=`echo $root | awk -F\. '{print $1}'`
 no=`echo $root | awk 'BEGIN{FS="[-.]"} {printf "%d\n", ($2+10000)/100000}'`
 
 if [ ! -d /cluster/bluearc/hg17/exoniphy/GFF/$chrom ] ; then 
     mkdir -p /cluster/bluearc/hg17/exoniphy/GFF/$chrom
 fi
 
 zcat $1 | /cluster/bin/phast/exoniphy - --hmm  /cluster/bluearc/hg17/exoniphy/training/indels.hmm --reflect-strand --extrapolate default --score --indels --alias "hg17=human; mm7=mouse; rn3=rat; canFam2=dog" --seqname $chrom --idpref $chrom.$no  > /cluster/bluearc/hg17/exoniphy/GFF/$chrom/$root.gff
 'EOF'
 # << for emacs
     chmod +x doExoniphy.sh
 
     rm -f jobList
     for dir in /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSynFrags/* ; do find $dir -name '*.ss.gz' | awk '{printf "doExoniphy.sh %s\n", $1}' >> jobList ; done
 
     para create jobList
         # 27070 jobs
     para try; para check
     para push
 
 #Completed: 27059 of 27070 jobs
 #Crashed: 11 jobs
 #CPU time in finished jobs:    8573545s  142892.41m  2381.54h   99.23d  0.272 y
 #IO & Wait Time:                 73412s    1223.54m    20.39h    0.85d  0.002 y
 #Average job time:                 320s       5.33m     0.09h    0.00d
 #Longest running job:                0s       0.00m     0.00h    0.00d
 #Longest finished job:             593s       9.88m     0.16h    0.01d
 #Submission to last job:         22823s     380.38m     6.34h    0.26d    
 
     # crashed jobs all on random chroms, chrM, etc., and appear to be
     # due to all species not being present; okay to ignore
 
     # collect predictions and create track
     rm -f exoniphy.gff
     for dir in /cluster/bluearc/hg17/exoniphy/GFF/chr* ; do \
 	rm -f files.* tmp.gff ;\
 	find $dir -name "chr*.gff" > files ;\
 	split -l 1000 files files. ;\
 	for l in files.* ; do cat `cat $l` >> tmp.gff ; done ;\
 	refeature --sort tmp.gff >> exoniphy.gff ;\
     done
     ldHgGene -genePredExt -gtf hg17 exoniphyDog exoniphy.gff
 
 
 # COW SYNTENY (Done, Heather, Dec. 2005)
 # Data from Harris A. Lewin <h-lewin@uiuc.edu>
 
 ssh hgwdev
 cd /cluster/data/hg17/bed
 mkdir syntenyCow
 cd syntenyCow
 
 hgLoadBed -noBin hg17 syntenyCow syntenyCow.bed
 
 # add to kent/src/hg/makeDb/trackDb/human/hg17/trackDb.ra
  
 ###########################################################################
 # New Conservation track (WORKING 2005-12-15 kate)
 
 # Pairwise alignments needed for: monDom2, danRer3, bosTau2
 # Use existing alignments for:  
 # macaque_rheMac1
 # rat_rn3 
 # mouse_mm7
 # dog_canFam2
 # chicken_galGal2
 # xenopus_xenTro1
 # fugu_fr1
 # rabbit_oryCun1
 # armadillo_dasNov1
 # elephant_loxAfr1
 # tenrec_echTel1
 # tetraodon_tetNig1
 
 #########################################################################
 # BLASTZ danRer3 (DONE - 2005-12-20 kate)
 # Includes both randoms
     ssh pk
     mkdir /cluster/data/hg17/bed/blastz.danRer3.2005-12-20
     cd /cluster/data/hg17/bed
     ln -s blastz.danRer3.2005-12-20 blastz.danRer3
     cd blastz.danRer3
 
     cat << 'EOF' > DEF
 # human target, zebrafish query
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/x86_64:/cluster/home/angie/schwartzbin:/parasol/bin
 
 BLASTZ=blastz.v7.x86_64
 BLASTZ_ABRIDGE_REPEATS=1
 
 # use parameters suggested for human-fish evolutionary distance
 # recommended in doBlastzChainNet.pl help
 # (previously used for  hg16-fr1, danrer1-mm5)
 BLASTZ_H=2000
 BLASTZ_Y=3400
 BLASTZ_L=6000
 BLASTZ_K=2200
 BLASTZ_Q=/san/sanvol1/scratch/blastz/HoxD55.q
 
 # TARGET: Human hg17
 SEQ1_DIR=/san/sanvol1/scratch/hg17/nib
 SEQ1_SMSK=/cluster/bluearc/hg17/linSpecRep.notInZebrafish
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 SEQ1_LEN=/san/sanvol1/scratch/hg17/chrom.sizes
 
 # QUERY: zebrafish danRer3
 # Use all chroms, including both randoms (chrUn and chrNA)
 SEQ2_DIR=/san/sanvol1/scratch/danRer3/nib
 SEQ2_SMSK=/san/sanvol1/scratch/danRer3/linSpecRep.notInOthers
 SEQ2_LEN=/cluster/bluearc/danRer3/chrom.sizes
 SEQ2_CHUNK=30000000
 SEQ2_LAP=1000
 
 BASE=/cluster/data/hg17/bed/blastz.danRer3.2005-12-20
 TMPDIR=/scratch/tmp
 'EOF'
     # << happy emacs
 
     /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk -chainMinScore=5000 -chainLinearGap=loose  \
         -stop=net \
 	`pwd`/DEF >& blastz.out &
 
     # mistakenly started this in blastz.danRer3.2005-12-18 dir --
     # need to move DEF file and blastz.out to 2005-12-20 dir.
 
     # bogus stop at net step -- thinks it can't find chains
     # I'm just restarting there
     /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk -chainMinScore=5000 -chainLinearGap=loose  \
         -continue=net  \
 	`pwd`/DEF >& blastz.2.out &
 
     # stopped because vsDanRer3 downloads already there from
     # previous run. 
     ssh hgwdev "rm -fr /usr/local/apache/htdocs/goldenPath/hg17/vsDanRer3"
     /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk -chainMinScore=5000 -chainLinearGap=loose  \
         -continue=download  \
 	`pwd`/DEF >& blastz.3.out &
 
     # measurements
     ssh hgwdev "featureBits hg17 chainDanRer2Link" >& danRer2.fb; cat danRer2.fb
         # 70696998 bases of 2866216770 (2.467%) in intersection
     ssh hgwdev "featureBits hg17 chainDanRer3Link" >& danRer3.fb; cat danRer3.fb
         # 55625762 bases of 2866216770 (1.941%) in intersection
 
     # not sure why there's lower coverage from the newer assembly.
     # It's possibly due to different parameters used in the other
     # alignment.  Rachel is experimenting with hg18/danRer3, and
     # if warranted, we might replace this later
 
 #########################################################################
 # BLASTZ bosTau2 (DONE - 2005-12-19 kate)
     ssh pk
     mkdir /cluster/data/hg17/bed/blastz.bosTau2.2005-12-19
     cd /cluster/data/hg17/bed
     rm blastz.bosTau2
     ln -s blastz.bosTau2.2005-12-19 blastz.bosTau2
     cd blastz.bosTau2
 
     cat << 'EOF' > DEF
 # human vs. cow
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn/x86_64:/cluster/bin/x86_64:/parasol/bin
 
 BLASTZ=blastz.x86_64
 
 # using parameter used when not using lineage specific repeat
 # abridging.  This parameter restricts the # matches used by
 # dynamic masking.  (We can't currently use LSR repeat abridging
 # when either assembly sequence is in .2bit).
 
 BLASTZ_M=50
 
 # TARGET: Human (hg17)
 SEQ1_DIR=/san/sanvol1/scratch/hg17/nib
 SEQ1_LEN=/san/sanvol1/scratch/hg17/chrom.sizes
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY: Cow (bosTau2)
 # chunk it as we can't do whole-genome on 2bits
 SEQ2_DIR=/scratch/hg/bosTau2/bosTau2.noBin0.2bit
 SEQ2_LEN=/scratch/hg/bosTau2/noBin0.sizes
 SEQ2_CHUNK=300000000
 SEQ2_LAP=10000
 
 BASE=/cluster/data/hg17/bed/blastz.bosTau2.2005-12-19
 TMPDIR=/scratch/tmp
 'EOF'
     # << happy emacs
     # use chain parameters for "close" species
     /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk -chainMinScore=3000 -chainLinearGap=medium \
 	`pwd`/DEF >& blastz.out &
 
     ssh hgwdev "featureBits hg17 chainBosTau1Link" >& bosTau1.fb; cat bosTau1.fb
     ssh hgwdev "featureBits hg17 chainBosTau2Link" >& bosTau2.fb; cat bosTau2.fb
 
     #	swapping to get the lift over file in the other direction (Hiram)
     ssh pk
     mkdir /cluster/data/bosTau2/bed/blastz.hg17.swap
     cd /cluster/data/bosTau2/bed
     ln -s blastz.hg17.swap blastz.hg17
     cd blastz.hg17.swap
     
     time /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-swap -bigClusterHub=pk -chainMinScore=3000 -chainLinearGap=medium \
 	/cluster/data/hg17/bed/blastz.bosTau2.2005-12-19/DEF > swap.out 2>&1 &
     #	this failed during the load of the tables, but that is OK, we
     #	just wanted the liftOver files from this
 
     #	manually cleaned this up since the run faild during the MySQL
     #	load due to out of space problems.  These tables do not need to
     #	be loaded anyway.
     sh kkstore02
     cd /cluster/data/bosTau2/bed/blastz.hg17.swap
 
     rm -fr psl/
 
     rm -fr axtChain/run/chain/
     rm -f  axtChain/noClass.net
     rm -fr axtChain/net/
     rm -fr axtChain/chain/
 
 #########################################################################
 # BLASTZ rheMac2 (2006-02-08 kate)
     ssh pk
     mkdir /cluster/data/hg17/bed/blastz.rheMac2.2006-02-08
     cd /cluster/data/hg17/bed
     ln -s blastz.rheMac2.2006-02-08 blastz.rheMac2
     cd blastz.rheMac2
 
     cat << 'EOF' > DEF
 # macaca mulatta vs. hg18
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn/x86_64:/cluster/bin/x86_64:/parasol/bin
 
 ALIGN=blastz-run
 BLASTZ=blastz.v7.x86_64
 
 # TARGET - hg17
 SEQ1_DIR=/san/sanvol1/scratch/hg17/nib
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 SEQ1_LEN=/san/sanvol1/scratch/hg17/chrom.sizes
 
 # QUERY - macaca mulatta
 SEQ2_DIR=/san/sanvol1/scratch/rheMac2/rheMac2.2bit
 SEQ2_CHUNK=5000000
 SEQ2_LAP=0
 SEQ2_LEN=/san/sanvol1/scratch/rheMac2/rheMac2.sizes 
 
 BASE=/san/sanvol1/scratch/hg17/blastz.rheMac2/
 RAW=$BASE/raw
 TMPDIR=/scratch/tmp
 'EOF'
     # << happy emacs
     # use chain parameters for "close" species
     /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk -chainMinScore=3000 -chainLinearGap=medium \
 	`pwd`/DEF >& blastz.out &
 
     /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk -chainMinScore=3000 -chainLinearGap=medium \
 	-continue=chainRun \
 	`pwd`/DEF >& continueChainRun.out &
 
     # NOTE: must set -fileServer (e.g. to pk) if using base dir on SAN
 
     /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk -fileServer=pk \
         -chainMinScore=3000 -chainLinearGap=medium \
 	-continue=chainMerge \
 	`pwd`/DEF >& continueChainMerge.out &
 
     # netClass was crashing as it expected a bin in the
     # unsplit gap table.  Robert added the bin field.
 
     /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk -fileServer=pk \
 	-continue=download \
 	`pwd`/DEF >& continueDownload.out &
 
     ssh hgwdev "featureBits hg17 chainRheMac1Link" >& rheMac1.fb; cat rheMac1.fb
     ssh hgwdev "featureBits hg17 chainRheMac2Link" >& rheMac2.fb; cat rheMac2.fb
 
     ssh kkstore02 
     cd /cluster/data/hg17/bed/blastz.rheMac2
     cp -rp mafNet /san/sanvol1/scratch/hg17/mafNet/rheMac2
 
 # SWAP CHAIN AND NET ALIGNMENTS OVER TO RHESUS (rheMac2) 
 # CREATE CHAIN AND NET TRACKS, AXTNET, MAFNET, LIFTOVER AND ALIGNMENT DOWNLOADS
 # (DONE, 2006-03-22, hartera) 
     # Do the swap of hg17/rheMac2 alignments over to rheMac2 to produce
     # rheMac2/hg17 alignments. 
     ssh pk
     cd /cluster/data/hg17/bed/blastz.rheMac2
     # use chain parameters for "close" species
     /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
         -swap -bigClusterHub=pk -chainMinScore=3000 -chainLinearGap=medium \
         `pwd`/DEF >& swap.log &
     # Took about 3 hours 40 minutes to run.
 
 #############################################################################
 # 17-WAY MULTIZ ALIGNMENTS (DONE - 2005-12-20 kate)
 #       # redo fix overlaps from xenTro1 and tetNig1 (2006-04-08 kate)
 
     # copy net mafs to cluster-friendly storage for multiz run (2006-01-25 kate)
 
     ssh kkstore01
     cd /cluster/data/hg17/bed/blastz.monDom2
     cp -rp mafNet /san/sanvol1/scratch/hg17/mafNet/monDom2
 
     ssh kkstore02
     cd /cluster/data/hg17/bed
     mkdir -p multiz17way.2005-12-20
     ln -s multiz17way.2005-12-20 multiz17way
     cd multiz17way
 
     # copy MAF's to cluster-friendly server
     # These MAF's already on bluearc:
     #  canFam2, fr1, galGal2, panTro1, rn3
     mkdir -p /san/sanvol1/scratch/hg17/mafNet
     cd /san/sanvol1/scratch/hg17/mafNet
     ln -s /cluster/bluearc/hg17/mafNet/{*} .
 
     # copy others
     foreach s (rheMac1 oryCun1 dasNov1 \
             loxAfr1 bosTau2 monDom1 xenTro1 tetNig1 danRer3)
         echo $s
         cp -Rp /cluster/data/hg17/bed/blastz.$s/mafNet $s
     end
     # a few more
     set s = echTel1
     cp -Rp /cluster/data/hg17/bed/blastz.$s/mafNet $s
     set s = mm7
     cp -Rp /cluster/data/hg17/bed/blastz.$s/mafNet $s
     set s = canFam2
     cp -Rp /cluster/data/hg17/bed/blastz.$s/mafNet $s
     set s = rheMac2
     cp -Rp /cluster/data/hg17/bed/blastz.$s/mafNet $s
 
     # thanks for the tree, Hiram! Taken from mm7 17way...
     # Hiram says this is derived from the latest ENCODE
     # tree, with some species removed and branch lengths
     # adjusted.  The ENCODE tree  from the Sept. freeze is:
     # ftp://kronos.nhgri.nih.gov/pub/outgoing/elliott/msa/SEP-2005/phylo/tree_4d.tba.v2.nh
     cd /cluster/data/hg17/bed/multiz17way
     cat << '_EOF_' > 17way.nh
 (((((((((
 (human_hg17:0.006690,chimp_panTro1:0.007571):0.024272,
   macaque_rheMac2:0.0592):0.023960,
   ((rat_rn3:0.081728,mouse_mm7:0.077017):0.229273,
       rabbit_oryCun1:0.206767):0.1065):0.023026,
 (cow_bosTau2:0.159182,dog_canFam2:0.147731):0.039450):0.028505,
 armadillo_dasNov1:0.149862):0.015994,
 (elephant_loxAfr1:0.104891,tenrec_echTel1:0.259797):0.040371):0.218400,
 monodelphis_monDom2:0.371073):0.189124,
 chicken_galGal2:0.454691):0.123297,
 xenopus_xenTro1:0.782453):0.156067,
 ((tetraodon_tetNig1:0.199381,fugu_fr1:0.239894):0.492961,
     zebrafish_danRer3:0.782561):0.156067);
 '_EOF_'
 
     /cluster/bin/phast/draw_tree 17way.nh > 17way.ps
     /cluster/bin/phast/all_dists 17way.nh > 17way.distances.txt
     grep hg17 17way.distances.txt | sort -k3,3n | \
         awk '{printf ("%.4f\t%s\n", $3, $2)}' > distances.txt
     # edit distances.txt to include featureBits, and chain parameters
     # from blastz run.
     cat distances.txt
         # 0.0143  chimp_panTro1
         # 0.0902  macaque_rheMac2
         # 0.2563  armadillo_dasNov1
         # 0.2651  dog_canFam2
         # 0.2677  elephant_loxAfr1
         # 0.2766  cow_bosTau2
         # 0.3682  rabbit_oryCun1
         # 0.4226  tenrec_echTel1
         # 0.4677  mouse_mm7
         # 0.4724  rat_rn3
     # use loose chain params and score from here, down (5000)
         # 0.7119  monodelphis_monDom1
         # 0.9847  chicken_galGal2
         # 1.4357  xenopus_xenTro1
         # 1.6577  tetraodon_tetNig1
         # 1.6983  fugu_fr1
         # 1.7480  zebrafish_danRer3
 
     # the order in the browser display will be by tree topology,
     #  not by distance, so it will be: 
 #  >>         # 0.0143  chimp_panTro1
 #  >>         # 0.0902  macaque_rheMac2
 #  >>         # 0.4677  mouse_mm7
 #  >>         # 0.4724  rat_rn3
 #  >>         # 0.3682  rabbit_oryCun1
 #  >>         # 0.2651  dog_canFam2
 #  >>         # 0.2766  cow_bosTau2
 #  >>         # 0.2563  armadillo_dasNov1
 #  >>         # 0.2677  elephant_loxAfr1
 #  >>         # 0.4226  tenrec_echTel1
 #  >>         # 0.7119  monodelphis_monDom1
 #  >>         # 0.9847  chicken_galGal2
 #  >>         # 1.4357  xenopus_xenTro1
 #  >>         # 1.6577  tetraodon_tetNig1
 #  >>         # 1.6983  fugu_fr1
 #  >>         # 1.7480  zebrafish_danRer3
 
     # make output dir and run dir
     ssh pk
     cd /cluster/data/hg17/bed/multiz17way.2005-12-20
 
     # create species list and stripped down tree for autoMZ
     sed 's/[a-z][a-z]*_//g; s/:[0-9\.][0-9\.]*//g; s/;//' 17way.nh > tmp.nh
     echo `cat tmp.nh` > tree-commas.nh
     echo `cat tree-commas.nh` | sed 's/ //g; s/,/ /g' > tree.nh
     sed 's/[()]//g; s/,/ /g' tree.nh > species.lst
 
     mkdir -p maf run
     cd run
 
     # stash binaries 
     mkdir penn
     cp -p /cluster/bin/penn/multiz.v11.x86_64/multiz-tba/multiz penn
     cp -p /cluster/bin/penn/multiz.v11.x86_64/multiz-tba/maf_project penn
     cp -p /cluster/bin/penn/multiz.v11.x86_64/multiz-tba/autoMZ penn
 
 cat > autoMultiz.csh << 'EOF'
 #!/bin/csh -ef
     set db = hg17
     set c = $1
     set maf = $2
     set run = `pwd`
     set tmp = /scratch/tmp/$db/multiz.$c
     set pairs = /san/sanvol1/scratch/$db/mafNet
     rm -fr $tmp
     mkdir -p $tmp
     cp ../{tree.nh,species.lst} $tmp
     pushd $tmp
     foreach s (`cat species.lst`)
         set in = $pairs/$s/$c.maf
         set out = $db.$s.sing.maf
         if ($s == hg17) then
             continue
         endif
         if (-e $in.gz) then
             zcat $in.gz > $out
         else if (-e $in) then
             cp $in $out
         else
             echo "##maf version=1 scoring=autoMZ" > $out
         endif
     end
     set path = ($run/penn $path); rehash
     $run/penn/autoMZ + T=$tmp E=$db "`cat tree.nh`" $db.*.sing.maf $c.maf
     popd
     cp $tmp/$c.maf $maf
     rm -fr $tmp
 'EOF'
 # << happy emacs
     chmod +x autoMultiz.csh
 
 cat  << 'EOF' > spec
 #LOOP
 ./autoMultiz.csh $(root1) {check out line+ /cluster/data/hg17/bed/multiz17way.2005-12-20/maf/$(root1).maf}
 #ENDLOOP
 'EOF'
 # << happy emacs
 
     awk '{print $1}' /cluster/data/hg17/chrom.sizes > chrom.lst
 
     # REDO FOR OVERLAPS (2006-04-07 kate)
     mv ../maf ../maf.old
     # edit spec file to fix maf dir path
     gensub2 chrom.lst single spec jobList
     para create jobList
         # 46 files
     para try
     para check
     para push
     para time > run.time
         # 36 hrs (not typical -- previous runs were ~16 hrs)
 
 
 # PHASTCONS CONSERVATION (2006-01-05 kate)
 # Redone when multiz redone to fix overlaps (2006-04-12)
 # This process is distilled from Hiram and Adam's experiments
 # on mouse (mm7) 17way track.  Many parameters are now fixed, without
 # being experimentally derived, either because the experiments
 # were lengthy and produced similar results, or because they
 # weren't runnable given the alignment size.
 # These parameters are:
 # --rho
 # --expected-length
 # --target-coverage
 # Also, instead of generating cons and noncons tree models,
 # we use a single, pre-existing tree model -- Elliot Margulies' model 
 # from the (37-way) ENCODE alignments.
 #
 # NOTE: Redone 3/20/06, adding rheMac2 to non-informative options,
 #       by recommendation of Adam Siepel, to correct unwanted
 #       high conservation in regions with primate-only alignments
 
     # NOTE: reusing cluster-friendly chrom fasta files created earlier
     #cd /cluster/data/hg17
     #foreach f (`cat chrom.lst`)
         #echo $f
         #cp $f/*.fa /cluster/bluearc/hg17/chrom
     #end
 
     # Split chromosome MAF's into windows and use to generate
     # "sufficient statistics" (ss) files for phastCons input
     # NOTE: as the SAN fs has lotsa space, we're leaving these
     # big (temp) files unzipped, to save time during phastCons run.
     # Note also the larger chunk sizes from previous runs -- this
     # reduces run-time on the split, slows down the actual phastCons
     # enough so jobs don't crash (jobs are very quick, just a minute
     # or so), and according to Adam, will produce better results.
     # The previous small chunks were probably required by
     # the phyloFit step, which we are no longer using for the
     # human alignments.
     ssh pk
     mkdir /cluster/data/hg17/bed/multiz17way.2005-12-20/cons
     cd /cluster/data/hg17/bed/multiz17way.2005-12-20/cons
     cp /san/sanvol1/scratch/mm7/cons/elliotsEncode.mod .
         # edit, changing rheMac1 -> rheMac2
     mkdir run.split
     cd run.split
     set WINDOWS = /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/ss
     rm -fr $WINDOWS
     mkdir -p $WINDOWS
 
     cat << 'EOF' > doSplit.csh
 #!/bin/csh -ef
     set MAFS = /cluster/data/hg17/bed/multiz17way.2005-12-20/maf
     set WINDOWS = /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/ss
     cd $WINDOWS
     set c = $1
     echo $c
     rm -fr $c
     mkdir $c
     /cluster/bin/phast/$MACHTYPE/msa_split $MAFS/$c.maf -i MAF \
         -M /cluster/bluearc/hg17/chrom/$c.fa \
         -o SS -r $c/$c -w 10000000,0 -I 1000 -B 5000
     echo "Done" >> $c.done
 'EOF'
 # << happy emacs
     chmod +x doSplit.csh
 
     rm -f jobList
     foreach f (../../maf/*.maf) 
         set c = $f:t:r
 	echo "doSplit.csh $c {check out line+ $WINDOWS/$c.done}" >> jobList
     end
     
     para create jobList
         # 46 jobs
     para try
     para check
     para push
 # CPU time in finished jobs:       9511s     158.52m     2.64h    0.11d  0.000 y
 # IO & Wait Time:                  5391s      89.85m     1.50h    0.06d  0.000 y
 # Average job time:                 324s       5.40m     0.09h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:            2354s      39.23m     0.65h    0.03d
 # Submission to last job:          2358s      39.30m     0.66h    0.03d
 
     # check tree model on 5MB chunk, using params recommended by Adam,
     # (to verify branch lengths on 2X species)
     # he ok'ed the results -- not necessary for next human run
     ssh kolossus
     cd /cluster/data/hg17/bed/multiz17way.2005-12-20/cons
     /cluster/bin/phast/$MACHTYPE/phyloFit -i SS -E -p MED -s HKY85 \
         --tree "`cat ../tree-commas.nh`" \
         /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/ss/chr7/chr7.115000658-120000000.ss \
         -o phyloFit.tree
 
     # Run phastCons
     #	This job is I/O intensive in its output files, thus it is all
     #	working over in /scratch/tmp/
     cd ..
     mkdir run.cons
     cd run.cons
     cat > doPhast.csh << 'EOF'
 #!/bin/csh -fe
 set c = $1
 set f = $2
 set len = $3
 set cov = $4
 set rho = $5
 set tmp = /scratch/tmp/$f
 mkdir -p $tmp
 set san = /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
 cp -p $san/ss/$c/$f.ss ../elliotsEncode.mod $tmp
 pushd $tmp > /dev/null
 /cluster/bin/phast/$MACHTYPE/phastCons $f.ss elliotsEncode.mod \
    --rho $rho --expected-length $len --target-coverage $cov --quiet \
     --not-informative panTro1,rheMac2 \
 	--seqname $c --idpref $c --viterbi $f.bed --score > $f.pp
 popd > /dev/null
 mkdir -p $san/pp/$c $san/bed/$c
 sleep 1
 mv $tmp/$f.pp $san/pp/$c
 mv $tmp/$f.bed $san/bed/$c
 rm -fr $tmp
 'EOF'
     # emacs happy
     chmod a+x doPhast.csh
 
     #	root1 == chrom name, file1 == ss file name without .ss suffix
     # Create gsub file
     cat > template << 'EOF'
 #LOOP
 doPhast.csh $(root1) $(file1) 14 .008 .28
 #ENDLOOP
 'EOF'
     #	happy emacs
 
     # Create parasol batch and run it
     pushd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
     ls -1 ss/chr*/chr*.ss | sed 's/.ss$//' > \
         /cluster/data/hg17/bed/multiz17way/cons/run.cons/in.list
     popd
 
     gensub2 in.list single template jobList
     para create jobList
         # 333 jobs
     para try
     para check
     para push
     # NOTE: these jobs go fast -- some crashed apparently having
     # difficulty accessing input files.  Just restart them and
     # they work
 #CPU time in finished jobs:      15520s     258.67m     4.31h    0.18d  0.000 y
 #IO & Wait Time:                 15796s     263.27m     4.39h    0.18d  0.001 y
 #Average job time:                  94s       1.57m     0.03h    0.00d
 #Longest running job:                0s       0.00m     0.00h    0.00d
 #Longest finished job:             180s       3.00m     0.05h    0.00d
 #Submission to last job:         48266s     804.43m    13.41h    0.56d
 
     # create Most Conserved track
     ssh kolossus
     cd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
     #	The sed's and the sort get the file names in chrom,start order
     # (Hiram tricks -- split into columns on [.-/] with 
     #    identifying x,y,z, to allow column sorting and
     #    restoring the filename.  Warning: the sort column
     # will depend on how deep you are in the dir
     find ./bed -name "chr*.bed" | \
         sed -e "s/\// x /g" -e "s/\./ y /g" -e "s/-/ z /g" | \
 	sort -k7,7 -k9,9n | \
 	sed -e "s/ x /\//g" -e "s/ y /\./g" -e "s/ z /-/g" | \
 	xargs cat | \
 	awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", $1, $2, $3, $5, $5;}' | \
 	/cluster/bin/scripts/lodToBedScore /dev/stdin > mostConserved.bed
     #	~ 1 minute
     cp -p mostConserved.bed /cluster/data/hg17/bed/multiz17way/cons
 
     # load into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz17way/cons
     hgLoadBed -strict hg17 phastConsElements17way mostConserved.bed
         # Loaded 2212445 elements of size 5
     # compare with previous tracks
     hgsql hg17 -e "select count(*) from phastConsElements10way"
         # 2011952
     hgsql hg17 -e "select count(*) from phastConsElements"
         # 1601903
     # Try for 5% overall cov, and 70% CDS cov (used elen=14, tcov=.008, rho=.28)
     
     featureBits hg17 -enrichment refGene:cds phastConsElements17way
         # refGene:cds 1.065%, phastConsElements17way 5.116%, both 0.759%, cover 71.27%, enrich 13.93x
 
     # compare with previous tracks
     featureBits hg17 -enrichment refGene:cds phastConsElements10way
         # refGene:cds 1.062%, phastConsElements10way 5.003%, both 0.734%, cover 69.18%, enrich 13.83x
     featureBits hg17 -enrichment refGene:cds phastConsElements
         # refGene:cds 1.062%, phastConsElements 4.810%, both 0.771%, cover 72.65%, enrich 15.11x
 
     # experiments
     # previous tracks
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements
         # refGene:cds 0.873%, phastConsElements 4.497%, both 0.630%, cover 72.10%, enrich 16.04x
     hgsql hg17 -e "select count(*) from phastConsElements where chrom='chr7'"
         # 81785
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements10way
         # refGene:cds 0.873%, phastConsElements10way 4.700%, both 0.602%, cover 68.94%, enrich 14.67x
     hgsql hg17 -e "select count(*) from phastConsElements10way where chrom='chr7'"
         # 102959
 
     # len=13, cov=.007, rho=.27
     # looks best -- similar chr7 measurements to previous tracks
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_13_007_27
         # refGene:cds 0.874%, phastConsElements17way_13_007_27 4.854%, both 0.607%, cover 69.43%, enrich 14.31x
     hgsql hg17 -e "select count(*) from phastConsElements17way_13_007_27 where chrom='chr7'"
 
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_13_005_28
        # refGene:cds 0.873%, phastConsElements17way_13_005_28 4.802%, both 0.612%, cover 70.12%, enrich 14.60x
     hgsql hg17 -e "select count(*) from phastConsElements17way_13_005_28 where chrom='chr7'"
        # 95203 
 
     # experiments with other parameters, below
     # len=15, cov=.10 
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_15_10
         # refGene:cds 0.873%, phastConsElements17way 7.989%, both 0.627%, cover 71.77%, enrich 8.98x
     hgsql hg17 -e "select count(*) from phastConsElements17way_15_10 where chrom='chr7'"
         # 217767 
     # => too much overall covg, and too many elements
 
     # len=15, cov=.05
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_15_05
         # refGene:cds 0.873%, phastConsElements17way_15_05 6.880%, both 0.627%, cover 71.77%, enrich 10.43x
     hgsql hg17 -e "select count(*) from phastConsElements17way_15_05 where chrom='chr7'"
         # 166868
 
     # len=15, cov=.01
     # These values were used by Elliott for ENCODE
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_15_01
         # refGene:cds 0.873%, phastConsElements17way_15_01 5.721%, both 0.628%, cover 71.89%, enrich 12.57x
     hgsql hg17 -e "select count(*) from phastConsElements17way_15_01 where chrom='chr7'"
         # 106034
 
     # len=20, cov=.01
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_20_01
         # refGene:cds 0.873%, phastConsElements17way_20_01 7.751%, both 0.634%, cover 72.56%, enrich 9.36x
     hgsql hg17 -e "select count(*) from phastConsElements17way_20_01 where chrom='chr7'"
         # 106005
     # -> wrong direction on coverage
 
     # len=10, cov=.01
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_10_01
         # refGene:cds 0.873%, phastConsElements17way_10_01 4.653%, both 0.616%, cover 70.48%, enrich 15.15x
     hgsql hg17 -e "select count(*) from phastConsElements17way_10_01 where chrom='chr7'"
         # 108279
     # => looks good on coverage and element count, check smoothness in browser
     # => undersmoothed
 
     # len=10, cov=.05
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_10_05
         # refGene:cds 0.873%, phastConsElements17way_10_05 5.365%, both 0.615%, cover 70.44%, enrich 13.13x 
     hgsql hg17 -e "select count(*) from phastConsElements17way_10_05 where chrom='chr7'"
         # 178372
     # => fragmented elements
 
     # len=15, cov=.005
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_15_005
         # refGene:cds 0.873%, phastConsElements17way_15_005 5.444%, both 0.628%, cover 71.93%, enrich 13.21x
     hgsql hg17 -e "select count(*) from phastConsElements17way_15_005 where chrom='chr7'"
         # 90855
 
     # len=20, cov=.005
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_20_005
         # refGene:cds 0.873%, phastConsElements17way_20_005 7.373%, both 0.634%, cover 72.61%, enrich 9.85x
     hgsql hg17 -e "select count(*) from phastConsElements17way_20_005 where chrom='chr7'"
         # 91858
 
     # len=17, cov=.005 rho=.3
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_17_005
         # refGene:cds 0.873%, phastConsElements17way_17_005 6.126%, both 0.631%, cover 72.24%, enrich 11.79x
     hgsql hg17 -e "select count(*) from phastConsElements17way_17_005 where chrom='chr7'"
         # 91243
 
     # len=12, cov=.01, rho=.28 -panTro1
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_12_01_28_p
         # refGene:cds 0.873%, phastConsElements17way_12_01_28_p 4.829%, both 0.612%, cover 70.02%, enrich 14.50x
     hgsql hg17 -e "select count(*) from phastConsElements17way_12_01_28_p where chrom='chr7'"
         # 123638
 
     # len=13, cov=.01, rho=.25 -panTro1
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_13_01_25_p
         # refGene:cds 0.873%, phastConsElements17way_13_01_25_p 4.793%, both 0.594%, cover 67.99%, enrich 14.19x
     hgsql hg17 -e "select count(*) from phastConsElements17way_13_01_25_p where chrom='chr7'"
         # 131895
 
     # len=14, cov=.008, rho=.28
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_14_008_28
        # refGene:cds 0.874%, phastConsElements17way_14_008_28 5.227%, both 0.615%, cover 70.37%, enrich 13.46x
     hgsql hg17 -e "select count(*) from phastConsElements17way_14_008_28 where chrom='chr7'"
         # 106071
 
     # Create merged posterier probability file and wiggle track data files
     #	pk is currently closer to the san than any other machine
     ssh pk
     cd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/
     # sort by chromName, chromStart so that items are in numerical order 
     #  for wigEncode
     #next time try Angie's simpler sort, below
     find ./pp -name "chr*.pp" | \
         sed -e "s/\// x /g" -e "s/\./ y /g" -e "s/-/ z /g" | \
 	sort -k7,7 -k9,9n | \
 	sed -e "s/ x /\//g" -e "s/ y /\./g" -e "s/ z /-/g" | \
 	xargs cat | \
         nice wigEncode stdin phastCons17way.wig phastCons17way.wib
     # about 23 minutes for above
         # GOT HERE ON REDO
         # NOTE: remember to flip /gbdb link from cons.old to cons
     #foreach chr (`awk '{print $1}' /cluster/data/hg17/chrom.sizes`)
       #echo $chr
       set chr = chr22
       cat `ls -1 pp/$chr/$chr.*.pp | sort -t\. -k2,2n` \
         | nice wigEncode stdin phastCons17wayNewChr22.wig phastCons17wayNewChr22.wib
     #end
     date
 
     cp -p phastCons17way.wi? /cluster/data/hg17/bed/multiz17way/cons
 
     # Load gbdb and database with wiggle.
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz17way/cons
     ln -s /cluster/data/hg17/bed/multiz17way/cons/phastCons17way.wib \
         /gbdb/hg17/multiz17way/phastCons17way.wib
     hgLoadWiggle -pathPrefix=/gbdb/hg17/multiz17way hg17 \
         phastCons17way phastCons17way.wig
 
 ############################################################################
     ## Run phastCons on Placental mammals
     ssh pk
     cd /cluster/data/hg17/bed/multiz17way.2005-12-20/cons
     mkdir placental
     mkdir run.cons.alt
     cd run.cons.alt
 
     # create pruned trees 
     set tree_doctor = /cluster/bin/phast/tree_doctor
     sed 's/ /,/g' ../../species.lst 
     # hg17,panTro1,rheMac2,rn3,mm7,oryCun1,bosTau2,canFam2,dasNov1,loxAfr1,echTel1,monDom2,galGal2,xenTro1,tetNig1,fr1,danRer3
     mkdir placental
     $tree_doctor ../elliotsEncode.mod \
         --prune-all-but=hg17,panTro1,rheMac2,rn3,mm7,oryCun1,bosTau2,canFam2,dasNov1,loxAfr1,echTel1 \
                 > placental/placental.mod
 
     cat > doPhast.csh << 'EOF'
 #!/bin/csh -fe
 set c = $1
 set f = $2
 set len = $3
 set cov = $4
 set rho = $5
 set grp = $6
 set tmp = /scratch/tmp/hg17/$grp/$f
 mkdir -p $tmp
 set san = /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
 cp -p $san/ss/$c/$f.ss $grp/$grp.mod $tmp
 pushd $tmp > /dev/null
 /cluster/bin/phast/$MACHTYPE/phastCons $f.ss $grp.mod \
    --rho $rho --expected-length $len --target-coverage $cov --quiet \
     --not-informative panTro1,rheMac2 \
 	--seqname $c --idpref $c --viterbi $f.bed --score > $f.pp
 popd > /dev/null
 mkdir -p $san/$grp/pp/$c $san/$grp/bed/$c
 sleep 1
 mv $tmp/$f.pp $san/$grp/pp/$c
 mv $tmp/$f.bed $san/$grp/bed/$c
 rm -fr $tmp
 'EOF'
     # << emacs happy
 
     chmod a+x doPhast.csh
 
     # Create gsub file
     cat > template << 'EOF'
 #LOOP
 # template for 5% cov
 doPhast.csh $(root1) $(file1) 14 .2 .28 placental
 #ENDLOOP
 'EOF'
     cat > template << 'EOF'
 #LOOP
 # template same as vertebrate
 doPhast.csh $(root1) $(file1) 14 .008 .28 placental
 #ENDLOOP
 'EOF'
     #	happy emacs
 
     # Create parasol batch and run it
     pushd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
     ls -1 ss/chr*/chr*.ss | sed 's/.ss$//' > \
         /cluster/data/hg17/bed/multiz17way/cons/run.cons.alt/in.list
     popd
 
     gensub2 in.list single template jobList
     para create jobList
         # 333 jobs
     para try
     para check
     para push
 #.2
 #CPU time in finished jobs:      15164s     252.74m     4.21h    0.18d  0.000 y
 #IO & Wait Time:                 14852s     247.53m     4.13h    0.17d  0.000 y
 #Average job time:                  90s       1.50m     0.03h    0.00d
 #Longest running job:                0s       0.00m     0.00h    0.00d
 #Longest finished job:             170s       2.83m     0.05h    0.00d
 #Submission to last job:         86364s    1439.40m    23.99h    1.00d
 
 #.008
 #CPU time in finished jobs:      13712s     228.53m     3.81h    0.16d  0.000 y
 #IO & Wait Time:                 14407s     240.12m     4.00h    0.17d  0.000 y
 #Average job time:                  84s       1.41m     0.02h    0.00d
 #Longest running job:                0s       0.00m     0.00h    0.00d
 #Longest finished job:             159s       2.65m     0.04h    0.00d
 #Submission to last job:          5291s      88.18m     1.47h    0.06d
 
     ssh pk
     cd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/placental
     #	The sed's and the sort get the file names in chrom,start order
     # (Hiram tricks -- split into columns on [.-/] with 
     #    identifying x,y,z, to allow column sorting and
     #    restoring the filename.  Warning: the sort column
     # will depend on how deep you are in the dir
     find ./bed -name "chr*.bed" | \
         sed -e "s/\// x /g" -e "s/\./ y /g" -e "s/-/ z /g" | \
 	sort -k7,7 -k9,9n | \
 	sed -e "s/ x /\//g" -e "s/ y /\./g" -e "s/ z /-/g" | \
 	xargs cat | \
 	awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", $1, $2, $3, $5, $5;}' | \
 	/cluster/bin/scripts/lodToBedScore /dev/stdin > mostConserved.bed
     #	~ 1 minute
     cp -p mostConserved.bed /cluster/data/hg17/bed/multiz17way/cons/placental
 
     # load into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz17way/cons/placental
     hgLoadBed -strict hg17 phastConsElementsPlacental mostConserved.bed
         # .2
         # Loaded 3775983 elements of size 5
         # .008
         # Loaded 1290060 elements of size 5
 
     # compare with vertebrate cons
     hgsql hg17 -e "select count(*) from phastConsElements17way"
         # 2212445
     featureBits hg17 -enrichment refGene:cds phastConsElementsPlacental
     featureBits hg17 -enrichment refGene:cds phastConsElementsPlacental_14_2_28
     featureBits hg17 -enrichment refGene:cds phastConsElements17way
 
     featureBits hg17 -enrichment refGene:cds phastConsElementsPlacental
 # refGene:cds 1.070%, phastConsElementsPlacental 3.844%, both 0.667%, cover 62.32%, enrich 16.21x
 # refGene:cds 1.069%, phastConsElementsPlacental_14_008_28 3.844%, both 0.667%, cover 62.37%, enrich 16.22x
 
     featureBits hg17 -enrichment refGene:cds phastConsElementsPlacental_14_2_28
 #refGene:cds 1.070%, phastConsElementsPlacental_14_2_28 5.223%, both 0.691%, cover 64.62%, enrich 12.37x
     featureBits hg17 -enrichment refGene:cds phastConsElements17way
 #refGene:cds 1.070%, phastConsElements17way 5.116%, both 0.763%, cover 71.27%, enrich 13.93x
 
 
     # Create merged posterier probability file and wiggle track data files
     #	pk is currently closer to the san than any other machine
     ssh pk
     cd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/placental
     # sort by chromName, chromStart so that items are in numerical order 
     #  for wigEncode
     #next time try Angie's simpler sort, below
     find ./pp -name "chr*.pp" | \
         sed -e "s/\// x /g" -e "s/\./ y /g" -e "s/-/ z /g" | \
 	sort -k7,7 -k9,9n | \
 	sed -e "s/ x /\//g" -e "s/ y /\./g" -e "s/ z /-/g" | \
 	xargs cat | \
         nice wigEncode stdin phastConsPlacental.wig phastConsPlacental.wib
     # about 23 minutes for above
         # GOT HERE ON REDO
         # NOTE: remember to flip /gbdb link from cons.old to cons
 
     cp -p phastConsPlacental.wi?  \
                 /cluster/data/hg17/bed/multiz17way/cons/placental
 
     # Load gbdb and database with wiggle.
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz17way/cons/placental
     ln -s \
      /cluster/data/hg17/bed/multiz17way/cons/placental/phastConsPlacental.wib \
         /gbdb/hg17/multiz17way/phastConsPlacental.wib
     hgLoadWiggle -pathPrefix=/gbdb/hg17/multiz17way hg17 \
         phastConsPlacental phastConsPlacental.wig
 
 ############################################################################
     ## Run phastCons on subgroups (mammals, placentals, and w/o low-cov)
     ssh pk
     cd /cluster/data/hg17/bed/multiz17way.2005-12-20/cons
     mkdir run.cons.groups
     cd run.cons.groups
 
     # create pruned trees 
     set tree_doctor = /cluster/bin/phast/tree_doctor
     sed 's/ /,/g' ../../species.lst 
     # hg17,panTro1,rheMac2,rn3,mm7,oryCun1,bosTau2,canFam2,dasNov1,loxAfr1,echTel1,monDom2,galGal2,xenTro1,tetNig1,fr1,danRer3
 
     $tree_doctor ../elliotsEncode.mod \
         --prune-all-but=hg17,panTro1,rheMac2,rn3,mm7,bosTau2,canFam2,monDom2,galGal2,xenTro1,tetNig1,fr1,danRer3 \
                 > vertebrate-high.mod
     $tree_doctor ../elliotsEncode.mod \
         --prune-all-but=hg17,panTro1,rheMac2,rn3,mm7,bosTau2,canFam2,monDom2 \
                 > mammal-high.mod
     $tree_doctor ../elliotsEncode.mod \
         --prune-all-but=hg17,panTro1,rheMac2,rn3,mm7,bosTau2,canFam2 \
                 > placental-high.mod
     $tree_doctor ../elliotsEncode.mod \
         --prune-all-but=hg17,panTro1,rheMac2,rn3,mm7,oryCun1,bosTau2,canFam2,dasNov1,loxAfr1,echTel1,monDom2 \
                 > mammal.mod
     $tree_doctor ../elliotsEncode.mod \
         --prune-all-but=hg17,panTro1,rheMac2,rn3,mm7,oryCun1,bosTau2,canFam2,dasNov1,loxAfr1,echTel1 \
                 > placental.mod
     foreach f (*.mod)
         mkdir $f:r
         mv $f $f:r
     end
     cat > doPhast.csh << 'EOF'
 #!/bin/csh -fe
 set c = $1
 set f = $2
 set len = $3
 set cov = $4
 set rho = $5
 set grp = $6
 set tmp = /scratch/tmp/hg17/$grp/$f
 mkdir -p $tmp
 set san = /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
 cp -p $san/ss/$c/$f.ss $grp/$grp.mod $tmp
 pushd $tmp > /dev/null
 /cluster/bin/phast/$MACHTYPE/phastCons $f.ss $grp.mod \
    --rho $rho --expected-length $len --target-coverage $cov --quiet \
     --not-informative panTro1,rheMac2 \
 	--seqname $c --idpref $c --viterbi $f.bed --score > $f.pp
 popd > /dev/null
 mkdir -p $san/$grp/pp/$c $san/$grp/bed/$c
 sleep 1
 mv $tmp/$f.pp $san/$grp/pp/$c
 mv $tmp/$f.bed $san/$grp/bed/$c
 rm -fr $tmp
 'EOF'
     # emacs happy
     chmod a+x doPhast.csh
 
     #	root1 == chrom name, file1 == ss file name without .ss suffix
     # Create gsub file
     cat > template << 'EOF'
 #LOOP
 doPhast.csh $(root1) $(file1) 14   .21 .28 placental-high
 doPhast.csh $(root1) $(file1) 14   .2 .28 placental
 doPhast.csh $(root1) $(file1) 14   .11 .28 mammal
 doPhast.csh $(root1) $(file1) 14   .1 .28 mammal-high
 doPhast.csh $(root1) $(file1) 14 .0028 .28 vertebrate-high
 #ENDLOOP
 
 'EOF'
     #	happy emacs
 
     # Create parasol batch for just chr7 (for test purposes) and run it
     pushd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
     ls -1 ss/chr7/chr*.ss | sed 's/.ss$//' > \
         /cluster/data/hg17/bed/multiz17way/cons/run.cons.groups/in.list
     popd
 
     gensub2 in.list single template jobList
     para create jobList
         # 80 jobs
     para try
     para check
     para push
     # 24 minutes
 
 
     ## create Alt Most Conserved track
     ssh hgwdev
     cd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
 cat > loadAltElements.csh << 'EOF'
     set b = /cluster/data/hg17/bed/multiz17way/cons/run.cons.groups
     foreach d (mammal* placental* vertebrate*)
       echo $d
       cd $d
       find ./bed -name "chr*.bed" | \
         sed -e "s/\// x /g" -e "s/\./ y /g" -e "s/-/ z /g" | \
 	sort -k7,7 -k9,9n | \
 	sed -e "s/ x /\//g" -e "s/ y /\./g" -e "s/ z /-/g" | \
 	xargs cat | \
 	awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", $1, $2, $3, $5, $5;}' | \
 	/cluster/bin/scripts/lodToBedScore /dev/stdin \
             > $b/$d/mostConserved.bed
         set table = `echo $d | perl -wpe "s/(.*)/phastConsElements\u$1/;s/-(.*)/\u$1/"`
         hgLoadBed -strict hg17 $table $b/$d/mostConserved.bed
         featureBits hg17 -enrichment refGene:cds -chrom=chr7 $table
       cd ..
     end
 'EOF'
     csh loadAltElements.csh >&! loadAltElements.log &
     grep refGene loadAltElements.log | sort -n -k4
 # refGene:cds 0.884%, phastConsElementsPlacentalHigh 4.828%, both 0.606%, cover 68.51%, enrich 14.19x
 # refGene:cds 0.884%, phastConsElementsMammal 4.869%, both 0.580%, cover 65.62%, enrich 13.48x
 # refGene:cds 0.884%, phastConsElementsMammalHigh 4.887%, both 0.624%, cover 70.60%, enrich 14.45x
 # refGene:cds 0.884%, phastConsElementsPlacental 4.904%, both 0.558%, cover 63.14%, enrich 12.88x
 # refGene:cds 0.884%, phastConsElementsVertebrateHigh 4.965%, both 0.652%, cover 73.74%, enrich 14.85x
 
     featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way
 # refGene:cds 0.884%, phastConsElements17way 4.851%, both 0.623%, cover 70.48%, enrich 14.53x
 
     ssh kkstore02
     cd /cluster/data/hg17/bed/multiz17way/cons/run.cons.groups
 cat > makeAltWiggle.csh << 'EOF'
     set b = `pwd`
     set san = /san/sanvol1/scratch/hg17/multiz17way/cons
     pushd $san
     foreach d (mammal* placental* vertebrate*)
       echo $d
       cd $d
       set table = `echo $d | perl -wpe 's/(.*)/phastCons\u$1/;s/-(.*)/\u$1/'`
       echo $table
       find ./pp -name "chr*.pp" | \
         sed -e "s/\// x /g" -e "s/\./ y /g" -e "s/-/ z /g" | \
 	sort -k7,7 -k9,9n | \
 	sed -e "s/ x /\//g" -e "s/ y /\./g" -e "s/ z /-/g" | \
 	xargs cat | \
         nice wigEncode stdin $table.wig $table.wib
       mv $table.wig $table.wib $b/$d
       cd ..
     end
     popd
 'EOF'
     csh makeAltWiggle.csh >&! makeAltWiggle.log &
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz17way/cons/run.cons.groups
 cat > loadAltWiggle.csh << 'EOF'
     set b = `pwd`
     foreach d (mammal* placental* vertebrate*)
         echo $d
         cd $d
         set table = `echo $d | perl -wpe 's/(.*)/phastCons\u$1/;s/-(.*)/\u$1/'`
         echo $table
         ln -s `pwd`/$table.wib /gbdb/hg17/multiz17way
         hgLoadWiggle -pathPrefix=/gbdb/hg17/multiz17way hg17 $table $table.wig
         cd ..
     end
 'EOF'
     csh loadAltWiggle.csh >&! loadAltWiggle.log &
 
     # Create parasol batch for just chr7 (for test purposes) and run it
     pushd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
     ls -1 ss/chr7/chr*.ss | sed 's/.ss$//' > \
         /cluster/data/hg17/bed/multiz17way/cons/run.cons.groups/in.list
     popd
 
     gensub2 in.list single template jobList
     para create jobList
         # 80 jobs
     para try
     para check
     para push
     # 24 minutes
 
     # Downloads  (2006-02-22 kate)
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz17way
     mkdir mafDownloads
     cd mafDownloads
     # upstream mafs 
 cat > mafFrags.csh << 'EOF'
     date
     foreach i (1000 2000 5000)
         echo "making upstream$i.maf"
         nice featureBits hg17 refGene:upstream:$i -fa=/dev/null -bed=up.bad
         awk -F '\t' '{printf("%s\t%s\t%s\t%s\t%s\t%s\n", $1, $2, $3, substr($4, 0, 9), 0, $5)}' up.bad > up.bed
         rm up.bad
         nice mafFrags hg17 multiz17way up.bed upstream$i.maf \
                 -orgs=../species.lst
         nice gzip upstream$i.maf
         rm up.bed
     end
     date
 'EOF'
     time csh mafFrags.csh >&! mafFrags.log &
         # ~1 hour
 
     ssh kkstore02
     cd cluster/data/hg17/bed/multiz17way/mafDownloads
 cat > downloads.csh << 'EOF'
     date
     foreach f (../maf/chr*.maf)
 	set c = $f:t:r
         echo $c
 	nice gzip -c $f > $c.maf.gz
     end
     md5sum *.gz > md5sum.txt
     date
 'EOF'
     time csh downloads.csh >&! downloads.log
         # ~2 hours
         # GOT HERE
 
     ssh hgwdev
     set dir = /usr/local/apache/htdocs/goldenPath/hg17/multiz17way
     mkdir $dir
     ln -s /cluster/data/hg17/bed/multiz17way/mafDownloads/{*.gz,md5sum.txt} $dir
     cp /usr/local/apache/htdocs/goldenPath/mm7/multiz17way/README.txt $dir
     # edit README
 
 
 # PHASTCONS SCORES DOWNLOADABLES FOR 17WAY (2006-03-20 kate)
 
     ssh kkstore02
     cd /cluster/data/hg17/bed/multiz17way 
     mkdir phastConsDownloads
     cd phastConsDownloads
 cat > downloads.csh << 'EOF'
     date
     cd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/pp
     foreach chr (`awk '{print $1}' /cluster/data/hg17/chrom.sizes`)
       echo $chr
       cat `ls -1 $chr/$chr.*.pp | sort -t\. -k2,2n` \
         | nice gzip -c \
             > /cluster/data/hg17/bed/multiz17way/phastConsDownloads/$chr.gz
     end
     date
 'EOF'
     csh downloads.csh >&! downloads.log &
 
         # ~20 minutes
     # << happy emacs
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz17way/phastConsDownloads
     md5sum *.gz > md5sum.txt
     set dir = /usr/local/apache/htdocs/goldenPath/hg17/phastCons17way
     mkdir $dir
     ln -s /cluster/data/hg17/bed/multiz17way/phastConsDownloads/{*.gz,md5sum.txt} $dir
     cp /usr/local/apache/htdocs/goldenPath/hg17/phastCons/README.txt $dir
     # edit 
 
 
 # UPDATE MONKEY DOWNLOADS (2006-01-12 kate)
 # EXTRACT AXT'S AND MAF'S FROM THE RheMac1 NET
 # The chr1 was hugely oversized -- the other's were OK, but
 # axt's were numbered oddly.
     ssh kkstore2
     cd /cluster/data/hg17/bed/blastz.rheMac1/axtChain
     gunzip -c  hg17.rheMac1.net.gz | netSplit stdin humanNet
     gunzip -c hg17.rheMac1.all.chain.gz | chainSplit chain stdin
     mkdir ../axtNet.new ../mafNet.new
 cat > makeMaf.csh << 'EOF'
     foreach f (humanNet/chr*.net)
         set c = $f:t:r
         echo "axtNet on $c"
         netToAxt humanNet/$c.net chain/$c.chain /cluster/data/hg17/nib /cluster/data/rheMac1/rheMac1.2bit stdout | axtSort stdin ../axtNet.new/$c.axt
         axtToMaf ../axtNet.new/$c.axt \
             /cluster/data/hg17/chrom.sizes /cluster/data/rheMac1/chrom.sizes \
             ../mafNet.new/$c.maf -tPrefix=hg17. -qPrefix=rheMac1.
     end
     cp -rp ../mafNet.new /san/sanvol1/scratch/hg17/mafNet/rheMac1.new
 'EOF'
     csh makeMaf.csh >&! makeMaf.log &
     tail -100f makeMaf.log
     pushd /san/sanvol1/scratch/hg17/mafNet
     rm -fr rheMac1
     mv rheMac1.new rheMac1
     popd
 
     rm -fr axtNet
     mv axtNet.new axtNet
     cd axtNet
     nice gzip *.axt
     md5sum *.gz > md5sum.txt
     # cleanup
     cd ..
     rm -fr chain humanNet
 
     ssh hgwdev
     ln -s /cluster/data/hg17/bed/blastz.rheMac1/axtNet \
          /usr/local/apache/htdocs/goldenPath/rheMac1/axtNet
     # Request push to downloads server
 
 
 # UPDATE OPOSSUM DOWNLOADS (2006-01-17 kate)
 # Fix overlaps
 
     ssh kkstore2
     cd /cluster/data/hg17/bed/blastz.monDom1
     mv axtNet axtNet.old
     mv mafNet mafNet.old
     mkdir axtNet mafNet
     cd axtChain/chain
     nice gunzip *.gz
     cd ..
     nice gunzip -c  human.net.gz | netSplit stdin humanNet
 cat > makeMaf.csh << 'EOF'
     foreach f (humanNet/chr*.net)
         set c = $f:t:r
         echo "axtNet on $c"
         netToAxt humanNet/$c.net chain/$c.chain /cluster/data/hg17/nib /cluster/data/monDom1/monDom1.2bit stdout | axtSort stdin ../axtNet/$c.axt
         axtToMaf ../axtNet/$c.axt \
             /cluster/data/hg17/chrom.sizes /cluster/data/monDom1/chrom.sizes \
             ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=monDom1.
     end
     cp -rp ../mafNet /san/sanvol1/scratch/hg17/mafNet/monDom1.new
 'EOF'
     csh makeMaf.csh >&! makeMaf.log &
     tail -100f makeMaf.log
     pushd /san/sanvol1/scratch/hg17/mafNet
     rm -fr monDom1
     mv monDom1.new monDom1
     popd
 
     rm -fr axtNet
     mv axtNet.new axtNet
     cd axtNet
     nice gzip *.axt
     md5sum *.gz > md5sum.txt
 
     # cleanup
     cd ..
     rm -fr chain humanNet
 
     ssh hgwdev
     ln -s /cluster/data/hg17/bed/blastz.monDom1/axtNet \
          /usr/local/apache/htdocs/goldenPath/monDom1/axtNet
     # Request push to downloads server
 
 
 # UPDATE COW DOWNLOADS (2006-01-17 kate)
 # Fix overlaps
 
     ssh kkstore2
     cd /cluster/data/bosTau1/bed/zb.hg17
     mv axtNet axtNet.old
     mv mafNet mafNet.old
     mkdir axtNet mafNet
 cat > makeMaf.csh << 'EOF'
     foreach f (net/chr*.net)
         set c = $f:t:r
         echo "axtNet on $c"
         netToAxt net/$c.net chain/$c.chain /cluster/data/hg17/nib /cluster/data/bosTau1/bosTau1.2bit stdout | axtSort stdin ../axtNet/$c.axt
         axtToMaf ../axtNet/$c.axt \
             /cluster/data/hg17/chrom.sizes /cluster/data/bosTau1/chrom.sizes \
             ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=bosTau1.
     end
 'EOF'
     csh makeMaf.csh >&! makeMaf.log &
     tail -100f makeMaf.log
 
     cd axtNet
     nice gzip *.axt
     md5sum *.gz > md5sum.txt
 
     ssh hgwdev
     ln -s /cluster/data/hg17/bed/blastz.bosTau1/axtNet \
          /usr/local/apache/htdocs/goldenPath/bosTau1/axtNet
     # Request push to downloads server
 
 
 ##### UPDATE hg17 knownToVisiGene (2006-01-21 galt)
 # Create table that maps between known genes and visiGene database 
     # mapping to other species such as mouse, zebrafish, frog
     # requires visiGene probe track vgImageProbes be created first
     knownToVisiGene hg17 -fromProbePsl=vgImageProbes
 
 
 ##### UPDATE hg17 mmBlastTab (2006-01-22 galt)
 # Make the protein seqs from mm7.knownGenePep
 cd /cluster/data/hg17/bed/geneSorter/blastp
 mkdir mm7
 cd mm7
 pepPredToFa mm7 knownGenePep known.faa
 #       You may need to build this binary in src/hg/near/pepPredToFa
 /cluster/bluearc/blast229/formatdb -i known.faa -t known -n known
 mkdir -p /cluster/panasas/home/store/mm7/blastp/
 cp known.* /cluster/panasas/home/store/mm7/blastp/
 
 # Make parasol run directory
 ssh kk
 cd /cluster/data/hg17/bed/geneSorter/blastp/mm7
 mkdir run
 cd run
 mkdir out
 
 # Make blast script 
 # NOTE!! left off " b 1" from the end of the script because
 #  we wanted to be able to get the near-best, not just the best one.
 cat  << '_EOF_' > blastSome
 #!/bin/sh
 BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
 -p blastp -d /cluster/panasas/home/store/mm7/blastp/known \
 -i $1 -o $2 -e 0.001 -m 8
 '_EOF_'
     # << keep emacs happy
 chmod a+x blastSome
 
 # Make gensub2 file
 cat << '_EOF_' > gsub
 #LOOP
 blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
 #ENDLOOP
 '_EOF_'
     # << keep emacs happy
 
 # Create parasol batch
 #       this echo trick is used because otherwise the command line is
 #       too long and you can not do a simple ls
 echo ../../split/*.fa | wordLine stdin > split.lst
 gensub2 split.lst single gsub jobList
 para create jobList
 para try
 para check
 para push ... etc ...
 
 Completed: 7735 of 7735 jobs
 CPU time in finished jobs:      97096s    1618.26m    26.97h    1.12d  0.003 y
 IO & Wait Time:                564656s    9410.94m   156.85h    6.54d  0.018 y
 Average job time:                  86s       1.43m     0.02h    0.00d
 Longest running job:                0s       0.00m     0.00h    0.00d
 Longest finished job:             240s       4.00m     0.07h    0.00d
 Submission to last job:          1272s      21.20m     0.35h    0.01d
 
 
 # Load into database.
 ssh hgwdev
 cd /cluster/data/hg17/bed/geneSorter/blastp/mm7/run/out
 hgLoadBlastTab hg17 mmBlastTab -maxPer=1 *.tab
 Scanning through 7735 files
 Loading database with 33306 rows
 
 # changed mm6 to mm7 in src/hg/hgGene/hgGeneData/Human/hg17/otherOrgs.ra
 # and checked it in.
 
 # hgLoadBlastTab hg17 mmBlastTabTopN -maxPer=250 *.tab
 #  (not done, this was only used for research)
 
 # hgLoadBlastTab hg17 mmBlastNearBest -topPercent=5 *.tab > hgMmNearBest.stats
 #  (this will be the new way to go)
 Reading seq lengths from hg17.knownGenePep
 Finding max gene combined-coverage scores in 7735 files
 Scanning through 7735 files
 Loading database with 51520 rows
 
 
 ##########################################################################
 # MYTOUCH FIX - jen - 2006-01-24
   sudo mytouch hg17 gencodeGeneClassJun05 0508301200.00
   note - gencodeGeneClassJun05 table on dev only
   sudo mytouch hg17 knownGeneLink 0506050000.00
   sudo mytouch hg17 ensGtp 0505241200.00
   sudo mytouch hg17 ccdsInfo 0505241200.00
 
 
 ##########################################################################
 # BLASTZ OPOSSUM monDom2 (WORKING - 2006-01-23 - Hiram)
 
     ssh kk
     #	running out of disk space on store5:
     [hiram@kk /cluster/data/hg17/bed] df -h .
     #Filesystem            Size  Used Avail Use% Mounted on
     #			   1.5T  1.3T   79G  95% /cluster/store5
     #	So, keep this elsewhere, and symlink it:
     cd /cluster/data/hg17/bed
     ln -s /cluster/store9/hg17/bed/blastzMonDom2.2006-01-23 \
 	./blastzMonDom2.2006-01-23
     ln -s blastzMonDom2.2006-01-23 blastz.monDom2
 
     cd /cluster/data/hg17/bed/blastzMonDom2.2006-01-23
 
     cat << '_EOF_' > DEF
 # human vs. opossum
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/parasol/bin
 
 BLASTZ=blastz.v7
 
 # Specific settings for chicken (per Webb email to Brian Raney)
 BLASTZ_H=2000
 BLASTZ_Y=3400
 BLASTZ_L=10000
 BLASTZ_K=2200
 BLASTZ_Q=/cluster/data/blastz/HoxD55.q
 BLASTZ_ABRIDGE_REPEATS=0
 
 # TARGET: Human (hg17)
 SEQ1_DIR=/scratch/hg/hg17/bothMaskedNibs
 SEQ1_LEN=/cluster/data/hg17/chrom.sizes
 SEQ1_IN_CONTIGS=0
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY: Opossum monDom2
 SEQ2_DIR=/scratch/hg/monDom2/monDom2.2bit
 SEQ2_LEN=/scratch/hg/monDom2/chrom.sizes
 SEQ2_IN_CONTIGS=0
 SEQ2_CHUNK=30000000
 SEQ2_LIMIT=100
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/blastzMonDom2.2006-01-23
 TMPDIR=/scratch/tmp
 '_EOF_'
     #	<< happy emacs
 
     cd /cluster/data/hg17/bed/blastzMonDom2.2006-01-23
     time /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=kk -chainMinScore=5000 -chainLinearGap=loose \
 	`pwd`/DEF > blastz.out 2>&1 &
 
     #	real    1122m44.191s
     #	failed during the load of chr19
     #	hgLoadChain hg17 chr19_chainMonDom2 chr19.chain
     #	Out of memory needMem - request size 56 bytes
 
     #	So, go to kolossus:
     ssh kolossus
     #	There isn't any hg17 db here yet, get it established with a
     #	chromInfo and a 2bit sequence:
     hgsql -e "create database hg17;" mysql
     cd /cluster/data/hg17
     twoBitInfo hg17.2bit stdout |
         awk '{printf "%s\t%s\t/gbdb/hg17/hg17.2bit\n", $1,$2}' \
 		> chromInfo.kolossus.tab
     hgsql hg17 < $HOME/kent/src/hg/lib/chromInfo.sql
     hgsql hg17 \
 -e 'load data local infile "chromInfo.kolossus.tab" into table chromInfo;'
     #	it appears /gbdb/hg17 already exists
     ln -s /cluster/data/hg17/hg17.2bit /gbdb/hg17/hg17.2bit
     #	now, loading only chr19:
     cd /cluster/data/hg17/bed/blastzMonDom2.2006-01-23/axtChain
     hgLoadChain hg17 chr19_chainMonDom2 chain/chr19.chain
     #	real    33m31.689s
     #	while that is running, back on hgwdev, get the other chains loaded
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastzMonDom2.2006-01-23/axtChain
     cp loadUp.csh loadUp.noChr19.csh
     #	change the foreach line to eliminate the chr19.chain:
     diff loadUp.csh loadUp.noChr19.csh
     < foreach f (*.chain)
     ---
     > foreach f (`ls *.chain | grep -v chr19.chain`)
     #	And then run that script
     time ./loadUp.noChr19.csh > load.noChr19.out 2>&1
     #	real    76m8.757s
     
     #	When the kolossus load finishes, email to push-request and ask
     #	for the two tables to be pushed from kolossus to hgwdev:
     #	chr19_chainMonDom2
     #	chr19_chainMonDom2Link
 
     #	then, continuing:
     time /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-continue=download -bigClusterHub=pk -chainMinScore=5000 \
 	-chainLinearGap=loose `pwd`/DEF > download.out 2>&1 &
     #	real    2m42.505s
 
     #	now, back on kolossus to run a featurebits
     time featureBits hg17 chainMonDom2Link >fb.hg17.chainMonDom2Link 2>&1
     #	355119482 bases of 2866216770 (12.390%) in intersection
     featureBits hg17 chainMonDom1Link
     #	456069062 bases of 2866216770 (15.912%) in intersection
 
     #	Then, to swap the results:
     ssh kk
     cd /cluster/data/hg17/bed/blastz.monDom2
     time /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-swap -bigClusterHub=pk -chainMinScore=5000 \
 	-chainLinearGap=loose `pwd`/DEF > swap.out 2>&1 &
     #	running 2006-01-30 11:25
     #	real    47m27.082s
     #	failed during the load - as with the Hg18 experiment, something
     #	is really huge about these results.
 
 #########################################################################
 # BUILD MAF ANNOTATION FOR MULTIZ17WAY  (kate 2006-02-16)
 # Redo to fix overlaps (2006-04-09 kate)
 # rebuild frames to get bug fix, using 1-pass maf methodology (2006-06-09 markd)
 
     ssh kkstore01
     cd /cluster/data/rheMac2
     twoBitInfo -nBed rheMac2.2bit rheMac2.N.bed
 
     ssh kkstore02
     cd /cluster/data/hg17/bed/multiz17way
     mkdir anno 
     cd anno
     mkdir maf run
     cd run
     rm sizes nBeds
     foreach i (`cat /cluster/data/hg17/bed/multiz17way/species.lst`)
         ln -s  /cluster/data/$i/chrom.sizes $i.len
         ln -s  /cluster/data/$i/$i.N.bed $i.bed
         echo $i.bed  >> nBeds
         echo $i.len  >> sizes
     end
 
     rm jobs.csh
     echo date > jobs.csh
     foreach i (../../maf/*.maf)
         echo nice mafAddIRows -nBeds=nBeds -sizes=sizes $i /cluster/data/hg17/hg17.2bit ../maf/`basename $i` >> jobs.csh
         echo "echo $i" >> jobs.csh
     end 
     echo date >> jobs.csh
 
     # do smaller jobs first
     tac jobs.csh > jobsRev.csh
     mv jobsRev.csh jobs.csh
     
     csh jobs.csh >&! jobs.log &
         # 1.5 hrs.
         # 9 hours for redo -- something wrong ?
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz17way/anno/maf
     mkdir -p /gbdb/hg17/multiz17way/anno/maf
     ln -s /cluster/data/hg17/bed/multiz17way/anno/maf/*.maf \
         /gbdb/hg17/multiz17way/anno/maf
 cat > loadMaf.csh << 'EOF'
     date
     hgLoadMaf -pathPrefix=/gbdb/hg17/multiz17way/anno/maf \
                 hg17 multiz17way
     date
 'EOF'
     csh loadMaf.csh >&! loadMaf.log &
 
     # load summary table on kolossus, as it crashes on hgwdev
     ssh kolossus
     cd /cluster/data/hg17/bed/multiz17way/anno/maf
     cat *.maf | \
         nice hgLoadMafSummary hg17 -minSize=30000 -mergeGap=1500 \
             -maxSize=200000  multiz17waySummary stdin
 
         # Created 3212623 summary blocks from 114139253 components and 17522217 mafs from stdin
     # request push to hgwdev
 
     # Dropped unused indexes (2006-05-09 kate)
     # NOTE: this is not required in the future, as the loader
     # has been fixed to not generate these indexes
     hgsql hg17 -e "alter table multiz17waySummary drop index chrom_2"
     hgsql hg17 -e "alter table multiz17waySummary drop index chrom_3"
 
     ssh kkstore02
     cd /cluster/data/hg17/bed/multiz17way
     set sanDir = /san/sanvol1/scratch/hg17/multiz17way/frames
     mkdir -p $sanDir/maf
     cp -rp maf/* $sanDir/maf
     mkdir frames
     cd frames
     cp /cluster/data/mm7/bed/multiz17wayFrames/mkMafFrames .
     cp /cluster/data/mm7/bed/multiz17wayFrames/Makefile .
     #edit Makefile to correct species names and set and sanDir
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz17way/frames
     make getGenes >&! getGenes.log &
         # ~1 minute
     make getFrames >&! getFrames.log &
         # ~2 hours
     # NOTE: if jobs get hung up (e.g. running for hours, when
     #   they should run for minutes, do 'para stop' so that
     #   the 'para make' can restart the job
     make loadDb >&! loadDb.log &
 
     ###
     # rebuild frames to get bug fix, using 1-pass maf methodology
     # (2006-06-09 markd)
     ssh kkstore02
     cd /cluster/data/hg17/bed/multiz17way/frames
     mv mafFrames/ mafFrames.old
     nice tcsh # easy way to get process niced
     (cat  ../maf/*.maf | genePredToMafFrames hg17 stdin stdout bosTau2 genes/bosTau2.gp.gz canFam2 genes/canFam2.gp.gz danRer3 genes/danRer3.gp.gz fr1 genes/fr1.gp.gz galGal2 genes/galGal2.gp.gz hg17 genes/hg17.gp.gz mm7 genes/mm7.gp.gz oryCun1 genes/oryCun1.gp.gz panTro1 genes/panTro1.gp.gz rheMac2 genes/rheMac2.gp.gz rn3 genes/rn3.gp.gz xenTro1 genes/xenTro1.gp.gz |  gzip >multiz17way.mafFrames.gz)>&log&
     ssh hgwdev
     cd /cluster/data/hg17/bed/multiz17way/frames
 
     hgLoadMafFrames hg17 multiz17wayFrames multiz17way.mafFrames.gz >&log&
 
 
 # EXTRACT LINEAGE-SPECIFIC REPEATS FOR RAT (DONE 2/8/06 angie)
     ssh kolossus
     mkdir /cluster/data/hg17/rmsk
     cd /cluster/data/hg17/rmsk
     ln -s ../*/chr*.fa.out .
     # Run Arian's DateRepsinRMoutput.pl to add extra columns telling 
     # whether repeats in -query are also expected in -comp species.  
     # Even though we already have the human-mouse linSpecReps,
     # extractLinSpecReps requires two columns of DateRepsinRMoutput.pl
     # additions.  So add mouse, then ignore it.  
     # Rat in extra column 1, Mouse in extra column 2
     foreach outfl ( *.out )
         echo "$outfl"
         /cluster/bluearc/RepeatMasker/DateRepeats \
           ${outfl} -query human -comp rat -comp mouse
     end
     # Now extract rat (extra column 1), ignore mouse.
     cd ..
     mkdir linSpecRep.notInRat
     foreach f (rmsk/*.out_rat*_mus-musculus)
         set base = $f:t:r:r
         echo $base.out.spec
         /cluster/bin/scripts/extractLinSpecReps 1 $f > \
                         linSpecRep.notInRat/$base.out.spec
     end
     # Distribute and clean up.
     rsync -av linSpecRep.notInRat /san/sanvol1/scratch/hg17/
     rm -r rmsk
 
 
 # BLASTZ/CHAIN/NET RN4 (DONE 2/10/06 angie)
     ssh kkstore01
     mkdir /cluster/data/hg17/bed/blastz.rn4.2006-02-08
     cd /cluster/data/hg17/bed/blastz.rn4.2006-02-08
     cat << '_EOF_' > DEF
 # human vs. rat
 
 BLASTZ_ABRIDGE_REPEATS=1
 
 # TARGET: Human
 SEQ1_DIR=/san/sanvol1/scratch/hg17/nib
 SEQ1_SMSK=/san/sanvol1/scratch/hg17/linSpecRep.notInRat
 SEQ1_LEN=/cluster/data/hg17/chrom.sizes
 SEQ1_CHUNK=10000000
 SEQ1_LAP=10000
 
 # QUERY: Rat
 SEQ2_DIR=/san/sanvol1/scratch/rn4/nib
 SEQ2_SMSK=/san/sanvol1/scratch/rn4/linSpecRep.notInHuman
 SEQ2_LEN=/cluster/data/rn4/chrom.sizes
 SEQ2_CHUNK=10000000
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/blastz.rn4.2006-02-08
 '_EOF_'
     # << for emacs
     doBlastzChainNet.pl DEF -chainLinearGap medium \
       -bigClusterHub pk -smallClusterHub pk -workhorse pk \
       -blastzOutRoot /san/sanvol1/scratch/blastzHg17Rn4Out >& do.log &
     tail -f do.log
     rm -f /cluster/data/hg17/bed/blastz.rn4
     ln -s blastz.rn4.2006-02-08 /cluster/data/hg17/bed/blastz.rn4
 
 # UPDATE WGRNA TRACK (DONE, 2006-02-15, Fan)
 
   ssh hgwdev
   cd /cluster/data/hg17/bed
 
   mkdir wgRna-2006-02-15
   cd wgRna-2006-02-15
 
 # Received the data file, wg_track_hg17_feb2006_completed.txt, from Michel Weber's email
 # (Michel.Weber@ibcg.biotoul.fr)
 # and place it under cd /cluster/data/hg17/bed/wgRna-2006-02-15.
 
   cp -p wg_track_hg17_feb2006_completed.txt wgRna.tab
   hgLoadBed -sqlTable=/cluster/home/fanhsu/src/hg/lib/wgRna.sql hg17 wgRna wgRna.tab
 
 # Compared to previous data, 2 records deleted, 27 records added.
 
 
 ########################################################################
 #  BLASTZ Opossum monDom4 (DONE - 2006-02-21 - 2006-02-26 - Hiram)
     ssh pk
     mkdir /cluster/data/hg17/bed/blastzMonDom4.2006-02-21
     cd /cluster/data/hg17/bed
     ln -s blastzMonDom4.2006-02-21 blastz.monDom4
     cd blastzMonDom4.2006-02-21
 
     cat << '_EOF_' > DEF
 # human vs. opossum
 export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/x86_64:/parasol/bin
 
 BLASTZ=blastz.v7.x86_64
 
 # settings for more distant organism alignments
 BLASTZ_H=2000
 BLASTZ_Y=3400
 BLASTZ_L=10000
 BLASTZ_K=2200
 BLASTZ_M=50
 BLASTZ_Q=/cluster/data/blastz/HoxD55.q
 
 # TARGET: Human (hg17)
 SEQ1_DIR=/scratch/hg/hg17/bothMaskedNibs
 SEQ1_LEN=/cluster/data/hg17/chrom.sizes
 SEQ1_CHUNK=50000000
 SEQ1_LAP=10000
 
 # QUERY: Opossum monDom4
 SEQ2_DIR=/san/sanvol1/scratch/monDom4/monDom4.2bit
 SEQ2_LEN=/san/sanvol1/scratch/monDom4/chrom.sizes
 SEQ2_CHUNK=30000000
 SEQ2_LAP=0
 
 BASE=/cluster/data/hg17/bed/blastzMonDom4.2006-02-21
 TMPDIR=/scratch/tmp
 '_EOF_'
     #  << happy emacs
 
     time /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk -chainMinScore=5000 -chainLinearGap=loose \
 	-continue=chainMerge `pwd`/DEF > chainMerge.out 2>&1 &
     time /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
 	-bigClusterHub=pk -chainMinScore=5000 -chainLinearGap=loose \
 	-swap `pwd`/DEF > swap.out 2>&1 &
 
     time nice -n +19 featureBits hg17 chainMonDom4Link \
 	> fb.hg17.chainMonDom4Link 2>&1
     time nice -n +19 featureBits monDom4 chainHg17Link \
 	> fb.monDom4.chainHg17Link 2>&1
 
 ########################################################################
 ##  Measuring MonDom4 chain pile ups (DONE - 2006-02-26 - Hiram)
     ssh kkstore02
     cd /cluster/data/hg17/bed/blastz.monDom4/axtChain
     #	extract coordinates on the target genome of the chains
     zcat hg17.monDom4.all.chain.gz | grep "^chain " \
     | awk '{printf "%s\t%s\t%s\t%s\t%s\n", $3, $6, $7, $5, $2}' \
         | gzip -c > target.chain.bed.gz
     # turn that into a wiggle graph with bedItemOverlapCount
     # use HGDB_CONF for read-only access to the hg17 DB in bedItemOverlapCount
     #	it wants to read chromInfo ...
     export HGDB_CONF=~/.hg.conf.read-only
     #	ignore chains longer than 1,000,000
     zcat target.chain.bed.gz | awk '$3-$2<1000000 {print}' \
 	| sort -k1,1 -k2,2n \
 	    | bedItemOverlapCount hg17 stdin \
 		| wigEncode stdin monDom4PileUps.wig monDom4PileUps.wib
     #	Do the same for the query coordinates to find out where these
     #	chains are coming from
     zcat hg17.monDom4.all.chain.gz | grep "^chain " \
     | awk '{printf "%s\t%s\t%s\t%s\t%s\n", $8, $11, $12, $10, $2}' \
         | gzip -c > query.chain.bed.gz
     zcat query.chain.bed.gz | awk '$3-$2<1000000 {print}' \
     | sort -k1,1 -k2,2n \
         | bedItemOverlapCount monDom4 stdin \
             | wigEncode stdin hg17PileUps.wig hg17PileUps.wib
     #	load those wiggles
     ssh hgwdev
     cd /cluster/data/hg17/bed/blastz.monDom4/axtChain
     ln -s `pwd`/monDom4PileUps.wib /gbdb/hg17/wib
     ln -s `pwd`/hg17PileUps.wib /gbdb/monDom4/wib
 
     hgLoadWiggle -verbose=2 hg17 monDom4PileUp monDom4PileUps.wig
     hgLoadWiggle -verbose=2 monDom4 hg17PileUps hg17PileUps.wig
     #	add wiggle track type entries to the respective trackDb.ra files
 
 
 
 # UPDATE hg17 knownToVisiGene (2006-03-07 galt)
 # Create table that maps between known genes and visiGene database 
     # mapping to other species such as mouse, zebrafish, frog
     # requires visiGene probe track vgImageProbes be created first
     knownToVisiGene hg17 -fromProbePsl=vgImageProbes
 
 ############################################################################
 # Add Landmark track (2006-03-08 giardine)
 # Note: This track is for regulatory regions and other landmarks that are not
     #included in other tracks.  It is being gathered from the locus experts
     #that are contributing data to the Human Mutation track.  This should
     #be helpful in understanding the data in the mutation track.
 #table definitions for autoSql
 autoSql landmark.as landmark -dbLink
 #change index on bin to normal index not primary key
 #move bin in struct so works as bed 4+
 #copy autoSql files to hg/lib and hg/inc (add .o file to makefile)
 #cat together landmark files from sources in landmark.bed then sort
 grep "^chr" landmark.bed | sort -k1,1 -k2,2n > sortedLandmark.bed
 #loading
 hgsql hg17 < landmark.sql
 hgLoadBed hg17 landmark sortedLandmark.bed -noSort -oldTable -tab
 #add to trackDb.ra file (human hg17 level)
 
 #changed landmark track to provide links and attributes in prep for ORegAnno
 #data.  Got set of test data by grabbing their .gff file used for custom
 #tracks and converting to bed, then to landmarks format.
 cd humPhen/landmarkData/June06/
 #convert data to new formats then
 cat newLandmark.txt landmarkORA.txt  > allLandmarks.txt
 grep "^chr" allLandmarks.txt | sort -k1,1 -k2,2n > sortedAllLandmark.txt
 #start new tables
 cd humPhen/kent/src/hg/lib/
 autoSql landmark.as landmark -dbLink
 #move bin in .h file to end of structure, to make load work
 mv landmark.h ../inc/landmark.h
 #change primary key to indexes where not unique, add index on landmarkId
 #limit name, landmarkType, raKey size to 64
 hgsql -e "drop table landmark;" hg17
 hgsql hg17 < landmark.sql
 cd ~giardine/humPhen/landmarkData/June06/
 hgLoadBed hg17 landmark sortedAllLandmark.txt -noSort -oldTable -tab
 hgsql hg17
         load data local infile "landmarkAttrORA.txt" into table landmarkAttr;
         load data local infile "landmarkAttrLinkORA.txt" into table landmarkAttrLink;
         load data local infile "landmarkAttrCat.txt" into table landmarkAttrCat;
 cd ../../kent/src/
 make clean
 make libs
 cd hg
 make cgi
 cd makeDb/trackDb
 make DBS=hg17 update
 #test in hgwdev-giardine
 
 #redo landmarks, moving categories out of database
 convertORAformat < ORegAnnoBed
 #start new tables
 cd humPhen/kent/src/hg/lib/
 autoSql landmark.as landmark -dbLink
 #move bin in .h file to end of structure, to make load work
 mv landmark.h ../inc/landmark.h
 #change primary key to indexes, add primary key on landmarkId
 #limit name, landmarkType, raKey size to 64
 #only need to reload attributes rest of data & tables same
 hgsql -e "drop table landmarkAttr;" hg17
 hgsql -e "drop table landmarkAttrCat;" hg17
 cd ../../../../landmarkData/June06/
 hgsql hg17
         #cut and paste in create table landmarkAttr
         load data local infile "landmarkAttrORA.txt" into table landmarkAttr;
 #Records: 2028  Deleted: 0  Skipped: 0  Warnings: 8 ???
 cd ../../kent/src/
 make clean
 make libs
 cd hg
 make cgi
 cd makeDb/trackDb
 make DBS=hg17 update
 #test in hgwdev-giardine
 
 
 ############################################################################
 # hg15 -> hg17 LIFTOVER CHAINS (STARTED 3/9/06, DONE 3/10/06 Fan)
     # I used a size of 10kb instead of 3kb for the split (blat query) sizes in
     # hg17.  This had a huge affect on the amount of hits in the blat, which
     # then had a huge effect on the amount of chains.  I should also mention 
     # that hg17 chromosomes chr1 and chr2 were split further 
     # into more than a single query file.  This helped a LOT in avoiding 
     # cluster hippos classically associated with those chroms.
     
     ######## LIFTOVER PREPARATION
     # Split up hg17
     ssh pk
     cd /san/sanVol1/scratch/hg17
     mkdir -p liftSplits/{split,lift}
     bash
     for fa in /cluster/data/hg17/?{,?,*hap*}/*.fa; do
       c=`basename $fa .fa`
       echo $c
       faSplit -lift=liftSplits/lift/${c}.lft size $fa -oneFile 10000 liftSplits/split/$c
     done
     mkdir -p biggerSplits/split
     cd biggerSplits/
     ln -s ../liftSplits/lift
     cd split/
     ln -s ../../liftSplits/split/* .
     faSplit sequence chr1.fa 5 chr1_
     faSplit sequence chr2.fa 5 chr2_
     rm chr{1,2}.fa
 
     # Make some dirs
     cd /san/sanVol1/scratch
     mkdir -p hg15
 
     # Copy 11.ooc files to hg15 subdirectory.
     cp -p /cluster/store5/gs.16/build33/11.ooc hg15
 
     ## First, copy over scripts. (Already done before)
 
     # mkdir -p /san/sanVol1/scratch/fan
     # cp -p /san/sanVol1/scratch/fan/*.sh /san/sanVol1/scratch/fan
     # cp /san/sanVol1/scratch/andy/psl.header /san/sanVol1/scratch/fan
 
     ######## LIFTOVER BLATTING  
 
     # HG15
     ssh pk
     cd /cluster/data/hg15
 #    makeLoChain-align hg15 /scratch/hg/hg15/bothMaskedNibs hg17 \     
 
     makeLoChain-align hg15 /scratch/hg/hg15/chromTrfMixedNib hg17 \
 /san/sanVol1/scratch/hg17/biggerSplits/split
     cd bed
 
     mv blat.hg17.2006-03-09 /san/sanVol1/scratch/hg15 
     cd /san/sanVol1/scratch/hg15/blat.hg17.2006-03-09/run/
     sed 's/^blat/blat.sh/; s/\}.*$/}/' spec | awk '{print "/san/sanVol1/scratch/fan/" $0 " hg15ToHg17"}' > newspec
     para create newspec
     para try
     para push
 # Saw some failures, keep pushing again, they finally all finished.  
 # The problems were all from one node.  
 # Used "para remove machine ..." to remove that node from the cluster.
 # Completed: 2376 of 2376 jobs
 # CPU time in finished jobs:     626355s   10439.25m   173.99h    7.25d  0.020 y
 # IO & Wait Time:                 49512s     825.20m    13.75h    0.57d  0.002 y
 # Average job time:                 284s       4.74m     0.08h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:            3693s      61.55m     1.03h    0.04d
 # Submission to last job:          4165s      69.42m     1.16h    0.05d
 
     ######## LIFTOVER CHAINING
     # LIFTING
     ssh pk
     cd /san/sanVol1/scratch/fan
     cp mm7SplitLift.sh hg17SplitLift.sh
 
     # change andy to fan, mm7 to hg17, and chrX to chr2, and remove chrUn_random 
     vi hg17SplitLift.sh
 
     cat << 'EOF' > hg17ChainMergeSplit.sh
 #!/bin/bash
 cp -r chainRaw/ /scratch/fan/hg17Lifts
 pushd /scratch/fan/hg17Lifts
 mkdir chain
 /cluster/bin/x86_64/chainMergeSort chainRaw/*.chain | /cluster/bin/x86_64/chainSplit chain stdin
 cp -r chain `dirs +1`
 rm -rf chain chainRaw
 'EOF'
 
     chmod +x hg17ChainMergeSplit.sh
 
     # HG15
     cd /san/sanVol1/scratch/hg15/blat.hg17.2006-03-09/raw
     /san/sanVol1/scratch/fan/hg17SplitLift.sh
     cd ../    
     mkdir chainRun chainRaw
     cd chainRun
     cat > gsub << 'EOF'
 #LOOP
 /cluster/bin/x86_64/axtChain -verbose=0 -linearGap=medium -psl $(path1) /scratch/hg/hg15/chromTrfMixedNib  /san/sanVol1/scratch/hg17/nib {check out line+ ../chainRaw/$(root1).chain}
 #ENDLOOP
 'EOF'
     ls -1S ../psl/*.psl > in.lst
     gensub2 in.lst single gsub spec
     para create spec
     para push
     para time
 # Completed: 46 of 46 jobs
 # CPU time in finished jobs:       3546s      59.10m     0.98h    0.04d  0.000 y
 # IO & Wait Time:                   895s      14.92m     0.25h    0.01d  0.000 y
 # Average job time:                  97s       1.61m     0.03h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:             270s       4.50m     0.07h    0.00d
 # Submission to last job:           270s       4.50m     0.07h    0.00d
 
     ######### CHAINMERGE/NET/NETSUBSET
     ssh kolossus
     mkdir -p /scratch/fan/hg17Lifts
     cd /scratch/fan/hg17Lifts
 
     cp -r /san/sanvol1/scratch/hg15/blat.hg17.2006-03-09/chainRaw/ .
     mkdir chain
     /cluster/bin/x86_64/chainMergeSort chainRaw/* | /cluster/bin/x86_64/chainSplit chain stdin
 # about 30 minutes.
 
     cp -rp chain /san/sanvol1/scratch/hg15/blat.hg17.2006-03-09/
     rm -rf chain
     rm -rf chainRaw
 
     ssh pk
     cd /san/sanvol1/scratch/fan
     cat << 'EOF' > netOver.sh 
 #!/bin/bash
 
 chain=$1
 chrom=`basename $chain .chain`
 sizesHGOld=$2
 sizesHG17=/cluster/data/hg17/chrom.sizes
 chainDir=`dirname $chain`
 blatDir=`dirname $chainDir`
 net=${blatDir}/net/${chrom}.net
 over=${blatDir}/over/${chrom}.over
 
 mkdir -p ${blatDir}/{over,net}
 /cluster/bin/x86_64/chainNet $chain $sizesHGOld $sizesHG17 $net /dev/null
 /cluster/bin/x86_64/netChainSubset $net $chain $over
 'EOF'
 # << emacs
     chmod +x netOver.sh
 
     mkdir netRun
 
     cd netRun/
 
     find /san/sanVol1/scratch/hg15/blat.hg17.2006-03-09/chain -name "*.chain" \
      | awk '{print "/san/sanVol1/scratch/fan/netOver.sh " $1 " /cluster/data/hg15/chrom.sizes"}' >> spec
     para create spec
     para push
     para time
 # Completed: 44 of 44 jobs
 # CPU time in finished jobs:        427s       7.12m     0.12h    0.00d  0.000 y
 # IO & Wait Time:                   248s       4.13m     0.07h    0.00d  0.000 y
 # Average job time:                  15s       0.26m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              29s       0.48m     0.01h    0.00d
 # Submission to last job:            46s       0.77m     0.01h    0.00d
 
 # seems much faster than mm7.
 
     ########## FINISHING
     ssh hgwdev
 
     # HG15
     cd /san/sanvol1/scratch/hg15/blat.hg17.2006-03-09/over
     cat * >> ../hg15ToHg17.over.chain
     cd ../
     rm -rf psl/ net/ chain/ chainRaw/ over/
     cd ../
     cp -rp blat.hg17.2006-03-09/ /cluster/data/hg15/bed
 
     cd /cluster/data/hg15/bed
     ln -s blat.hg17.2006-03-09 blat.hg17
     ln -s `pwd`/blat.hg17/hg15ToHg17.over.chain liftOver/hg15ToHg17.over.chain
     ln -s `pwd`/liftOver/hg15ToHg17.over.chain /gbdb/hg15/liftOver/hg15ToHg17.over.chain
     mkdir -p /usr/local/apache/htdocs/goldenPath/hg15/liftOver
     cd /usr/local/apache/htdocs/goldenPath/hg15/liftOver
     cp /gbdb/hg15/liftOver/hg15ToHg17.over.chain .
     gzip hg15ToHg17.over.chain
     hgAddLiftOverChain hg15 hg17 /gbdb/hg15/liftOver/hg15ToHg17.over.chain
 
 # UPDATED hg17.knownToVisiGene (2006-03-14 galt)
 #  after making sure hg17.vgAllProbes was up to date (see makeVisiGene.doc)
 ssh hgwdev
 knownToVisiGene hg17 -fromProbePsl=vgAllProbes
 
 ########################################################################
 ### microRNA targets tracks  (DONE - 2006-03-17 - 2006-04-27 - Hiram)
 ### from: http://pictar.bio.nyu.edu/ Rajewsky Lab
 ### Nikolaus Rajewsky nr@scarbo.bio.nyu.edu
 ### Yi-Lu Wang ylw205@nyu.edu
 ### dg@thp.Uni-Koeln.DE
     ssh hgwdev
     mkdir /cluster/data/hg17/bed/picTar
     cd /cluster/data/hg17/bed/picTar
     wget --timestamping \
 	'http://pictar.bio.nyu.edu/ucsc/new_mammals_bed' -O newMammals.bed
 
     wget --timestamping \
 	'http://pictar.bio.nyu.edu/ucsc/new_mammals_chicken_bed' \
 	-O newMammalsChicken.bed
 
     grep -v "^track" newMammals.bed \
 	| hgLoadBed -strict hg17 picTarMiRNA4Way stdin
     #	Loaded 205263 elements of size 9
     grep -v "^track" newMammalsChicken.bed \
 	| hgLoadBed -strict hg17 picTarMiRNA5Way stdin
     #	Loaded 43081 elements of size 9
 
     nice -n +19 featureBits hg17 picTarMiRNA4Way
     #	608549 bases of 2866216770 (0.021%) in intersection
     nice -n +19 featureBits hg17 picTarMiRNA5Way
     #	109059 bases of 2866216770 (0.004%) in intersection
 
 ############################################################################
 # dbSNP BUILD 125 (Heather, March 2006)
 
 # Set up directory structure
 ssh kkstore02
 cd /cluster/data/dbSnp
 mkdir 125
 cd 125
 mkdir shared
 mkdir shared/data
 mkdir shared/schema
 mkdir organisms
 mkdir organisms/human_9606
 mkdir organisms/human_9606/rs_fasta
 mkdir organisms/human_9606/database
 mkdir organisms/human_9606/database/organism_data
 mkdir organisms/human_9606/database/organism_data/hg17
 mkdir organisms/human_9606/database/schema
 
 # Get data from NCBI
 
 # Shared data includes data dictionary, 
 # Shared data includes defined types such as validity, class, function, locType
 # Actually this is independent of hg17 build and should go in separate makeDoc
 cd shared/data
 ftp ftp.ncbi.nih.gov
 cd snp/database/organism_shared_data
 mget *.gz
 
 cd ../schema
 ftp ftp.ncbi.nih.gov
 cd snp/database/schema/shared_schema
 mget *.gz
 
 # using headers of fasta files for molType, class and observed
 cd ../organisms/human_9606/rs_fasta
 ftp ftp.ncbi.nih.gov
 cd snp/organisms/human_9606/rs_fasta
 mget *.gz
 
 cd ../database/organism_data/hg17
 ftp ftp.ncbi.nih.gov
 cd snp/organisms/human_9606/database/organism_data
 # ContigLoc table has coords, orientation, loc_type, and refNCBI allele
 get b125_SNPContigLoc_35_1.bcp.gz
 # ContigLocusId has function
 get b125_SNPContigLocusId_35_1.bcp.gz
 get b125_ContigInfo_35_1.bcp.gz
 # MapInfo has alignment weights
 get b125_SNPMapInfo_35_1.bcp.gz
 # SNP has validation status and heterozygosity
 get SNP.bcp.gz
 # done with FTP
 
 # rename
 mv b125_SNPContigLoc_35_1.bcp.gz ContigLoc.gz
 mv b125_SNPContigLocusId_35_1.bcp.gz ContigLocusId.gz
 mv b125_ContigInfo_35_1.bcp.gz ContigInfo.gz
 mv b125_SNPMapInfo_35_1.bcp.gz MapInfo.gz
 mv SNP.bcp.gz SNP.gz
 
 # edit table descriptions
 cd /cluster/data/dbSnp/125/organisms/human_9606/database/schema
 # get CREATE statements from human_9606_table.sql for our 5 tables
 # store in table.tmp
 # convert and rename tables
 sed -f 'mssqlToMysql.sed' table.tmp > table2.tmp
 rm table.tmp
 sed -f 'tableRename.sed' table2.tmp > table.sql
 rm table2.tmp
 
 # get header lines from rs_fasta
 cd /cluster/data/dbSnp/125/organisms/human_9606/rs_fasta
 /bin/csh gnl.csh
 
 # load on kkr5u00
 ssh kkr5u00
 hgsql -e mysql 'create database dbSnpHumanBuild125' 
 cd /cluster/data/dbSnp/125/organisms/human_9606/database/schema
 hgsql dbSnpHumanBuild125 < table.sql
 cd ../organism_data/hg17
 /bin/csh load.csh
 
 # note rowcount
 # ContigLoc     24135144 
 # SNP           10430754 
 # MapInfo       10271016 
 # ContigLocusId  9539145 
 
 # create working /scratch dir
 cd /scratch/snp
 mkdir 125
 cd 125
 mkdir human
 cd human
 
 # get hg17 ctgPos, load into dbSnpHumanBuild125, compare contig list between ctgPos and ContigInfo
 
 # get gnl files
 cp /cluster/data/dbSnp/125/organisms/human_9606/rs_fasta/*.gnl .
 
 # examine ContigInfo for group_term and edit pipeline.csh
 # use "ref_haplotype" 
 
 # filter ContigLoc into ContigLocFilter
 # this gets rid of alternate assemblies and poor quality alignments
 # uses ContigInfo and MapInfo (weight == 10 || weight == 3)
 # assumes all contigs are positively oriented
 # will abort if not true
 
 mysql> desc ContigLocFilter;
 #  +---------------+-------------+------+-----+---------+-------+
 #  | Field         | Type        | Null | Key | Default | Extra |
 #  +---------------+-------------+------+-----+---------+-------+
 #  | snp_id        | int(11)     | NO   |     |         |       |
 #  | ctg_id        | int(11)     | NO   |     |         |       |
 #  | chromName     | varchar(32) | NO   |     |         |       |
 #  | loc_type      | tinyint(4)  | NO   |     |         |       |
 #  | phys_pos_from | int(11)     | NO   |     |         |       |
 #  | phys_pos      | varchar(32) | YES  |     | NULL    |       |
 #  | orientation   | tinyint(4)  | NO   |     |         |       |
 #  | allele        | blob        | YES  |     | NULL    |       |
 #  +---------------+-------------+------+-----+---------+-------+
  
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpContigLocFilter dbSnpHumanBuild125 ref_haplotype
 # note rowcount
 # ContigLocFilter  10113426
 # how many are positive strand? hopefully 90%
 mysql> select count(*) from ContigLocFilter where orientation = 0;
 # 9161012
 
 # filter ContigLocusId into ContigLocusIdFilter
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpContigLocusIdFilter dbSnpHumanBuild125 ref_haplotype
 # note rowcount 
 # ContigLocusIdFilter  5352542
 
 # condense ContigLocusIdFilter into ContigLocusIdCondense (one SNP can have multiple functions)
 # assumes SNPs are in numerical order
 # will errAbort if not true
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpContigLocusIdCondense dbSnpHumanBuild125 
 # note rowcount
 # expect about 50% for human
 # ContigLocusIdCondense 4129899
 # could delete ContigLocusIdFilter table here
 
 # create chrN_snpFasta tables from *.gnl files
 # snpLoadFasta.error will report all SNPs with "lengthTooLong"
 # here we have 4428 SNPs with lengthTooLong
 # these are noted as ObservedNotAvailable
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpLoadFasta dbSnpHumanBuild125 
 
 # split ContigLocFilter by chrom (could start using pipeline.csh here)
 # pipeline.csh takes about 35 minutes to run
 # create the first chrN_snpTmp
 # we will reuse this table name, adding/changing columns as we go
 # at this point chrN_snpTmp will have the same description as ContigLocFilter
 # this opens a file handle for every chrom, so will not scale to scaffold-based assemblies
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpSplitByChrom dbSnpHumanBuild125 ref_haplotype
 
 # generate true coords using loc_type
 # possible errors logged to snpLocType.error:
 # "Missing quotes in phys_pos for range"
 # "Chrom end <= chrom start for range"
 # "Wrong size for exact"
 # "Unknown locType"
 # "Unable to get chromEnd"
 # We got none of these
 
 # possible exceptions logged to snpLocType.exceptions:
 # RefAlleleWrongSize
 # this run got just 40 
 
 # morph chrN_snpTmp 
 
 mysql> desc chr1_snpTmp;
 
 #  +---------------+-------------+------+-----+---------+-------+
 #  | Field         | Type        | Null | Key | Default | Extra |
 #  +---------------+-------------+------+-----+---------+-------+
 #  | snp_id        | int(11)     | NO   |     |         |       |
 #  | ctg_id        | int(11)     | NO   |     |         |       |
 #  | chromStart    | int(11)     | NO   |     |         |       |
 #  | chromEnd      | int(11)     | NO   |     |         |       |
 #  | loc_type      | tinyint(4)  | NO   |     |         |       |
 #  | orientation   | tinyint(4)  | NO   |     |         |       |
 #  | allele        | blob        | YES  |     | NULL    |       |
 #  +---------------+-------------+------+-----+---------+-------+
 
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpLoctype dbSnpHumanBuild125 ref_haplotype
 
 # expand allele as necessary
 # report syntax errors to snpExpandAllele.errors
 # this run had 63 of these
 # possible exceptions logged to snpExpandAllele.exceptions:
 # RefAlleleWrongSize
 # this run has 512 
 
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpExpandAllele dbSnpHumanBuild125 ref_haplotype
 
 # the next few steps prepare for working in UCSC space
 # sort by position
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpSort dbSnpHumanBuild125 ref_haplotype
 # get hg17 nib files
 # get hg17 chromInfo, load into dbSnpHumanBuild125 with editted path
 hgsql -e "rename table chrMT_snpTmp to chrM_snpTmp" dbSnpHumanBuild125
 
 # lookup reference allele in nibs
 # keep reverse complement to use in error checking (snpCheckAlleles)
 # check here for SNPs larger than 1024
 # errAbort if detected
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpRefUCSC dbSnpHumanBuild125
 
 # morph chrN_snpTmp 
 
 mysql> desc chr1_snpTmp;
 
 #  +--------------------+-------------+------+-----+---------+-------+
 #  | Field              | Type        | Null | Key | Default | Extra |
 #  +--------------------+-------------+------+-----+---------+-------+
 #  | snp_id             | int(11)     | NO   |     |         |       |
 #  | ctg_id             | int(11)     | NO   |     |         |       |
 #  | chromStart         | int(11)     | NO   |     |         |       |
 #  | chromEnd           | int(11)     | NO   |     |         |       |
 #  | loc_type           | tinyint(4)  | NO   |     |         |       |
 #  | orientation        | tinyint(4)  | NO   |     |         |       |
 #  | allele             | blob        | YES  |     | NULL    |       |
 #  | refUCSC            | blob        | YES  |     | NULL    |       |
 #  | refUCSCReverseComp | blob        | YES  |     | NULL    |       |
 #  +--------------------+-------------+------+-----+---------+-------+
 
 # compare allele from dbSNP to refUCSC
 # locType between is excluded from this check
 # log exceptions to snpCheckAllele.exceptions
 # if SNP is positive strand, expect allele == refUCSC
 # log RefAlleleMismatch if not
 # if SNP is negative strand, if not allele == refUCSC, then check for allele == refUCSCReverseComp
 # If allele == refUCSCRevComp, log RefAlleleNotRevComp
 # If allele doesn't match either of refUCSC or refUCSCReverseComp, log RefAlleleMismatch
 # This run we got:
 # 0 RefAlleleMismatch
 # 49763   RefAlleleNotRevComp
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpCheckAlleles dbSnpHumanBuild125
 
 # add class, observed and molType from chrN_snpFasta tables
 # log errors to snpReadFasta.errors
 # errors detected: no data available, duplicate data
 # This run we got:
 # 49 no data available
 # 226048 duplicate
 # chrN_snpFasta has class = 'in-del'
 # we promote this to 'deletion' for locType 1&2 and to 'insertion' for locType 3
 
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpReadFasta dbSnpHumanBuild125
 
 # morph chrN_snpTmp
 #  +--------------------+---------------+------+-----+---------+-------+
 #  | Field              | Type          | Null | Key | Default | Extra |
 #  +--------------------+---------------+------+-----+---------+-------+
 #  | snp_id             | int(11)       | NO   |     |         |       |
 #  | chromStart         | int(11)       | NO   |     |         |       |
 #  | chromEnd           | int(11)       | NO   |     |         |       |
 #  | loc_type           | tinyint(4)    | NO   |     |         |       |
 #  | class              | varchar(255)  | NO   |     |         |       |
 #  | orientation        | tinyint(4)    | NO   |     |         |       |
 #  | molType            | varchar(255)  | NO   |     |         |       |
 #  | allele             | blob          | YES  |     | NULL    |       |
 #  | refUCSC            | blob          | YES  |     | NULL    |       |
 #  | refUCSCReverseComp | blob          | YES  |     | NULL    |       |
 #  | observed           | blob          | YES  |     | NULL    |       |
 #  +--------------------+---------------+------+-----+---------+-------+
 
 # generate exceptions for class and observed
 
 # SingleClassBetweenLocType
 # SingleClassRangeLocType
 # NamedClassWrongLocType
 
 # ObservedNotAvailable
 # ObservedWrongFormat
 # ObservedWrongSize
 # ObservedMismatch
 
 # RangeSubstitutionLocTypeExactMatch
 
 # SingleClassTriAllelic
 # SingleClassQuadAllelic
 
 # This will also detect IUPAC symbols in allele
 
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpCheckClassAndObserved dbSnpHumanBuild125
 
 # add function
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpFunction dbSnpHumanBuild125
 
 # add validation status and heterozygosity
 # log error if validation status > 31 or missing
 # this run we got 8 missing
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpSNP dbSnpHumanBuild125
 
 # generate chrN_snp125 and snp125Exceptions tables
 cp snpCheckAlleles.exceptions snpCheckAlleles.tab
 cp snpCheckClassAndObserved.exceptions snpCheckClassAndObserved.tab
 cp snpExpandAllele.exceptions snpExpandAllele.tab
 cp snpLocType.exceptions snpLocType.tab
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpFinalTable dbSnpHumanBuild125
 
 # PAR SNPs
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpPAR dbSnpHumanBuild125
 hgsql -e 'load data local infile "snpPARexceptions.tab" into table snp125Exceptions' dbSnpHumanBuild125
 
 # concat into snp125.tab
 # cat chr*_snp125.tab >> snp125.tab
 /bin/sh concat.sh
 
 # load
 hgsql dbSnpHumanBuild125 < /cluster/home/heather/kent/src/hg/lib/snp125.sql
 hgsql -e 'load data local infile "snp125.tab" into table snp125' dbSnpHumanBuild125
 
 # check for multiple alignments
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpMultiple dbSnpHumanBuild125
 mysql> load data local infile 'snpMultiple.tab' into table snp125Exceptions;
 
 # run and review snpCompareLoctype (currently tuned for 124/125 differences)
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpCompareLoctype dbSnpHumanBuild125 snp124subset snp125
 # cat snpCompareLoctypeCounts.out
 # exactToExact = 8310192
 # exactToBetween = 107956
 # exactToRange = 16200
 # betweenToBetween = 206224
 # betweenToExact = 4012
 # betweenToRange = 715
 # rangeToRange = 98648
 # rangeToBetween = 3151
 # rangeToExact = 6198
 # oldToNew = 10224
 
 # 12043 coord changes in exact (.1%)
 # 1370 moved to different chroms
 
 # 3664 coord changes in between (1.7%)
 # 2260 off-by-one
 # 13 moved to different chroms
 
 # 22198 coord changes in range (22.5%)
 # 19548 look like fixes: observedLengthOld != coordSpanOld && observedLengthNew == coordSpanNew
 # 1296 look like errors: observedLengthOld == coordSpanOld && observedLengthNew != coordSpanNew
 
 # load on hgwdev
 cp snp125.tab /cluster/home/heather/transfer/snp
 hgsql dbSnpHumanBuild125 -e 'select * from snp125Exceptions' > /cluster/home/heather/transfer/snp/snp125Exceptions.tab
 ssh hgwdev
 mysql> load data local infile 'snp125.tab' into table snp125; 
 
 # create indexes
 mysql> alter table snp125 add index name (name);
 mysql> alter table snp125 add index chrom (chrom, bin);
 
 mysql> load data local infile 'snp125Exceptions.tab' into table snp125Exceptions; 
 mysql> alter table snp125Exceptions add index name(name);
 
 # create snp125ExceptionDesc table
 cd /cluster/data/dbSnp
 # add counts to exception.template
 hgsql hg17 < snp125ExceptionDesc.sql
 mysql> load data local file 'exception.template' into table snp125ExceptionDesc;
 
 #######
 # Add new case for ObservedWrongSize (Heather June 9, 2006)
 # revisions 1.25 and 1.26 kent/src/hg/snp/snpLoad/snpCheckClassAndObserved.c
 ssh kkr5u00
 cd /scratch/snp/125/human
 /bin/csh pipeline.csh
 # wait 35 minutes
 grep ObservedWrongSize snpCheckClassAndObserved.exceptions > ObservedWrongSize
 grep ObservedWrongSize snpPARexceptions.tab >> ObservedWrongSize
 cp ObservedWrongSize /cluster/home/heather/transfer/snp
 ssh hgwdev
 hgsql -e 'alter table snp125Exceptions drop index name' hg17
 hgsql -e 'load data local infile "/cluster/home/heather/transfer/snp/ObservedWrongSize" into table snp125Exceptions' hg17
 hgsql -e 'alter table snp125Exceptions add index name'
 
 # fix counts
 hgsql -e 'select count(*), exception from snp125Exceptions group by exception' hg17
 
 +----------+------------------------------------+
 | count(*) | exception                          |
 +----------+------------------------------------+
 |   785903 | MultipleAlignments                 |
 |      623 | NamedClassWrongLocType             |
 |     7686 | ObservedMismatch                   |
 |     4333 | ObservedNotAvailable               |
 |       97 | ObservedWrongFormat                |
 |    73558 | ObservedWrongSize                  |
 |      466 | RangeSubstitutionLocTypeExactMatch |
 |       62 | RefAlleleMismatch                  |
 |    99849 | RefAlleleNotRevComp                |
 |     1278 | RefAlleleWrongSize                 |
 |    20749 | SingleClassBetweenLocType          |
 |     2306 | SingleClassQuadAllelic             |
 |    15639 | SingleClassRangeLocType            |
 |    19330 | SingleClassTriAllelic              |
 +----------+------------------------------------+
 
 # edit /cluster/data/dbSNP/exception.template (need to automate this)
 hgsql -e 'delete from snp125ExceptionDesc' hg17
 hgsql -e 'load data local infile "/cluster/data/dbSNP/exception.template" into table snp125ExceptionDesc' hg17
 
 
 ###########################
 
 # add rs_fasta to seq/extFile (Heather Nov 2006)
 # use 126 rs_fasta files because I didn't save 125 version
 ssh hgwdev
 mkdir /gbdb/hg17/snp
 ln -s /cluster/store12/snp/126/human/rs_fasta/snp.fa /gbdb/hg17/snp/snp.fa
 cd /cluster/store12/snp/126/human/rs_fasta
 hgLoadSeq hg17 /gbdb/hg18/snp/snp.fa
 # clean up after hgLoadSeq
 rm seq.tab
 
 # look up id in extFile
 # move into separate table
 hgsql hg17 < snpSeq.sql
 hgsql -e 'insert into snpSeq select acc, file_offset from seq where extFile = 33852294' hg17
 hgsql -e 'delete from seq where extFile = 33852294' hg17
 hgsql -e 'alter table snpSeq add index acc (acc)' hg17
 
 #############################################################
 # Get panTro2 and rheMac2 allele for all SNPs (Heather, Dec 2006, Feb 2007 and
 # June 2007 [partial fix released 6/25/07: using hg17 instead of hg18 liftOver
 # files... for most but not all chroms! :( not documented below; error found 
 # by user]
 # 1/11/08 (angie): re-running panTro2Qual and subsequent chimp & summary 
 # steps, so hg17 liftOver files will have been used for all outputs.
 # Deletions will probably lift okay
 # The insertions have start == end so none of them will lift
 # 1/24/08 (angie): constant quality score of 98 for chimp chr{21,M,Y,Y_random}
 # was previously put in score field -- corrected to orthoScore.
 
 ssh hgwdev
 cd /san/sanvol1/snp/liftOver/hg17
 mkdir panTro2All
 mkdir rheMac2All
 mkdir input
 cd input
 hgsql -N -e 'select chrom, chromStart, chromEnd, name, score, strand from snp125' hg17 > snp125.bed
 lineFileSplit snp.bed lines 100000 snp-
 ln -s /san/sanvol1/snp/liftOver/hg17/input /san/sanvol1/snp/liftOver/hg17/panTro2All/input
 ln -s /san/sanvol1/snp/liftOver/hg17/input /san/sanvol1/snp/liftOver/hg17/rheMac2All/input
 cd ../panTro2All
 ./makeJobList.csh
 mkdir output
 mkdir unmapped
 cd ../rheMac2All
 ./makeJobList.csh
 mkdir output
 mkdir unmapped
 
 # cluster run
 ssh pk
 cd /san/sanvol1/snp/liftOver/hg17/panTro2All
 para create jobList; para try; para check; para push
 para time
 # Completed: 108 of 108 jobs
 # CPU time in finished jobs:      67758s    1129.29m    18.82h    0.78d  0.002 y
 # IO & Wait Time:                   961s      16.02m     0.27h    0.01d  0.000 y
 # Average job time:                 636s      10.60m     0.18h    0.01d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:            1543s      25.72m     0.43h    0.02d
 # Submission to last job:         61513s    1025.22m    17.09h    0.71d
 
 cd /san/sanvol1/snp/liftOver/hg17/rheMac2All
 para create jobList; para try; para check; para push
 para time
 # Completed: 108 of 108 jobs
 # CPU time in finished jobs:       1833s      30.56m     0.51h    0.02d  0.000 y
 # IO & Wait Time:                  1744s      29.06m     0.48h    0.02d  0.000 y
 # Average job time:                  33s       0.55m     0.01h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              82s       1.37m     0.02h    0.00d
 # Submission to last job:         59987s     999.78m    16.66h    0.69d
 
 # add sequence
 # next time do this at the same time as lift
 
 cd /san/sanvol1/snp/liftOver/hg17
 mkdir panTro2Seq
 mkdir panTro2Seq/input
 mkdir panTro2Seq/output
 cp panTro2All/output/snp*out panTro2Seq/input
 cd panTro2Seq
 ./makeJobList.csh
 
 cat << 'EOF' > makeJobList.csh
 #!/bin/tcsh
 rm -f jobList
 foreach fileName (`ls input/*`)
     set baseName = $fileName:t
     echo $baseName
     echo "/cluster/home/heather/kent/src/hg/snp/snpLoad/fetchSeq $fileName /scratch/hg/panTro2/panTro2.2bit output/$baseName" >> jobList
 end
 'EOF'
 
 cd /san/sanvol1/snp/liftOver/hg17
 mkdir rheMac2Seq
 mkdir rheMac2Seq/input
 mkdir rheMac2Seq/output
 cp rheMac2All/output/snp*out rheMac2Seq/input
 cd rheMac2Seq
 cat << 'EOF' > makeJobList.csh
 #!/bin/tcsh
 rm -f jobList
 foreach fileName (`ls input/*`)
     set baseName = $fileName:t
     echo $baseName
     echo "/cluster/home/heather/kent/src/hg/snp/snpLoad/fetchSeq $fileName /scratch/hg/rheMac2/rheMac2.2bit output/$baseName" >> jobList
 end
 'EOF'
 
 # cluster run for sequence
 ssh pk
 cd /san/sanvol1/snp/liftOver/hg17/panTro2Seq
 para create jobList; para try; para check; para push
 para time
 # Completed: 108 of 108 jobs
 # CPU time in finished jobs:      30509s     508.48m     8.47h    0.35d  0.001 y
 # IO & Wait Time:                   325s       5.42m     0.09h    0.00d  0.000 y
 # Average job time:                 286s       4.76m     0.08h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:             551s       9.18m     0.15h    0.01d
 # Submission to last job:          1195s      19.92m     0.33h    0.01d
 
 cd /san/sanvol1/snp/liftOver/hg17/rheMac2Seq
 para create jobList; para try; para check; para push
 para time
 # Completed: 108 of 108 jobs
 # CPU time in finished jobs:      28517s     475.28m     7.92h    0.33d  0.001 y
 # IO & Wait Time:                   576s       9.61m     0.16h    0.01d  0.000 y
 # Average job time:                 269s       4.49m     0.07h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:             509s       8.48m     0.14h    0.01d
 # Submission to last job:          1166s      19.43m     0.32h    0.01d
 
 # quality scores 
 # This takes about 24 hours for each species!!  Ugh.
 # Solution is to use -bedFile argument to hgWiggle
 ssh hgwdev
 cd /san/sanvol1/snp/liftOver/hg17
 cd panTro2Seq/output
 cat << 'EOF' > concat.csh
 #!/bin/tcsh
 rm -f all.out
 foreach fileName (`ls snp*.out`)
     cat $fileName >> all.out
 end
 'EOF'
 
 sort all.out > all.sort
 rm all.out
 
 cd /san/sanvol1/snp/liftOver/hg17
 mkdir panTro2Qual
 cp panTro2Seq/output/all.sort panTro2Qual
 cd panTro2Qual
 mkdir input
 splitFileByColumn all.sort input
 mkdir output
 
 # If we do this again, we should write a c program to read qac files into
 # memory -- much faster than one hgWiggle process per line.
 cat << 'EOF' > addQual.pl
 #!/usr/bin/perl -W
 $db=shift;
 $chromName=shift;
 while (<STDIN>)
 {
     my @fields = split;
     my $chrom = $fields[0];
     my $chromStart = $fields[1];
     my $chromEnd = $fields[2];
     my $name = $fields[3];
     my $strand = $fields[5];
     my $allele = $fields[6];
 
     $cmd="hgWiggle -db=$db -chrom=$chromName -position=$chrom:$chromStart-$chromStart -rawDataOut quality";
     open(RESULT, "$cmd |") or die "can't start '$cmd'\n";
     while ($line = <RESULT>)
     {
         $score = int($line);
         print "$chrom\t$chromStart\t$chromEnd\t$name\t$score\t$strand\t$allele\n";
     }
 }
 'EOF'
 cat << 'EOF' > getQual.csh
 #!/bin/tcsh
 foreach fileName (`ls input/*`)
     set chromName = $fileName:t:r
     echo $chromName
     addQual.pl panTro2 $chromName < $fileName > output/$chromName
 end
 'EOF'
 # << emacs
 ./getQual.csh
 
 cd rheMac2Seq/output
 concat.csh
 sort all.out > all.sort
 rm all.out
 cd /san/sanvol1/snp/liftOver/hg17
 mkdir rheMac2Qual
 cp rheMac2Seq/output/all.sort rheMac2Qual
 cd rheMac2Qual
 mkdir input
 splitFileByColumn all.sort input
 mkdir output
 ./getQual.csh
 
 # concatenate, merge and load
 # chimp has no qual scores for chr21, chrY and chrM, just use seq files
 cd /san/sanvol1/snp/liftOver/hg17/panTro2Qual/output
 grep chr21 ../../panTro2Seq/output/all.sort > chr21
 grep chrY ../../panTro2Seq/output/all.sort | grep -v random > chrY
 grep chrY ../../panTro2Seq/output/all.sort | grep random > chrY_random
 grep chrM ../../panTro2Seq/output/all.sort > chrM
 #-----------------------------------------------------------------------------
 # 1/11/08: replace outputs for chroms that apparently were skipped in the June 
 # run, and re-run subsequent steps for chimp.  
 cd /san/sanvol1/snp/liftOver/hg17/panTro2Qual
 mv output output-jun25
 foreach f (output-jun25/chr*)
   if ( "X"`cmp $f output-feb26/$f:t` == "X" ) then
     echo $f:t
   endif
 end
 #chr21
 #chrM
 #chrY
 #chrY_random
 # <<-- those are the ones that may not have actually been regenerated.
 
 # It appears that the Feb. outputs, instead of the June Seq files, were copied 
 # to the June output for those chroms.  oops!
 # As a minor improvement, skip duplicate rows instead of just copying.
 foreach chr (chr21 chrM chrY chrY_random)
   echo $chr
   uniq input/$chr.sort > output/$chr
 end
 # << emacs
 
 mkdir output-jun25-incorrect
 mv output-jun25/chr{21,M,Y,Y_random} output-jun25-incorrect
 cat output-jun25/chr* output/chr* > output/qual.tab
 
 # end 1/11/08 fix-specific; proceeding to post-concat.csh chimp steps.
 #-----------------------------------------------------------------------------
 ./concat.csh
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpOrtho hg17 snp125 qual.tab
 hgLoadBed hg17 snp125OrthoPanTro2 snpOrtho.tab -tab -sqlTable=/cluster/home/heather/kent/src/hg/lib/snp125OrthoPanTro2.sql
 #Loaded 9591230 elements of size 17
 # previously 9590961
 # add index
 hgsql hg17
   alter table snp125OrthoPanTro2 add index name (name);
   alter table snp125OrthoPanTro2 add index chrom (chrom, bin);
 # 1/24/08: these used to set score; should have set orthoScore all along.
 # tweak to match panTro2 assembly
   update snp125OrthoPanTro2 set orthoScore = 98 where orthoChrom = "chr21";
 #Query OK, 129170 rows affected (25.37 sec)
 #Rows matched: 129170  Changed: 129170  Warnings: 0
   update snp125OrthoPanTro2 set orthoScore = 98 where orthoChrom = "chrY";
 #Query OK, 22081 rows affected (25.16 sec)
 #Rows matched: 22081  Changed: 22081  Warnings: 0
   update snp125OrthoPanTro2 set orthoScore = 98 where orthoChrom = "chrY_random";
 #Query OK, 155 rows affected (25.41 sec)
 #Rows matched: 155  Changed: 155  Warnings: 0
 
 # macaque
 cd /san/sanvol1/snp/liftOver/hg17/rheMac2Qual/output
 ./concat.csh
 /cluster/home/heather/kent/src/hg/snp/snpLoad/snpOrtho hg17 snp125 qual.tab
 hgLoadBed hg17 snp125OrthoRheMac2 snpOrtho.tab -tab -sqlTable=/cluster/home/heather/kent/src/hg/lib/snp125OrthoRheMac2.sql
 # add index
 alter table snp125OrthoRheMac2 add index name (name);
 alter table snp125OrthoRheMac2 add index chrom (chrom, bin);
 
 # get hapmap subset for chimp
 # skip if lift wasn't size 1
 # this run 124822 skipped
 cd /cluster/data/hg17/bed/hapmap/rel21a
 time /cluster/home/heather/kent/src/hg/snp/snpLoad/hapmapOrtho hg17 \
   hapmapSnpsCombined snp125OrthoPanTro2
 #108.505u 16.869s 2:26.22 85.7%  0+0k 0+0io 4pf+0w
 hgLoadBed hg17 hapmapAllelesChimp hapmapOrtho.tab -tab -sqlTable=/cluster/home/heather/kent/src/hg/lib/hapmapAllelesChimp.sql
 #Loaded 3930564 elements of size 13
 hgsql hg17 -e 'alter table hapmapAllelesChimp add index name(name); \
   alter table hapmapAllelesChimp add index chrom (chrom, bin);'
 
 # get hapmap subset for macaque
 # this run 106607 skipped
 /cluster/home/heather/kent/src/hg/snp/snpLoad/hapmapOrtho hg17 hapmapSnpsCombined snp125OrthoRheMac2
 hgLoadBed hg17 hapmapAllelesMacaque hapmapOrtho.tab -tab -sqlTable=/cluster/home/heather/kent/src/hg/lib/hapmapAllelesMacaque.sql
 rm hapmapOrtho.tab
 rm hapmapOrtho.err
 rm bed.tab
 alter table hapmapAllelesMacaque add index name(name);
 alter table hapmapAllelesMacaque add index chrom (chrom, bin);
 
 ##############################################################################
 # HapMap Recombination Rate Phase 2 (Heather Feb. 2006)
 # Contacts:
 # Gil McVean [mcvean@stats.ox.ac.uk]                                                                                                 
 # Colin Freeman [cfreeman@stats.ox.ac.uk]                                                                                            
 # Simon Myers [smyers@broad.mit.edu]         
 # Data is missing chromEnd.  I am setting chromEnd = chromStart + 1 as a
 # kludge for now.
 # Solution is to interpolate range but remove gaps.
 
 ## ****************************************
 # This is a bad assumption about the data format -- here is a description.
 ## ****************************************
 # The recombination rates are for the regions _between_ snps, so these
 # files need to be processed slightly differently.  For each line i in
 # the file (except the header and the last line), the recombination
 # rate is for the position on the current line minus 1 [pos(${i})-1] to
 # the position on the subsequent line [pos({$i+1})].  The precision is
 # a bit obnoxious and can be truncated to 3 or 4 significant figures.
 # (Note that the recombination rate on the last line is 0, as this is a
 # placeholder.)  Here is an example:
 #
 # > head genetic_map_chr1.txt 
 # position COMBINED_rate(cM/Mb) Genetic_Map(cM)
 # 45413 2.98182170902573 0
 # 72434 2.08241435350679 0.0805718043995841
 # 78032 2.08135840137317 0.0922291599505152
 # 244859 2.88844902005393 0.439455937976397
 # 604461 2.88749757426825 1.47814798248583
 # 604484 2.88586385769306 1.47821439493004
 # 605296 2.88389196108775 1.48055771638249
 #
 ### BED format (like a bedGraph)
 # chr1 45412  72434  2.982
 # chr1 72433  78302  2.082
 # chr1 78031  244859 2.081
 # chr1 244858 604461 2.888
 # chr1 604460 604484 2.887
 # chr1 604483 605296 2.886
 # chr1 605295 .....  2.884
 #
 # See /cluster/data/hg16/bed/hapmap/recombination/Perlegen/makeBed.pl for an example.  /cluster/data/hg16/bed/hapmap/recombination/Perlegen/cmds.csh is also useful. 
 ## ****************************************
 
 ssh hgwdev
 cd /cluster/data/hg17/bed/hapmap
 mkdir recombination
 cd recombination
 mkdir phase2
 cd phase2
 wget --no-check-certificate -N https://mathgen.stats.ox.ac.uk/HapMap_Phase2_rates_hotspots/HapMap_Phase2_rates_hotspots.tgz  
 # data also available at
 # http://www.hapmap.org/downloads/recombination/2006-10_rel21_phaseII
 gunzip *.tgz
 tar xvf *.tar
 cat << 'EOF' > makeBed.csh
 #!/bin/tcsh
 rm -f recomb.bed
 foreach chrom (`cat chrom.list`)
     echo $chrom
     set fileName=`echo $chrom | awk '{printf "genetic_map_%s.txt", $1}'`
     makeBed.pl $chrom < $fileName >> recomb.bed
 end
 makeBed.pl chrX < genetic_map_chrX_par1.txt >> recomb.bed
 makeBed.pl chrX < genetic_map_chrX_non-par.txt >> recomb.bed
 makeBed.pl chrX < genetic_map_chrX_par2.txt >> recomb.bed
 'EOF'
 cat << 'EOF' > makeBed.pl
 #!/usr/bin/env perl
 $chromName = shift;
 while (<STDIN>) {
     my @fields = split;
     # skip header
     if ($fields[0] eq "position") { next; }
     print $chromName;
     print "\t";
     print $fields[0];
     print "\t";
     print $fields[0] + 1;
     print "\t";
     my $val1000 = $fields[1] * 1000;
     my $valRound = int($val1000);
     my $newVal = $valRound / 1000.0;
     print $newVal;
     print "\n";
 }
 'EOF'
 ./makeBed.csh
 hgLoadBed hg17 snpRecombRateHapmapPhase2 recomb.bed -tab -bedGraph=4
 hgsql -e 'alter table snpRecombRateHapmapPhase add index chrom (chrom, bin)' hg17
 
 ############
 # UPDATE hg17 knownToVisiGene (2006-04-05 galt)
 # Create table that maps between known genes and visiGene database 
     # mapping to other species such as mouse, zebrafish, frog
     # requires visiGene probe track vgImageProbes be created first
     knownToVisiGene hg17 -fromProbePsl=vgImageProbes
 #############################################################
 # ADD A NEW TRACK GROUP (DONE, 6/3/06, Fan)
 # Create a new track group, "phenDis".
 
     echo 'INSERT INTO grp (name, label, priority) VALUES ("phenDis", "Phenotype and Disease Associations", 2.5)' \
    | hgsql hg17
 
 #############################################################
 # hgMut - Human Mutation track - Belinda Giardine
 # list of tables by show tables like 'hgMut%'
 # summary of current load June 7, 2006
 #table definitions for autoSql
 autoSql hgMut.as hgMut -dbLink
 #move bin in struct so works as bed 4+
 #hgMut.sql: change INDEXes as needed, put in enums
 #shrink mutId to 64 chars, plus acc to 48
 #data files and details under ~giardine/humPhen/
 cd humPhen/hgMutData/April2006/
 cat hgMutHbVar.txt hgMutPah.txt hgMutBgmut.txt hgMutCftr.txt hgMutARdb.txt > hgMutUnsorted.txt
 grep "^chr" hgMutUnsorted.txt | sort -k1,1 -k2,2n > hgMut.bed
 #create tables
 hgsql hg17 < ../../hgMut.sql
 #loading
 hgLoadBed hg17 hgMut hgMut.bed -noSort -oldTable -tab
 #load small vocab control tables
 hgsql hg17 < hgMutLink.sql
 hgsql hg17 < hgMutAttrClass.sql
 hgsql hg17 < hgMutAttrName.sql
 hgsql hg17 < hgMutSrc.sql
 #from hgsql hg17
 load data local infile "hgMutExtLinkHbVar.txt" into table hgMutExtLink;
 load data local infile "hgMutExtLinkARdb.txt" into table hgMutExtLink;
 load data local infile "hgMutExtLinkBgmut.txt" into table hgMutExtLink;
 load data local infile "hgMutExtLinkCFTR.txt" into table hgMutExtLink;
 load data local infile "hgMutExtLinkPah.txt" into table hgMutExtLink;
 load data local infile "hgMutExtLinkSP.txt" into table hgMutExtLink;
 load data local infile "hgMutAttrHbVar2.txt" into table hgMutAttr;
 load data local infile "hgMutAttrHbvarProt2.txt" into table hgMutAttr;
 load data local infile "hgMutAttrARdb.txt" into table hgMutAttr;
 load data local infile "hgMutAttrARdbProt.txt" into table hgMutAttr;
 load data local infile "hgMutAliasHbVar.txt" into table hgMutAlias;
 load data local infile "hgMutAliasARdb.txt" into table hgMutAlias;
 load data local infile "hgMutAliasBgmut.txt" into table hgMutAlias;
 load data local infile "hgMutAliasPah.txt" into table hgMutAlias;
 load data local infile "hgMutExtLinkHbVarOmim.txt" into table hgMutExtLink;
 load data local infile "hgMutAttrLink.txt" into table hgMutAttrLink;
 load data local infile "hgMutAttrSP.txt" into table hgMutAttr;
 
 #############################################################
 # gv*  Belinda Giardine
 # These tables are to replace the hgMut tables
 # Most data is converted by me (on PSU machines) to loadable format and copied.
 # The Swiss-Prot/UniProt data, is generated from the UniProt database at UCSC, 
 # using perl scripts and table dumps.
 # scripts in kent/src/hg/utils/gvParsers/swissProt/
 # everything redone to not depend on the dv track in July 2006
 
 #make list of variants from Swiss-Prot (make sure featureClass 23 is variant)
 hgsql -N uniProt > spVars.txt <<end
 select feature.acc, start, end-start, featureType.val, featureId.val from
 feature, featureType, accToTaxon, featureId where featureClass=23 and
 featureType=featureType.id and accToTaxon.acc=feature.acc and taxon=9606 and
 feature.featureId=featureId.id;
 end
 #need list mapping 3 letter amino acids to 1 letter. (aminoInfoDump from PSU)
 #known gene protein map (kgProtMap) has psl data from (blastp)
 # with qName being the spId
 hgsql -N hg17 > kgProtMapDump.txt <<end
 select kgProtMap.* from kgProtMap, uniProt.feature where kgProtMap.qName =
 uniProt.feature.acc;
 end
 #table join duplicates; perl script to throw out extra before use
 uniqueRows < kgProtMapDump.txt > kgProtMapUniq.txt
 #check variables for output and input file names
 computeSpVars > errors.txt
 #errors.txt will list variants that couldn't be mapped
 #July 18, 2006
 #37 gaps, 564 proteins (2228 variants) not in kgProtMap (test one did align)
 #found 22389
 #Swiss-Prot attributes: 
 hgsql hg17 < listSPconnections.sql > listSPconnections.txt
 hgsql proteome < listSpXref2.sql > listSpXref2.txt
 convertOmimTitle > gvLinkSPomim.txt
 hgsql hg17 < listGeneVals.sql > listGeneVals.txt
 convertDisPoly > gvLinkSPuni.txt
 cat gvLinkSPuni.txt gvLinkSPomim.txt > gvLinkSp.txt
 cp gvLinkSp.txt ../../../gv/gvData/
 
 #creating gv* tables and loading 
 #June 27, 2006
 autoSql gv.as gv -dbLink
 #edit indexes and string lengths in .sql file
 id=48, srcId=48, raKey=48, attrType=48,
 primary key=index on bin and attr and link ids (id, attrType for attrs)
 #do enums
 #add unique index, to prevent doubles
 UNIQUE KEY (chrom(12), chromStart, chromEnd, name)
 #added id field to gvPos struct so can keep ID when change name
 #    char *id;   /* Added field to hold ID if change name */
 #set to null in gv.c file
 
 #reload data July 2006 with more data and corrected Swiss-Prot data
 #also moved gv*(except gvPos) and omimTitle to hgFixed
 #prep data: concatenate all the gvPos data, sort
 cat gvPosSP.txt gvPosHbVar.txt gvPosARdb.txt gvPosBgmut.txt gvPosCftr.txt
 gvPosPah.txt gvPosSrd5a2.txt gvPosBrca.txt > gvPosAll.txt
 grep "^chr" gvPosAll.txt | sort -k1,1 -k2,2n > gvPosSortedHg17.bed
 
 #load tables
 hgLoadBed hg17 gvPos gvPosSortedHg17.bed -noSort -oldTable -tab
 hgsql hg17 < gvSrc.sql
 hgsql hg17
         load data local infile "gvBrca.txt" into table gv;
         load data local infile "gvAttrBrca.txt" into table gvAttr;
         load data local infile "gvLinkBrca.txt" into table gvLink;
         load data local infile "gvLinkSP.txt" into table gvLink;
         load data local infile "gvLinkSPgene.txt" into table gvLink;
         load data local infile "gvSP.txt" into table gv;
         load data local infile "gvAttrSP.txt" into table gvAttr;
         load data local infile "gvAttrLongSP.txt" into table gvAttrLong;
         load data local infile "gvLinkHbVar.txt" into table gvLink;
         load data local infile "gvHbVar.txt" into table gv;
         load data local infile "gvAttrHbVar.txt" into table gvAttr;
         load data local infile "gvARdb.txt" into table gv;
         load data local infile "gvAttrARdb.txt" into table gvAttr;
         load data local infile "gvBgmut.txt" into table gv;
         load data local infile "gvAttrBgmut.txt" into table gvAttr;
         load data local infile "gvAttrLongBgmut.txt" into table gvAttrLong;
         load data local infile "gvLinkBgmut.txt" into table gvLink;
         load data local infile "gvCftr.txt" into table gv;
         load data local infile "gvAttrCftr.txt" into table gvAttr;
         load data local infile "gvPah.txt" into table gv;
         load data local infile "gvAttrPah.txt" into table gvAttr;
         load data local infile "gvLinkPah.txt" into table gvLink;
         load data local infile "gvSrd5a2.txt" into table gv;
         load data local infile "gvAttrSrd5a2.txt" into table gvAttr;
         load data local infile "gvAttrConservedDisease.txt" into table gvAttr;
 
 #get disease association predictions for conserved variants
 #get list a variants that are already done
 hgsql -N hg17 > gvWithDiseaseStatus.txt <<end
 select id from gvAttr where attrType = 'disease';
 end
 #use table browser to get variants that intersect most conserved track
 #set conserved variants that are null to likely
 computeDiseaseAssocCons > gvAttrConservedDisease.txt
 
 #Belinda Giardine Sept 2006
 #reload tables, removed ones with sequence mismatches, added label and strand
 #added new lsbd BTKbase
 #Sequence mismatches were determined by using the position in the reference
 #sequence to fetch the sequence affected by the variant.  Then for substitions
 #and deletions with the nts deleted listed, the sequence was compared.
 #Insertions and large deletions could not be checked.
 
 #Belinda Giardine Dec 2006
 #reload tables, additions to previous sources and more IDbases
 #details in hg18 doc
 
 #Belinda Giardine Jan 2007
 #reload tables, additions and corrections, details in hg18 doc
 
 #############################################################
 # Illumina Hap300 (Heather, July 2006)
 
 ssh hgwdev
 cd /cluster/data/hg17/bed
 mkdir illumina
 cd illumina
 trim.pl < Illumina_HumanHap300_SNPlist_01.13.2006.txt > trim.out
 hgsql hg17 < illuminaTmp.sql
 hgsql -e "load data local infile 'trim.out' into table illuminaTmp" hg17
 # illuminaLookup generates bin
 /cluster/home/heather/kent/src/hg/snp/snpLoad/illuminaLookup hg17 illuminaTmp snp125 snp125Exceptions illuminaLookup.out illuminaLookup.err
 # errors:
 # unexpected chrom chr1 for snp rs1291584
 # unexpected chrom chr17 for snp rs3826555
 # unexpected locType between for snp rs2036773
 # unexpected locType between for snp rs2249255
 # unexpected locType between for snp rs8051412
 # unexpected locType between for snp rs1017238
 # unexpected locType between for snp rs5019493
 # 16 with locType = range*
 # 402 not found!
 # None that have multiple alignments.
 hgsql hg17 < snpArrayIllumina300.sql
 hgsql -e "load data local infile 'illuminaLookup.out' into table snpArrayIllumina300" hg17
 hgsql -e "alter table snpArrayIllumina300 add index name (name)" hg17
 hgsql -e "alter table snpArrayIllumina300 add index chrom (chrom, bin)" hg17
 
 #############################################################
 # Illumina Hap550 and Hap650 (Heather, April 2007)
 # Transfer from hg18 for Bert Gold at NCI
 
 ssh hgwdev
 cd /cluster/data/hg17/bed/illumina
 hgsql hg18 < getHg18-550.sql > 550.hg18
 hgsql hg18 < getHg18-650.sql > 650.hg18
 # get name, chrom, chromStart, chromEnd, strand observed from snp125
 # where class = "single" and locType = "exact" and chromEnd = chromStart + 1
 # Including tri/quad allelic and multiple-aligning for now
 hgsql hg17 < getHg17.sql > snp125single.hg17
 # sort and join
 sort 550.hg18 > 550.hg18.sort
 sort 650.hg18 > 650.hg18.sort
 sort snp125single.hg17 > snp125single.hg17.sort
 # 560704 lines in 550.join
 # 660137 lines in 650.join
 # 687 lines in 550.missing
 # 706 lines in 650.missing
 join 550.hg18.sort snp125single.hg17.sort > 550.join
 join 650.hg18.sort snp125single.hg17.sort > 650.join
 join -v 1 550.hg18.sort snp125single.hg17.sort > 550.missing
 join -v 1 650.hg18.sort snp125single.hg17.sort > 650.missing
 # fix column order
 awk '{print $2, $3, $4, $1, 0, $5, $6}' 550.join > 550.bed
 awk '{print $2, $3, $4, $1, 0, $5, $6}' 650.join > 650.bed
 # load
 hgLoadBed hg17 snpArrayIllumina550 550.bed -sqlTable=snpArrayIllumina550.sql
 hgLoadBed hg17 snpArrayIllumina650 650.bed -sqlTable=snpArrayIllumina650.sql
 # indices
 mysql> alter table snpArrayIllumina550 add index name (name);
 mysql> alter table snpArrayIllumina550 add index chrom (chrom, bin);
 mysql> alter table snpArrayIllumina650 add index name (name);
 mysql> alter table snpArrayIllumina650 add index chrom (chrom, bin);
 
 #############################################################
 # Affy 500K (Heather, September 2006)
 # look up rsId using position
 
 ssh hgwdev
 cd /cluster/data/hg17/bed/snp/affyData/500K
 # awk to create bed format from tsv files
 /bin/csh cmds.csh
 hgsql hg17 < affy250Nsp.sql
 hgsql hg17 < affy250Sty.sql
 hgsql -e "load data local infile 'Mapping250K_Nsp.bed' into table affy250Nsp" hg17
 hgsql -e "load data local infile 'Mapping250K_Sty.bed' into table affy250Sty" hg17
 
 # look up dbSNP rsIDs using position
 # affy250Nsp
 # 4311 missing, 7276 multiple
 /cluster/home/heather/kent/src/hg/snp/snpLoad/affyLookup hg17 affy250Nsp snp125
 mv affyLookup.out affy250Nsp.bed
 mv affyLookup.err affy250Nsp.err
 hgsql hg17 < snpArrayAffy250Nsp.sql
 hgLoadBed hg17 snpArrayAffy250Nsp affy250Nsp.bed -sqlTable=snpArray250Nsp.sql -tab
 hgsql -e "alter table snpArrayAffy250Nsp add index name (name)" hg17
 hgsql -e "alter table snpArrayAffy250Nsp add index chrom (chrom, bin)" hg17
 
 # affy250Sty
 # 3540 missing, 6901 multiple
 /cluster/home/heather/kent/src/hg/snp/snpLoad/affyLookup hg17 affy250Sty snp125
 mv affyLookup.out affy250Sty.bed
 mv affyLookup.err affy250Sty.err
 hgsql hg17 < snpArrayAffy250Sty.sql
 hgLoadBed hg17 snpArrayAffy250Sty affy250Sty.bed -sqlTable=snpArray250Sty.sql -tab
 hgsql -e "alter table snpArrayAffy250Sty add index name (name)" hg17
 hgsql -e "alter table snpArrayAffy250Sty add index chrom (chrom, bin)" hg17
 
 
 #############################################################
 # Affy 10K (Sept. 2006, Heather)
 # look up rsId using position
 
 ssh hgwdev
 cd /cluster/data/hg17/bed/snp/affyData/10K100Kagain
 
 # affy10
 # 14 missing, 807 multiple
 cp affy10K.txt affy10Temp.bed
 hgLoadBed hg17 affy10Temp affy10Temp.bed -sqlTable=affy10Temp.sql -tab -noBin
 /cluster/home/heather/kent/src/hg/snp/snpLoad/affyLookup hg17 affy10Temp snp125
 mv affyLookup.out affy10.bed
 mv affyLookup.err affy10.err
 hgLoadBed hg17 snpArrayAffy10 affy10.bed -sqlTable=snpArrayAffy10.sql -tab
 
 # affy10v2
 # 12 missing, 716 multiple
 hgLoadBed hg17 affy10v2Temp affy10v2Temp.bed -sqlTable=affy10v2Temp.sql -tab -noBin
 /cluster/home/heather/kent/src/hg/snp/snpLoad/affyLookup hg17 affy10v2Temp snp125
 mv affyLookup.out affy10v2.bed
 mv affyLookup.err affy10.errv2
 hgLoadBed hg17 snpArrayAffy10v2 affy10v2.bed -sqlTable=snpArrayAffy10v2.sql -tab
 
 # affy50HindIII
 # 156 missing, 1396 multiple
 hgLoadBed hg17 affy50HindIIITemp affy50HindIII.bed -sqlTable=affy50HindIIITemp.sql -tab -noBin
 /cluster/home/heather/kent/src/hg/snp/snpLoad/affyLookup hg17 affy50HindIIITemp snp125
 mv affyLookup.out affy50HindIII.bed
 mv affyLookup.err affy50HindIII.err
 hgLoadBed hg17 snpArrayAffy50HindIII affy50HindIII.bed -sqlTable=snpArrayAffy50HindIII.sql -tab
 hgsql -e "alter table snpArrayAffy50HindIII add index name (name)" hg17
 hgsql -e "alter table snpArrayAffy50HindIII add index chrom (chrom, bin)" hg17
 
 # affy50XbaI
 # 115 missing, 1745 multiple
 hgLoadBed hg17 affy50XbaITemp affy50XbaI.bed -sqlTable=affy50XbaITemp.sql -tab -noBin
 /cluster/home/heather/kent/src/hg/snp/snpLoad/affyLookup hg17 affy50XbaI snp125
 mv affyLookup.out affy50XbaI.bed
 mv affyLookup.err affy50XbaI.err
 hgLoadBed hg17 snpArrayAffy50XbaI affy50XbaI.bed -sqlTable=snpArrayAffy50XbaI.sql -tab
 hgsql -e "alter table snpArrayAffy50XbaI add index name (name)" hg17
 hgsql -e "alter table snpArrayAffy50XbaI add index chrom (chrom, bin)" hg17
 
 
 #########################################################################
 # REGULATORY POTENTIAL (DONE - 2006-06-14 - Hiram)
     #	download data from "James Taylor" <james@bx.psu.edu>
     ssh kkstore02
     mkdir /cluster/store11/hg17/bed/regPotential7X
     cd /cluster/data/hg17/bed
     ln -s /cluster/store11/hg17/bed/regPotential7X ./regPotential7X
     cd regPotential7X
     
     #	This is a lot of data
     time for C in 1 2 3 4 5 6 7 8 9 X Y 10 11 12 13 14 15 16 17 18 19 20 21 22
     do
     wget --timestamping \
 "http://www.bx.psu.edu/~james/esperr_rp_7way_scores/genome_scores_hg17/chr${C}.scores.truncated.bz2"
     done
     #	real    115m1.855s
 
     wget --timestamping \
 "http://www.bx.psu.edu/~james/esperr_rp_7way_scores/genome_scores_hg17/trackDb.html" -O description.html
 
     time for C in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y
     do
 	bzcat chr${C}.scores.truncated.bz2
     done | wigEncode -noOverlap  stdin regPotential7X.wig regPotential7X.wib
     #	Converted stdin, upper limit 1.00, lower limit -0.00
     #	real    33m48.487s
 
     #	Loading the table on hgwdev
     ssh hgwdev
     cd /cluster/data/hg17/bed/regPotential7X
     ln -s /cluster/data/hg17/bed/regPotential7X/regPotential7X.wib \
 	/gbdb/hg17/wib/regPotential7X.wib
     #	using the tmpDir is faster since it is on local disk and it will
     #	clean up any temporary .tab file it creates there
     time hgLoadWiggle -tmpDir=/scratch/tmp \
 	hg17 regPotential7X regPotential7X.wig
 
     #	How about a histogram of the data.
     ssh kolossus
     cd /cluster/data/hg17/bed/regPotential7X
     time hgWiggle -verbose=2 -doHistogram -hBinSize=0.01 -hBinCount=100 \
 	-hMinVal=0.0 -db=hg17 regPotential7X > histogram.data 2>&1
     #	real    2m48.810s
     #	73 % of the data values are zero
 
     #	create download gzip files from the bz2 files:
     for F in chr*.scores.truncated.bz2
     do
 	C=`echo $F | awk -F'.' '{print $1}'`
 	echo -n "${C}.regPotential7X.hg17.gz working ... "
 	bzcat ${F} | gzip > ${C}.regPotential7X.hg17.gz
 	echo
     done
 
 #########################################################################
 ####### RE-BUILD RGD HUMAN QTL TRACKS (DONE 06/21/06 Fan) ##############
 # DELETED RECORD FROM rgdQtlLink SO CONSISTENT WITH REMOVAL FROM rgdQtl
 # (DONE, 2006-06-30, hartera)
 
 ssh hgwdev
 mkdir -p /cluster/store8/rgd/human12062005
 rm /cluster/data/hg17/bed/rgdQtl
 ln -s /cluster/store8/rgd/human12062005 /cluster/data/hg17/bed/rgdQtl
 cd /cluster/data/hg17/bed/rgdQtl
 
 # download data files from RGD
 
 wget --timestamp ftp://rgd.mcw.edu/pub/RGD_genome_annotations/human/rgd_human_qtl_12062005.gff
 
 # remove extra line feed character at the end of lines
 
 rmLf rgd_human_qtl_12062005.gff > rgdQtl.gff
 
 # create rgdQtl.tab
 awk '{print $1"\t"$4"\t"$5"\t"$10}'  rgdQtl.gff |sed -e 's/Chr/chr/g'| \
 sed -e 's/"//g' |sed -e 's/RGD://g' | sed -e 's/;//g' |sort -u > rgdQtl.tab
 
 # create rgdQtlLink.tab
 
 cat rgdQtl.gff |cut -f 9 |sed -e 's/; Note /\t/g'|\
 sed -e 's/Alignment //' |sed -e 's/;Note /\t/' |\
 sed -e 's/"//g' |sed -e 's/RGD://' >j.tmp
 cut -f 2 j.tmp >j.1
 cut -f 1,3 j.tmp >j.2
 paste j.1 j.2 |sort -u >rgdQtlLink.tab
 rm j.1 j.2 j.tmp
 
 # load rgdQtl table
 hgLoadBed hg17 rgdQtl rgdQtl.tab
 
 # check rgdQtl table
 checkTableCoords hg17 rgdQtl
 
 # Go the following error messages:
 
 #hg17.rgdQtl has 2 records with end > chromSize.
 #hg17.rgdQtl has 2 records with end > chromSize.
 #hg17.rgdQtl has 2 records with end > chromSize.
 #hg17.rgdQtl has 1 records with end > chromSize.
 #hg17.rgdQtl has 2 records with end > chromSize.
 #hg17.rgdQtl has 3 records with end > chromSize.
 #hg17.rgdQtl has 1 records with end > chromSize.
 #hg17.rgdQtl has 3 records with end > chromSize.
 #hg17.rgdQtl has 2 records with end > chromSize.
 #hg17.rgdQtl has 1 records with end > chromSize.
 #hg17.rgdQtl has 3 records with end > chromSize.
 #hg17.rgdQtl has 2 records with end > chromSize.
 #hg17.rgdQtl has 1 records with end > chromSize.
 #hg17.rgdQtl has 1 records with end > chromSize.
 #hg17.rgdQtl has 1 records with end > chromSize.
 #hg17.rgdQtl has 1 records with end > chromSize.
 #hg17.rgdQtl has 1 records with end < start.
 
 hgsql hg17 -N -e 'select "do1", name, c.size from rgdQtl r, chromInfo c where chromEnd > c.size and r.chrom=c.chrom' >doall
 
 cat << '_EOF_' > do1
 hgsql hg17 -e "update rgdQtl set chromEnd = '${2}' where name='${1}'"
 '_EOF_'
 
 chmod +x do*
 doall
 
 checkTableCoords hg17 rgdQtl
 #hg17.rgdQtl has 1 records with end < start.
 
 hgsql hg17 -e 'select * from rgdQtl where chromEnd < chromStart'
 # bin     chrom   chromStart      chromEnd        name
 # 9       chr10   7135612 371019  BW63_H
 # 0       chr20   77628133        5242324 AASTH39_H
 
 # Don't know why checkTableCoords only catches one of the two erros.
 hgsql hg17 -e "update rgdQtl set chromStart = 271019 where name='BW63_H'"
 hgsql hg17 -e "update rgdQtl set chromEnd = 7135612 where name='BW63_H'"
 
 # Delete the following record.  The RGD QTL is very questionable. 
 hgsql hg17 -e "delete from rgdQtl where name='AASTH39_H'"
 
 # load rgdQtlLink table
 hgsql hg17 -e "drop table hg17.rgdQtlLink;"
 hgsql hg17 <~/kent/src/hg/lib/rgdQtlLink.sql
 hgsql hg17 -e 'load data local infile "rgdQtlLink.tab" into table hg17.rgdQtlLink;'
 
 # Delete the record from rgdQtlLink table that was removed from the rgdQtl
 # table above. (hartera, 2006-06-30)
 hgsql hg17 -e "delete from rgdQtlLink where name='AASTH39_H'"
 
 ########################################################################
 
 #########################################################################
 #Reload omimTitle table			Belinda Giardine June 28, 2006
 #fetched omim.txt.Z from OMIM downloads.
 #parse out title lines (*FIELD* TI)
 convertTitle < omim.txt > omimTitle.txt
 #load into omimTitle table
 truncate table omimTitle;
 load data local infile "omimTitle.txt" into table omimTitle;
 
 #############################################################
 # Lift SV track from hg16 (Heather, July 2006)
 # hg16 SV track is comprised of 7 subtracks:
 # cnpFosmid, cnpSebat, cnpIafrate, cnpSharp, delConrad, delMccarroll, delHinds
 # Use the same table formats as hg16; pre-create
 # (No bin for del tables)
 
 cd /cluster/data/hg17/bed
 mkdir svMixed
 cd svMixed
 
 # I got hg17 coords from Andy Sharp for cnpFosmid and delHinds
 trimFosmid.pl < cnpFosmid.txt > cnpFosmid.bed
 hgLoadBed -tab hg17 cnpFosmid cnpFosmid.bed
 hinds.pl < hinds.txt > delHinds.bed
 hgLoadBed -tab -noBin hg17 delHinds delHinds.bed
 
 # (7-27-2006 Brooke Rhead -- edited the cnpFosmid table)
 # According to Andy Sharp, the name='Gap' items should be removed from
 # cnpFosmid.  I dumped the table, removed the 'Gap' lines, then dumped the
 # table again.
 
 cd /cluster/data/hg17/bed/svMixed
 hgsql hg17 -e "select * from cnpFosmid" > cnpFosmid_withGaps.bed
 
 hgsql hg17
 delete from cnpFosmid where name='Gap';
 
 hgsql hg17 -e "select * from cnpFosmid" > cnpFosmid_withoutGaps.bed
 
 # Simple lifts for delMccarroll
 cat << '_EOF_' > liftConrad.csh
 #!/bin/csh
 hgsql -N -e 'select * from delMccarroll' hg16 > delMccarroll.hg16
 liftOver -minMatch=0.7 delMccarroll.hg16 /gbdb/hg16/liftOver/hg16ToHg17.over.chain.gz delMccarroll.bed delMccarroll.err
 hgLoadBed -sqlTable=delMccarroll.sql -tab -noBin hg17 delMccarroll delMccarroll.bed
 '_EOF_'
 
 # Lift both chromStart/chromEnd and thickStart/thickEnd for delConrad and join
 cat << '_EOF_' > liftConrad.csh
 #!/bin/csh
 hgsql -N -e 'select chrom, chromStart, chromEnd, name, score, strand from delConrad' hg16 > delConrad.hg16.1
 hgsql -N -e 'select chrom, thickStart, thickEnd, name, score, strand from delConrad' hg16 > delConrad.hg16.2
 liftOver -minMatch=0.7 delConrad.hg16.1 /gbdb/hg16/liftOver/hg16ToHg17.over.chain.gz delConrad.tmp.1 delConrad.err.1
 liftOver -minMatch=0.7 delConrad.hg16.2 /gbdb/hg16/liftOver/hg16ToHg17.over.chain.gz delConrad.tmp.2 delConrad.err.2
 trimConrad.pl < delConrad.tmp.1 > delConrad.trim.1
 trimConrad.pl < delConrad.tmp.2 > delConrad.trim.2
 sort delConrad.trim.1 > delConrad.sort.1
 sort delConrad.trim.2 > delConrad.sort.2
 join delConrad.sort.1 delConrad.sort.2 > delConrad.join
 awk '{print $2, $3, $4, $1, 1000, $5, $7, $8}' delConrad.join > delConrad.bed
 hgLoadBed -sqlTable=delConrad.sql -noBin hg17 delConrad delConrad.bed
 '_EOF_'
 
 # Andy Sharp says the Sebat data has already been lifted, so be conservative here
 # Create hg16.cnpSebatLiftCandidate that excludes 5 rows that had wild proliferations
 cat << '_EOF_' > liftSebat.csh
 hgsql -N -e 'select chrom, chromStart, chromEnd, name, probes, individuals from cnpSebatLiftCandidate' hg16 > cnpSebat.hg16
 liftOver -minMatch=0.7 -bedPlus=4 cnpSebat.hg16 /gbdb/hg16/liftOver/hg16ToHg17.over.chain.gz cnpSebat.bed cnpSebat.err
 hgLoadBed -sqlTable=cnpSebat.sql -tab hg17 cnpSebat cnpSebat.bed
 '_EOF_'
 
 # For Andy's data, use bacEndPairs first, then lift the remainder
 cat << '_EOF_' > liftSharp.csh
 # assumes a copy of hg16.cnpSharp in hg17.cnpSharpHg16Copy
 /cluster/home/heather/kent/src/hg/snp/snpLoad/cnpLookup hg17 bacEndPairs cnpSharpHg16Copy cnpSharpLookup.out cnpSharpLookup.lift cnpSharpLookup.log
 sed -e 's/Gain and Loss/GainAndLoss/' cnpSharpLookup.lift > cnpSharpLookup.lift.fix
 mv cnpSharpLookup.lift.fix cnpSharpLookup.lift
 liftOver -minMatch=0.7 -bedPlus=4 cnpSharpLookup.lift /gbdb/hg16/liftOver/hg16ToHg17.over.chain.gz cnpSharp.bed cnpSharp.err
 sed -e 's/GainAndLoss/Gain And Loss/' cnpSharp.bed > cnpSharp.bed.fix
 mv cnpSharp.bed.fix cnpSharp.bed
 hgLoadBed -tab -sqlTable=cnpSharp.sql hg17 cnpSharp cnpSharpLookup.out
 hgLoadBed -tab -oldTable hg17 cnpSharp cnpSharp.bed
 '_EOF_'
 
 # For the Iafrate data, the BAC End lookup wasn't good, so just lift
 # Create hg16.cnpIafrateLiftCandidate that excludes 2 rows that had wild proliferations
 cat << '_EOF_' > liftIafrate.csh
 hgsql -N -e 'select chrom, chromStart, chromEnd, name, variationType, score from cnpIafrateLiftCandidate' hg16 > cnpIafrate.hg16
 sed -e 's/Gain and Loss/GainAndLoss/' cnpIafrate.hg16 > cnpIafrate.hg16.fix
 mv cnpIafrate.hg16.fix cnpIafrate.hg16
 liftOver -minMatch=0.7 -bedPlus=4 cnpIafrate.hg16 /gbdb/hg16/liftOver/hg16ToHg17.over.chain.gz cnpIafrate.bed cnpIafrate.err
 sed -e 's/GainAndLoss/Gain And Loss/' cnpIafrate.bed > cnpIafrate.bed.fix
 mv cnpIafrate.bed.fix cnpIafrate.bed
 hgLoadBed -sqlTable=cnpIafrate.sql -tab hg17 cnpIafrate cnpIafrate.bed
 '_EOF_'
 
 ##############################################################################
 # Add HapMap CNVRs from Matt Hurles (Heather Dec 2006)
 ssh hgwdev
 cd /cluster/data/hg17/bed/svRedon
 # File from Matthew Hurles (meh@sanger.ac.uk) was essentially bed 4
 # I decided to use bed 6 with score always 0 and strand always +
 awk '{printf "%st%d\t%d\tcnp%s\t0\t%s\n", $1, $4, $5, $3, $7}' input.gff > input.bed
 hgLoadBed hg17 cnpRedon input.bed
 
 ##############################################################################
 #  dbRIP POLYALUL1SVA track added (2006-07-14 - DONE - Hiram)
 #  dbRIP polyAluL1SVA
 #	Data provider: Dr. Liang at the Liang lab:
 #	http://falcon.roswellpark.org/index.html
 #	Ping.Liang@roswellpark.org
 
 #	Adding this track is a new data type into our browser.
 #	data definitions for dbRIP and polyGenotype were added to
 #	the hg/lib/ directory:
 #	-rw-rw-r--  1  351 Jul 13 12:20 polyGenotype.as
 #	-rw-rw-r--  1  694 Jul 13 12:22 polyGenotype.sql
 #	-rw-rw-r--  1 6398 Jul 13 12:22 polyGenotype.c
 #	-rw-rw-r--  1   980 Jul 10 17:59 dbRIP.as
 #	-rw-rw-r--  1 11408 Jul 13 11:16 dbRIP.c
 #	-rw-rw-r--  1  1578 Jul 13 12:06 dbRIP.sql
 #	With associated .h files in hg/inc/
 #	-rw-rw-r--  1 4600 Jul 10 18:00 dbRIP.h
 #	-rw-rw-r--  1 4375 Jul 13 16:16 polyGenotype.h
 #	Changes in hgTracks and hgc to make this track appear as it does
 #	at their browser:
 #	http://falcon.roswellpark.org:9090/cgi-bin/hgTables
 
 #	For this first instance of the track, the data was obtained
 #	directly from their Genome browser via the tables browser,
 #	dumping the tables:
 #	hg17.polyAluL1 and hg17.polyGenotype
 #	saving these data dumps to:
 #		(after a couple of versions were used ...)
     ssh hgwdev
     mkdir /cluster/data/hg17/bed/dbRIP
     cd /cluster/data/hg17/bed/dbRIP
     #	-rw-rw-r--  1  994485 Aug  1 16:03 dbRIP.2006-08-01.txt.gz
     #	-rw-rw-r--  1   18532 Aug  1 16:05 polyGenotype.2006-08-01.txt.gz
 
     #	Rearrange their data columns to more closely match the
     #	standard BED definitions, and split into three different
     #	data sets:
 zcat dbRIP.2006-08-01.txt.gz | headRest 1 stdin | awk -F'\t' '
 {
 chromStart=$6
 chromStart -= 1
 chromEnd=$7
 if (match($1,"^RIP_SVA_.*")) {
 printf "%s\t%s\t%s\t%s\t0\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $5, chromStart, chromEnd, $1, $12, $2, $3, $4, $8, $9, $10, $11, $15, $13, $14, $16, $17, $18, $19, $20, $21
 }
 }' | sort -k1,1 -k2,2n > dbRIP.SVA.txt
 
 zcat dbRIP.2006-08-01.txt.gz | headRest 1 stdin | awk -F'\t' '
 {
 chromStart=$6
 chromStart -= 1
 chromEnd=$7
 if (match($1,"^RIP_L1_.*")) {
 printf "%s\t%s\t%s\t%s\t0\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $5, chromStart, chromEnd, $1, $12, $2, $3, $4, $8, $9, $10, $11, $15, $13, $14, $16, $17, $18, $19, $20, $21
 }
 }' | sort -k1,1 -k2,2n > dbRIP.L1.txt
 
 zcat dbRIP.2006-08-01.txt.gz | headRest 1 stdin | awk -F'\t' '
 {
 chromStart=$6
 chromStart -= 1
 chromEnd=$7
 if (match($1,"^RIP_Alu_.*")) {
 printf "%s\t%s\t%s\t%s\t0\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $5, chromStart, chromEnd, $1, $12, $2, $3, $4, $8, $9, $10, $11, $15, $13, $14, $16, $17, $18, $19, $20, $21
 }
 }' | sort -k1,1 -k2,2n > dbRIP.Alu.txt
 
     #	Create three specific sql table create definitions:
     sed -e "s/dbRIP/dbRIP_SVA/" $HOME/kent/src/hg/lib/dbRIP.sql > dbRIP_SVA.sql
     sed -e "s/dbRIP/dbRIP_L1/" $HOME/kent/src/hg/lib/dbRIP.sql > dbRIP_L1.sql
     sed -e "s/dbRIP/dbRIP_Alu/" $HOME/kent/src/hg/lib/dbRIP.sql > dbRIP_Alu.sql
 
     #	And loading those three data tables:
     hgLoadBed -verbose=3 -maxChromNameLength=6 -strict -tab -notItemRgb \
         -sqlTable=dbRIP_SVA.sql hg17 dbRIP_SVA dbRIP.SVA.txt
 
     hgLoadBed -verbose=3 -maxChromNameLength=6 -strict -tab -notItemRgb \
 	-sqlTable=dbRIP_L1.sql hg17 dbRIP_L1 dbRIP.L1.txt
 
     hgLoadBed -verbose=3 -maxChromNameLength=6 -strict -tab -notItemRgb \
 	-sqlTable=dbRIP_Alu.sql hg17 dbRIP_Alu dbRIP.Alu.txt
 
     #	And an associated table of genotype frequencies
     #	Add three extra rows to the original data to provide a better handle
     #	on MySQL lookups for allele Frequency
     hgsql hg17 -e "drop table polyGenotype;"
     hgsql hg17 < $HOME/kent/src/hg/lib/polyGenotype.sql
     zcat polyGenotype.2006-08-01.txt.gz | headRest 1 stdin | \
 awk -F'\t' '
 {
 sampleSize = $3 + $4 + $5
 plus = ($3 * 2) + $4
 minus = ($5 * 2) + $4
 if ((plus + minus) < 1) { alleleFreq=0 } else
         { alleleFreq = plus / (plus + minus) }
 if (sampleSize > 0) {
 heteroZyg = (2 * alleleFreq * (1.0 - alleleFreq)) * ((sampleSize * 2)/((sampleSize * 2) - 1))
 } else {
 heteroZyg = 2 * alleleFreq * (1.0 - alleleFreq)
 }
 printf "%s\t%s\t%d\t%d\t%d\t%.3f\t%.3f\t%.3f\n", $1, $2, $3, $4, $5, sampleSize, alleleFreq, heteroZyg
 }
 ' > polyGenotype.txt
 
     hgsql hg17 -e \
 	'load data local infile "polyGenotype.txt" into table polyGenotype;'
 
     #	A composite track was added to human/hg17/trackDb.ra to contain
     #	these three tracks, and search methods to get the name column
     #	participating in the search.  Need to figure out how to get some
     #	of the other text-rich columns participating in the search.
 
 ##############################################################################
 # hg17 -> hg15 LIFTOVER CHAINS (DONE 7/27/06 Fan)
     # I used a size of 10kb instead of 3kb for the split (blat query) sizes in
     # hg15.  This had a huge affect on the amount of hits in the blat, which
     # then had a huge effect on the amount of chains.  I should also mention 
     # that hg15 chromosomes chr1 and chr2 were split further 
     # into more than a single query file.  This helped a LOT in avoiding 
     # cluster hippos classically associated with those chroms.
     
     ######## LIFTOVER PREPARATION
     # Split up hg15
     ssh pk
     cd /san/sanVol1/scratch/hg15
     mkdir -p liftSplits/{split,lift}
     bash
     for fa in /cluster/data/hg15/?{,?}/*.fa; do
       c=`basename $fa .fa`
       echo $c
       faSplit -lift=liftSplits/lift/${c}.lft size $fa -oneFile 10000 liftSplits/split/$c
     done
     mkdir -p biggerSplits/split
     cd biggerSplits/
     ln -s ../liftSplits/lift
     cd split/
     ln -s ../../liftSplits/split/* .
     faSplit sequence chr1.fa 5 chr1_
     faSplit sequence chr2.fa 5 chr2_
     rm chr{1,2}.fa
 
     # Make some dirs
     # cd /san/sanVol1/scratch
     # mkdir -p hg17
 
     # Copy 11.ooc files to hg17 subdirectory.
     # cp -p /cluster/store5/gs.16/build33/11.ooc hg17
 
     ## First, copy over scripts. (Already done before)
 
     # mkdir -p /san/sanVol1/scratch/fan
     # cp -p /san/sanVol1/scratch/fan/*.sh /san/sanVol1/scratch/fan
     # cp /san/sanVol1/scratch/andy/psl.header /san/sanVol1/scratch/fan
 
     ######## LIFTOVER BLATING  
 
     # HG17
     ssh kk
     cd /cluster/data/hg17
     #makeLoChain-align hg17 /scratch/hg/hg17/nib hg15 /san/sanVol1/scratch/hg15/biggerSplits/split
 
     makeLoChain-align hg17 /scratch/hg/hg17/bothMaskedNibs hg15  /san/sanVol1/scratch/hg15/liftOver/biggerSplits/split 
 
 # Completed: 2392 of 2392 jobs
 # CPU time in finished jobs:   25651277s  427521.28m  7125.35h  296.89d  0.813 y
 # IO & Wait Time:                 74118s    1235.30m    20.59h    0.86d  0.002 y
 # Average job time:               10755s     179.25m     2.99h    0.12d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:           82545s    1375.75m    22.93h    0.96d
 # Submission to last job:         82579s    1376.32m    22.94h    0.96d
 
     ssh kkstore02
 
     cd /cluster/data/hg17
     cd bed
 
     mv blat.hg15.2006-07-25 /san/sanVol1/scratch/hg17 
 
     ssh pk
     cd /san/sanVol1/scratch/hg17/blat.hg15.2006-07-25/run/
     sed 's/^blat/blat.sh/; s/\}.*$/}/' spec | awk '{print "/san/sanVol1/scratch/fan/" $0 " hg17ToHg15"}' > newspec
     para create newspec
     para try
     para push
 # Completed: 2392 of 2392 jobs
 # CPU time in finished jobs:     612316s   10205.26m   170.09h    7.09d  0.019 y
 # IO & Wait Time:                 12421s     207.02m     3.45h    0.14d  0.000 y
 # Average job time:                 261s       4.35m     0.07h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:            3524s      58.73m     0.98h    0.04d
 # Submission to last job:          3588s      59.80m     1.00h    0.04d
 
     ######## LIFTOVER CHAINING
     # LIFTING
     ssh pk
     cd /san/sanVol1/scratch/fan
     cp mm7SplitLift.sh hg15SplitLift.sh
 
     # change andy to fan, mm7 to hg15, and chrX to chr2, and remove chrUn_random 
     vi hg15SplitLift.sh
 
     cat << 'EOF' > hg15ChainMergeSplit.sh
 #!/bin/bash
 cp -r chainRaw/ /scratch/fan/hg15Lifts
 pushd /scratch/fan/hg15Lifts
 mkdir chain
 /cluster/bin/x86_64/chainMergeSort chainRaw/*.chain | /cluster/bin/x86_64/chainSplit chain stdin
 cp -r chain `dirs +1`
 rm -rf chain chainRaw
 'EOF'
 
     chmod +x hg15ChainMergeSplit.sh
 
     # HG17
     cd /san/sanVol1/scratch/hg17/blat.hg15.2006-07-25/raw
     /san/sanVol1/scratch/fan/hg15SplitLift.sh
 # There was an extra file, nib22.fa, under /cluster/data/hg15/nib, which should not be there.
 # -rw-rw-r--  1 2429 protein  50466533 May 20  2003 nib22.fa
 # This caused hg15SplitLift.sh to end abnormally.
 
     cd ../    
     mkdir chainRun chainRaw
     cd chainRun
     cat > gsub << 'EOF'
 #LOOP
 /cluster/bin/x86_64/axtChain -verbose=0 -linearGap=medium -psl $(path1) /scratch/hg/hg17/bothMaskedNibs  /san/sanVol1/scratch/hg15/nib {check out line+ ../chainRaw/$(root1).chain}
 #ENDLOOP
 'EOF'
     ls -1S ../psl/*.psl > in.lst
     gensub2 in.lst single gsub spec
     para create spec
     para try
     para push
     para time
 # Completed: 44 of 44 jobs
 # CPU time in finished jobs:       3596s      59.94m     1.00h    0.04d  0.000 y
 # IO & Wait Time:                   919s      15.31m     0.26h    0.01d  0.000 y
 # Average job time:                 103s       1.71m     0.03h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:             274s       4.57m     0.08h    0.00d
 # Submission to last job:           284s       4.73m     0.08h    0.00d
     ######### CHAINMERGE/NET/NETSUBSET
     ssh kolossus
     mkdir -p /scratch/fan/hg15Lifts
     cd /scratch/fan/hg15Lifts
 
     cp -r /san/sanvol1/scratch/hg17/blat.hg15.2006-07-25/chainRaw/ .
     mkdir chain
     /cluster/bin/x86_64/chainMergeSort chainRaw/* | /cluster/bin/x86_64/chainSplit chain stdin
 # about 30 minutes.
 
     cp -rp chain /san/sanvol1/scratch/hg17/blat.hg15.2006-07-25/
     rm -rf chain
     rm -rf chainRaw
 
     ssh pk
     cd /san/sanvol1/scratch/fan
     cat << 'EOF' > netOver.sh 
 #!/bin/bash
 
 chain=$1
 chrom=`basename $chain .chain`
 sizesHGOld=$2
 sizesHG15=/cluster/data/hg15/chrom.sizes
 chainDir=`dirname $chain`
 blatDir=`dirname $chainDir`
 net=${blatDir}/net/${chrom}.net
 over=${blatDir}/over/${chrom}.over
 
 mkdir -p ${blatDir}/{over,net}
 /cluster/bin/x86_64/chainNet $chain $sizesHGOld $sizesHG15 $net /dev/null
 /cluster/bin/x86_64/netChainSubset $net $chain $over
 'EOF'
     chmod +x netOver.sh
 
     mkdir netRun
 
     cd netRun/
 
     find /san/sanVol1/scratch/hg17/blat.hg15.2006-07-25/chain -name "*.chain" \
      | awk '{print "/san/sanVol1/scratch/fan/netOver.sh " $1 " /cluster/data/hg17/chrom.sizes"}' > spec
     para create spec
     para push
     para time
 # Completed: 46 of 46 jobs
 # CPU time in finished jobs:        438s       7.30m     0.12h    0.01d  0.000 y
 # IO & Wait Time:                   118s       1.97m     0.03h    0.00d  0.000 y
 # Average job time:                  12s       0.20m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              28s       0.47m     0.01h    0.00d
 # Submission to last job:            67s       1.12m     0.02h    0.00d
 
     ########## FINISHING
     ssh hgwdev
 
     # HG17
     cd /san/sanvol1/scratch/hg17/blat.hg15.2006-07-25/over
     cat * >> ../hg17ToHg15.over.chain
     cd ../
     rm -rf psl/ net/ chain/ chainRaw/ over/
     cd ../
     cp -rp blat.hg15.2006-07-25/ /cluster/data/hg17/bed
 
     cd /cluster/data/hg17/bed
     ln -s blat.hg15.2006-07-25 blat.hg15
     ln -s `pwd`/blat.hg15/hg17ToHg15.over.chain liftOver/hg17ToHg15.over.chain
     ln -s `pwd`/liftOver/hg17ToHg15.over.chain /gbdb/hg17/liftOver/hg17ToHg15.over.chain
     mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/liftOver
 
     gzip /cluster/store5/gs.18/build35/bed/blat.hg15/hg17ToHg15.over.chain
     ln -s /cluster/store5/gs.18/build35/bed/blat.hg15/hg17ToHg15.over.chain.gz /gbdb/hg17/liftOver/
     cp -p /cluster/store5/gs.18/build35/bed/blat.hg15/hg17ToHg15.over.chain.gz /cluster/data/hg17/bed/liftOver 
 
     cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver
     ln -s /cluster/store5/gs.18/build35/bed/blat.hg15/hg17ToHg15.over.chain.gz hg17ToHg15.over.chain.gz
     hgAddLiftOverChain hg17 hg15 
 
 ############################################################################
 
 ############################################################################
 # Reload CCDS (ccdsInfo, ccdsGene, ccdsKgMap) (2006-08-15 markd)
     cd /cluster/data/genbank/data/ccds/hg17
     ftp ftp-private.ncbi.nih.gov (user ccds, needs password)
     ftp> get CCDS.20060815.tar.gz
     mkdir /scratch/tmp/ccds
     cd /scratch/tmp/ccds
     tar -zxf  /cluster/data/genbank/data/ccds/hg17/CCDS.20060815.tar.gz
 
     # import ccds database tables
     hgsql -e 'create database ccds'
     hgsql ccds </cluster/data/genbank/etc/createTables.sql 
     hgsql ccds </cluster/data/genbank/etc/createKeys.sql 
     /cluster/data/genbank/bin/x86_64/ccdsImport ccds data/*.txt
 
     # create and load ccdsGene and ccdsInfo tables from imported database
     /cluster/data/genbank/bin/x86_64/ccdsMkTables -loadDb ccds hg17 ccdsInfo ccdsGene
     checkTableCoords hg17 -verbose=2 ccdsGene
 
     /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap  -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap
     rm -rf /scratch/tmp/ccds
 
     # 2006-08-23 - found bug with some source genes missing from ccdsInfo, fixed ccdsMkTables
     # and reload
     /cluster/data/genbank/bin/x86_64/ccdsMkTables -loadDb ccds hg17 ccdsInfo ccdsGene
     checkTableCoords hg17 -verbose=2 ccdsGene
     joinerCheck -database=hg17 -identifier=ccdsGeneId ~/compbio/kent/src/hg/makeDb/schema/all.joiner
     /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap  -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap
     # request push of 
         ccdsGene
         ccdsInfo
         ccdsKgMap
 
     # << emacs
 ##########################################################################
 #  hars 1 to 202  Sol 09/10/2006
 
     set bedDir = /gbdb/hg17/haseq/bed
     mkdir -p $bedDir/hars
     pushd /projects/hg/wet/Sol/hars1to49
     cp -p hars_1to202.hg17.bed  $bedDir/hars/hars_1to202.bed
     hgLoadBed hg17 hars         $bedDir/hars/hars_1to202.bed
     rm -f                       $bedDir/hars/hars_1to202.bed
     popd
 
 # BUILD HPRD DATA FOR KNOWN GENE DETAILS PAGE LINKS (DONE 9/11/06)
 
 # First, build hprdToCdna.tab and hprdToUniProt.tab. 
 # See hg18.txt for details.
 
     cd ~/data/hprd
     mkdir hg17
     cd hg17
 
     hgsql hg17 -e 'drop table hprdToCdna'
     hgsql hg17 <~/src/hg/lib/hprdToCdna.sql
     hgsql hg17 -e 'load data local infile "../hprdToCdna.tab" into table hprdToCdna'
 
     hgsql hg17 -e 'drop table hprdToUniProt'
     hgsql hg17 <~/src/hg/lib/hprdToUniProt.sql
     hgsql hg17 -e 'load data local infile "../hprdToUniProt.tab" into table hprdToUniProt'
 
 # build knownToHprd table
 
     hgsql hg17 -N -e 'select kgId,hprdId from hprdToCdna, kgXref where cdnaId=kgId' >j.kg1
     hgsql hg17 -N -e 'select kgId,hprdId from hprdToUniProt, kgXref where uniProtId=spId' >j.kg2
 
     cat j.kg1 j.kg2 |sort -u >knownToHprd.tab
     wc knownToHprd.tab
 
     hgsql hg17 -e 'drop table knownToHprd'
     hgsql hg17 <~/src/hg/lib/knownToHprd.sql
 
     hgsql hg17 -e 'load data local infile "knownToHprd.tab" into table knownToHprd'
     hgsql hg17 -e 'select count(*) from knownToHprd'
 
 # 19,345 records created.
 
 # remove temporary files.
 
     rm j*
 
 # Do the same for hg17.  See hg17.txt for details.
 
 
 ############################################################################
 # Reload CCDS (ccdsInfo, ccdsGene, ccdsKgMap) (2006-09-20 markd)
 # Reloaded due to bug that results in multiple versions of the same accession
 # in the ccdsInfo table.
 
     cd /cluster/data/genbank/data/ccds/hg17
     ftp ftp-private.ncbi.nih.gov (user ccds, needs password)
     get CCDS.20060920.tar.gz
     mkdir /scratch/tmp/ccds
     cd /scratch/tmp/ccds
     tar -zxf  /cluster/data/genbank/data/ccds/hg17/CCDS.20060920.tar.gz
 
     # import ccds database tables
     hgsql -e 'drop database ccds; create database ccds'
     hgsql ccds </cluster/data/genbank/etc/createTables.sql 
     hgsql ccds </cluster/data/genbank/etc/createKeys.sql 
     /cluster/data/genbank/bin/x86_64/ccdsImport ccds data/*.txt
 
     # create and load ccdsGene and ccdsInfo tables from imported database
     /cluster/data/genbank/bin/x86_64/ccdsMkTables -loadDb ccds hg17 ccdsInfo ccdsGene
     /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap  -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap
     checkTableCoords hg17 -verbose=2 ccdsGene
     joinerCheck -database=hg17 -identifier=ccdsGeneId ~/compbio/kent/src/hg/makeDb/schema/all.joiner
     rm -rf /scratch/tmp/ccds
 
     # request push of 
         ccdsGene
         ccdsInfo
         ccdsKgMap
 
     # << emacs
 
 
 #########################################################
 # BUILD GAD TRACK (Done, 9/26/06, Fan)
 
    mkdir /cluster/store/gad060926
    ln -s /cluster/store/gad060926 /cluster/data/gad
 
 # Receive "allxlsAStxt.txt" from GAD/NIA 
 # contact person: Shenoy, Narmada, shenoyn@grc.nia.nih.gov  
    
    hgsql hg17 -e 'drop table gadAll'
    hgsql hg17 <~/src/hg/lib/gadAll.sql
    hgsql hg17 -e 'load data local infile "allxlsAStxt.txt" into table gadAll ignore 2 lines'
    hgsql hg17 -e 'create index geneSymbol on gadAll(geneSymbol(10))'
 
 # create gad table
 
    hgsql hg17 -N -e \
    'select "chr",chromosome, chromStart, chromEnd, geneSymbol from gadAll where chromStart <>0'|\
    sed -e 's/chr\t/chr/' |grep -v "chr\." |grep -v " "|sort -u >gad.bed
 
    hgLoadBed hg17 gad gad.bed
 
 #####################################################################
 # YALE TRANSCRIPTIONALLY ACTIVE REGIONS (TARs/TransFrags) TRACK IDENTIFIED 
 # USING A WHOLE GENOME TILING ARRAY (DONE, 2006-10-20, hartera)
 # Data is from the paper: Bertone et al. Science 24 December 2004:
 # Vol. 306. no. 5705, pp. 2242 - 2246. From Mark Gerstein's lab at Yale.
 # Contact at Yale: Joel S. Rozowsky, joel.rozowsky@yale.edu
 # The data consist of Transcriptionally Active Regions (TARs or TransFrags)
 # found using Affymetrix genome tiling arrays. The data is from the lab
 # of Mark Gerstein at Yale.
      ssh kkstore02
      mkdir /cluster/data/hg17/bed/yaleBertoneTars/
      cd /cluster/data/hg17/bed/yaleBertoneTars/
      # download Bertone et al. data from this URL:
     #http://dart.gersteinlab.org/cgi-bin/ar/download.cgi?ID=TAR_data_NCBI31.txt
      # and put it in this directory.
      # The sequences used to design the microarrays were from
      # UCSC hg13/NCBI Build 31 so the sequences
      # should be aligned again using Blat since this is probably better
      # than using liftOver across so many assemblies.
 
      # Get sequences from TARs file and put in FASTA format:
      # Remove characters from Windows:
      dos2unix TAR_data_NCBI31.txt
      # The TARs are in order of IDs in the file so the first TAR has ID 1, the
      # second is 2 up to the last which is 17517. These IDs are used to link
      # to the DART database of TARs at Yale so use these IDs in the FASTA
      # header lines. Need to add "TAR" as prefix to ID so that it is unique
      # in the seq table.
    awk 'BEGIN {FS="\t";n=0;}{if ($1 ~ /^chr/) print ">TAR"n"\n"$14"\n";n++;}' \
          TAR_data_NCBI31.txt > yaleBertoneTARSeqs.fa
      ssh pk
      mkdir -p /san/sanvol1/scratch/hg17/TARs/
      cp /cluster/data/hg17/bed/yaleBertoneTars/yaleBertoneTARSeqs.fa \
         /san/sanvol1/scratch/hg17/TARs/
      # Set up to Blat the TAR sequences against hg17
      cd /cluster/data/hg17/bed/yaleBertoneTars
      ls -1 /san/sanvol1/scratch/hg17/TARs/yaleBertoneTARSeqs.fa > tars.lst
      ls -1 /san/sanvol1/scratch/hg17/nib/*.nib > genome.lst
      # output dir
      mkdir psl
 
      cat << '_EOF_' > template.sub
 #LOOP
 /cluster/bin/x86_64/blat -repeats=lower -minIdentity=90 -ooc=/san/sanvol1/scratch/hg17/11.ooc $(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
 #ENDLOOP
 '_EOF_'
     # << for emacs
      gensub2 genome.lst tars.lst template.sub para.spec
      para create para.spec
      para try, para check, para push ...
      para time
 # Completed: 46 of 46 jobs
 # CPU time in finished jobs:        429s       7.16m     0.12h    0.00d  0.000 y
 # IO & Wait Time:                   153s       2.54m     0.04h    0.00d  0.000 y
 # Average job time:                  13s       0.21m     0.00h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:              38s       0.63m     0.01h    0.00d
 # Submission to last job:           107s       1.78m     0.03h    0.00d
      # sort and then filter
      pslSort dirs raw.psl tmp psl
      # use these parameters as for Genbank alignments of native mRNAs
      # for finished assemblies.
      pslCDnaFilter -minId=0.96 -minCover=0.25 -localNearBest=0.001 \
        -minQSize=20 -minNonRepSize=16 -ignoreNs -bestOverlap \
        raw.psl yaleBertoneTars.psl
 #                         seqs    aligns
 #             total:     17512   37530
 # drop minNonRepSize:     121     254
 #     drop minIdent:     3827    14532
 #     drop minCover:     571     897
 #       weird over:     232     837
 #        kept weird:     197     201
 #    drop localBest:     2359    3896
 #              kept:     17498   17951
      
      # 99.9% were kept. 
      # check how many aligned
      grep '>' yaleBertoneTARSeqs.fa | wc -l
      # 17517
      # 99.89% of the original set of sequences are in this filtered PSL file.
      pslCheck yaleBertoneTars.psl
      # psl is ok
 
      # load into database
      ssh hgwdev
      cd /cluster/data/hg17/bed/yaleBertoneTars
      hgLoadPsl hg17 yaleBertoneTars.psl
 
      # Add sequences to /gbdb/hg18 and to seq and extFile tables.
      mkdir -p /gbdb/hg17/yaleTARs/
      ln -s /cluster/data/hg17/bed/yaleBertoneTars/yaleBertoneTARSeqs.fa \
            /gbdb/hg17/yaleTARs/
      hgLoadSeq hg17 /gbdb/hg17/yaleTARs/yaleBertoneTARSeqs.fa
     
      # trackDb.ra entry is in trackDb/human/trackDb.ra and 
      # a description exist already as this track is also on hg18.
 
 ######################################################################
 ## reload tfbsCons table - it was based on a newer version of tfbs names that
 #	are not yet public domain (2006-11-03 - Hiram)
     mkdir /cluster/data/hg17/bed/tfbsCons
     cd /cluster/data/hg17/bed/tfbsCons
     cp -p /cluster/store6/weirauch/TFLOC/hg17/tfbsConsSites.bed .
     hgLoadBed -strict hg17 tfbsConsSites \
 	-sqlTable=$HOME/kent/src/hg/lib/tfbsConsSites.sql \
 	tfbsConsSites.bed -tab
 # this leads to a bunch of extra names in Factors
     hgsql -N -e "select name from tfbsConsSites;" hg17 | sort -u > names.new
     hgsql -N -e "select name from tfbsConsFactors;" hg17 \
 	| sort -u > names.factors
     comm -13 names.new names.factors > names.extra.factors
 for N in `cat extra.names.factors`
 do
         echo "delete from tfbsConsFactors where name=\"${N}\";" hg17
         hgsql -e "delete from tfbsConsFactors where name=\"${N}\";" hg17
 done
 
 #########################################################
 # BUILD GAD TRACK (Re-Re-Done, 12/12/06, Fan)
 
    mkdir /cluster/store12/gad061211
    rm /cluster/data/gad
    ln -s /cluster/store12/gad061211 /cluster/data/gad
 
 # Receive "GAD-Hg17DATA.txt" from GAD/NIA 
 # contact person: Shenoy, Narmada, shenoyn@grc.nia.nih.gov  
    
    hgsql hg17 -e 'drop table gadAll'
    hgsql hg17 <~/src/hg/lib/gadAll.sql
    hgsql hg17 -e 'load data local infile "GAD-Hg17DATA.txt" into table gadAll ignore 1 lines'
    hgsql hg17 -e 'create index geneSymbol on gadAll(geneSymbol(10))'
 
 # create gad table
 
    hgsql hg17 -N -e \
    'select "chr",chromosome, chromStart, chromEnd, geneSymbol from gadAll
    where chromStart <>0 and chromosome<>""'|\
    sed -e 's/chr\t/chr/' |grep -v "chr\." |grep -v " "|sort -u >gad.bed
 
    hgLoadBed hg17 gad gad.bed
 
 ##########################################################################
 #  xxBlastTab - Help filter out unwanted paralogs  (Galt 2007-01-11)
 #
 # Background: The xxBlastTab tables are made with a simple blastall 
 # (blastp with -b 1) which chooses the best match.  Unfortunately this
 # means that if there is no proper match it will still pick something
 # even though it's probably not orthologous. This is especially a problem
 # in organisms like rat knownGene which has only 30% gene coverage. 
 # The strategy here is to filter our xxBlastTab using synteny mappings from
 # the chains. This is done by simply taking $db.kg and using /gbdb/$db chains
 # and pslMap to lift the genes to the target xx assembly.  Then hgMapToGene
 # will find which of those mapped ids have good overlap with xx.knownGene.
 # The final mapping is then created by doing an inner join between 
 # the traditional xxBlastTab and the mapping table produced above.
 # Then simply drop the old table and rename the new table.
 #
 # We are starting with xxBlastTab tables already built in the usual way with
 # blastall/blastp, probably with doHgNearBlastp.pl script.
 #
 # I created a new utility script called synBlastp.csh since I have to do this
 # several times. 
 #
 # we want to update hg17 for rat and mouse, 
 # so check ./hgGeneData/Human/hg17/otherOrgs.ra for current settings
 
 ssh hgwdev
 
 synBlastp.csh hg17 rn3
 #hg17.rnBlastTab results:
 #new number of unique query values:
 #10728
 #new number of unique target values
 #5177
 #old number of unique query values:
 #24030
 #old number of unique target values
 #5535
 
 synBlastp.csh hg17 mm7
 #new number of unique query values:
 #25506
 #new number of unique target values
 #13462
 #old number of unique query values:
 #32951
 #old number of unique target values
 #14803
 
 
 
 #####################################################################
 ##########################################################################
 # GenBank gbMiscDiff table (markd 2007-01-10)
 # Supports `NCBI Clone Validation' section of mgcGenes details page
 
    # genbank release 157.0 now contains misc_diff fields for MGC clones
    # reloading mRNAs results in gbMiscDiff table being created.
    ./bin/gbDbLoadStep -reload -srcDb=genbank -type=mrna hg17
 
 ############
 # UPDATE hg17 knownToVisiGene (DONE galt 2007-02-15)
 # Create table that maps between known genes and visiGene database 
     # mapping to other species such as mouse, zebrafish, frog
     # requires visiGene probe track vgImageProbes be created first
     knownToVisiGene hg17 -fromProbePsl=vgImageProbes
 
 #########################################################
 # Chimp Paralogy data from Eichlers lab (DONE Heather Feb. 2007)
 cd /cluster/data/hg17/bed/eichler
 hgLoadBed hg17 chimpParalogy chimpParalogy.bed -tab -sqlTable=chimpParalogy.sql
 
 ############################################################################
 # Reload CCDS (ccdsInfo, ccdsGene, ccdsKgMap) (2007-03-02 markd)
 
     cd /cluster/data/genbank/data/ccds/
     ftp ftp-private.ncbi.nih.gov (user ccds, needs password)
     get CCDS.20070228.tar.gz
     mkdir /scratch/tmp/ccds
     cd /scratch/tmp/ccds
     tar -zxf /cluster/data/genbank/data/ccds/CCDS.20070228.tar.gz
 
     # import ccds database tables
     /cluster/data/genbank/bin/x86_64/ccdsImport ccds data/*.txt
 
     # create and load ccdsGene and ccdsInfo tables from imported database
     /cluster/data/genbank/bin/x86_64/ccdsMkTables -loadDb ccds hg17 ccdsInfo ccdsGene
     /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap  -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap
     checkTableCoords hg17 -verbose=2 ccdsGene
     joinerCheck -database=hg17 -identifier=ccdsGeneId ~/compbio/genbank/kent/src/hg/makeDb/schema/all.joiner
 
     # build initial version of ccdsMgcMap table, updated by nightly  genbank update
     /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap -loadDb -db=hg17 ccdsGene mgcGenes ccdsMgcMap
     
     # load trackDb
     cd kent/src/hg/makeDb/trackDb
     make alpha
     # check in browser
 
     # request push of 
         ccdsGene
         ccdsInfo
         ccdsKgMap
         ccdsMgcMap
     # << emacs
 
 #####################################################
 # Vista Enhancers (galt 2007-02-23 done)
 #
 # Vista from Lawrence-Berkeley has assayed
 # 301 human conserved non-coding intra- and inter-
 # genic elements for their ability to promote
 # lacZ in mouse embryos.  A positive looks like
 # a mouse with a purple spine.
 #
 # They provided a custom track with two tracks for pos and neg.
 # http://enhancer-test.lbl.gov/cgi-bin/customTrack.cgi
 # I am combining the tracks into one with high score for pos.
 #
 cd /cluster/data/hg17/bed
 mkdir vistaEnhancers
 cd vistaEnhancers
 wget -O custTrk "http://enhancer-test.lbl.gov/cgi-bin/customTrack.cgi"
 cat custTrk | head -116 | tail +2 > pos
 cat custTrk | tail +118 > neg
 cat pos | gawk '{print $1"\t"$2"\t"$3"\t"$4"\t900"}' > bed5
 cat neg | gawk '{print $1"\t"$2"\t"$3"\t"$4"\t200"}' >> bed5
 wc -l bed5
 #301 bed5
 hgLoadBed hg17 vistaEnhancers bed5
 #Loaded 301 elements of size 5
 
 # add to human/trackDb.ra
 track vistaEnhancers
 shortLabel Vista Enhancers
 longLabel Vista HMR-Conserved Non-coding Human Enhancers from LBNL
 group regulation
 priority 93
 visibility hide
 color 50,70,120
 type bed 5 .
 useScore 1
 url http://enhancer-test.lbl.gov/cgi-bin/imagedb.pl?form=presentation&show=1&experiment_id=$$
 
 ###
 # UPDATES (2007-10-18, conodera)
 # see also /projects/compbiousr/wet/browser/vista_enhancer/17Oct2007/Makefile
 
 cd /projects/compbiousr/wet/browser/vista_enhancer/
 
 # download data file from the vista browser (coordinates are for hg17)
 # http://enhancer.lbl.gov/cgi-bin/imagedb.pl?show=1;search.result=yes;form=search;search.form=no;action=search;search.sequence=1
 # save as enhancerbrowser.datadownload.txt
 
 # give elements with positive label a score of 900, 
 # give elements with negative label a score of 200.
 # print to 5-field bed file
 vista_enhancer.hg17.txt: enhancerbrowser.datadownload.txt
         grep ">" $< \
         | sed -e 's/>//' \
         | tr :- '       ' \
         | sed -e 's/positive/900/'\
         | sed -e 's/negative/200/' \
         | awk '{print $$1"\t"$$2"\t"$$3"\telement_"$$6"\t"$$8}' \
         > $@; \
 hgLoadBed hg17 vistaEnhancers vista_enhancer.hg17.txt;
 # loaded 446 elements of length 5
 
 
 #########################################################################
 # EPONINE-TSS (TRANSCRIPTON START SITE) PREDICTION 
 # (DONE, 2007-03-08, hartera)
 # The Eponine software is version 2 and has not changed in several years
 # (contact: Thomas Down at Sanger, td2@sanger.ac.uk). The version downloaded
 # for hg16 should be the same as the current version but download again just
 # to check. The application includes the TSS model file: eponine-tss2.xml 
 
      ssh kkstore02
      # Eponine runs fine on 2.5Mb contig, but barfs on much larger contig;
      # chop up sequence at gaps into ~2.5Mb chunks for cluster run.
      mkdir /san/sanvol1/scratch/hg17/chunks
      cd /cluster/data/hg17
      foreach f (?{,?}/NT_*/NT_??????.fa)
        set ctg = $f:t:r
        /cluster/bin/x86_64/faSplit -minGapSize=10 \
         -lift=/san/sanvol1/scratch/hg17/chunks/${ctg}.lft \
         gap $f 2500000 /san/sanvol1/scratch/hg17/chunks/${ctg}.chunk
      end
      # seems to ignore the chunk part of the file name
      mkdir /cluster/data/hg17/bed/eponine
      cd /cluster/data/hg17/bed/eponine
      wget --timestamping \
        http://www.sanger.ac.uk/Software/analysis/eponine/eponine-scan.jar
      # file has the same date and same size as the one downloaded for hg16
      # the script requires all of the path setting found in my .tcshrc file.
      # Using only set path = (/usr/java/jre1.5.0_06/bin $path)
      # as in the doEpo file for hg16 does not work.
      cat << '_EOF_' > doEpo
 #!/bin/csh -ef
 set path = (/usr/java/jre1.5.0_06/bin /bin /usr/bin /usr/X11R6/bin \
              /usr/local/bin . /cluster/home/hartera/bin/x86_64 \
              /cluster/bin/x86_64 /projects/compbio/bin/x86_64 \
              /projects/compbio/bin /projects/compbio/bin/x86_64-linux \
              /cluster/bin/scripts)
 java -jar ./eponine-scan.jar -threshold 0.999 -seq $1 > $2
 '_EOF_'
      # << emacs
      chmod a+x doEpo
      cp /dev/null jobList
      foreach f (/san/sanvol1/scratch/hg17/chunks/NT*.fa)
         echo "./doEpo {check in line+ $f} {check out exists out/$f:t:r.gff}" \
       >> jobList
      end
      mkdir out
      ssh pk
      cd /cluster/data/hg17/bed/eponine
      /parasol/bin/para create jobList
      /parasol/bin/para try, check, push, check etc.....
      /parasol/bin/para time
 # Completed: 1415 of 1415 jobs
 # CPU time in finished jobs:     104501s    1741.68m    29.03h    1.21d  0.003 y
 # IO & Wait Time:                  6594s     109.91m     1.83h    0.08d  0.000 y
 # Average job time:                  79s       1.31m     0.02h    0.00d
 # Longest running job:                0s       0.00m     0.00h    0.00d
 # Longest finished job:             127s       2.12m     0.04h    0.00d
 # Submission to last job:           488s       8.13m     0.14h    0.01d
      # lift chunks -> contigs
     mkdir contigs/
     foreach l (/san/sanvol1/scratch/hg17/chunks/*.lft)
       set ctg = $l:t:r
       liftUp contigs/$ctg.gff $l warn out/${ctg}*.gff
     end
     # lift contigs -> chrom
     liftUp eponine.gff /cluster/data/hg17/jkStuff/liftAll.lft \
            warn contigs/NT_*.gff
     # Translate to bed 4 + float-score -- it would be a shame to lose 
     # those scores in genePred or bed 5 (int score)
     awk 'BEGIN {i=0;} \
          {printf "%s\t%d\t%d\t%s.%d\t%s\t%s\n", $1, $4-1, $5, $1, i, $6, $7; \
           i = i + 1;}' \
       eponine.gff > eponine.bed
     # load up
     ssh hgwdev
     cd /cluster/data/hg17/bed/eponine
     sed -e 's/bed6FloatScore/eponine/g' \
       $HOME/kent/src/hg/lib/bed6FloatScore.sql > eponine.sql
     hgLoadBed hg17 eponine eponine.bed -tab -sqlTable=eponine.sql
     # Loaded 61013 elements of size 6
     # trackDb.ra entry and eponine.html already exist in trackDb directory.
 
 ###########################################################################
 # ACEScan Track (DONE 2007-03-15  Andy
 
 ssh hgwdev
 cd /cluster/data/hg17/bed
 mkdir acescan
 cd acescan/
 cp ~/acescan.gff .
 tail +2 acescan.gff > acescan.nh.gff
 ldHgGene -out=gp hg17 acescan acescan.nh.gff 
 rm *.gff
 ldHgGene -predTab hg17 acescan acescan.hg17.gp
 
 ###########################################################################
 # augustusHints track (DONE 2007-4-5 Mario)
 
 mkdir -p /cluster/data/hg17/bed/augustus/usingHints/predictions/Trefseq.hmRNA.hsEST.R.Xp.final
 cd /cluster/data/hg17/bed/augustus/usingHints/predictions/Trefseq.hmRNA.hsEST.R.Xp.final
 wget http://augustus.gobics.de/predictions/hg17/usingEvidence/augustus.hg17.Trefseq.hmRNA.hsEST.R.Xp.gff
 wget http://augustus.gobics.de/predictions/hg17/usingEvidence/augustus.hg17.Trefseq.hmRNA.hsEST.R.pep.aa
 ldHgGene -bin hg17 augustusHints augustus.hg17.Trefseq.hmRNA.hsEST.R.Xp.gff
 hgPepPred hg17 generic augustusHintsPep augustus.hg17.Trefseq.hmRNA.hsEST.R.Xp.pep.aa
 
 ###########################################################################
 # augustus de novo track (DONE 2007-4-5 Mario)
 
 mkdir -p /cluster/data/hg17/bed/augustus/usingHints/predictions/Xp.RA.it
 cd /cluster/data/hg17/bed/augustus/usingHints/predictions/Xp.RA.it
 wget http://augustus.gobics.de/predictions/hg17/deNovo/augustus.hg17.Xp.RA.it.gff
 wget http://augustus.gobics.de/predictions/hg17/deNovo/augustus.hg17.Xp.RA.it.pep.aa
 ldHgGene -bin hg17 augustusXRA augustus.hg17.Xp.RA.it.gff
 hgPepPred hg17 generic augustusXRAPep augustus.hg17.Xp.RA.it.pep.aa
 
 
 ###########################################################################
 # SwitchDB TSS Track (DONE 2007-04-12 Andy)
 
 ssh hgwdev
 mkdir /cluster/data/hg17/bed/switchDbTss
 cd /cluster/data/hg17/bed/switchDbTss
 # (obtained from Nathan Trinklein <nathant@switchgeargenomics.com>)
 cp ~/all_tss_switchdb_psgene.gz .
 gunzip all_tss_switchdb_psgene.gz
 cat << "EOF" > reformat.awk
 BEGIN{FS="\t"}
 {
 if (NR > 1)
    {
    if ($9 !~ "^PSEUDO.*")
        {
        pseudo = "none";
        }
    else
        {
        pseudo = $9;
        }
    printf("%s\t%d\t%d\t%s\t1000\t%s\t%s\t%s\t%s\t%s\t%s\n", $2, $8, $8+1, $6, $5, $7, $1, $3, $4, pseudo); 
    }
 }
 EOF
 awk -f reformat.awk all_tss_switchdb_psgene > switchDbTss.bed
 ln -s ~/kent/src/hg/lib/switchDbTss.sql
 hgLoadBed -sqlTable=switchDbTss.sql hg17 switchDbTss switchDbTss.bed
 
 ############################################################################
 # enable ORFeome track build.  (markd 2007-05-02)
     cd ~/kent/src/hg/makeDb/genbank
     cvs update -d etc
     # edit etc/genbank.conf to add 
         hg17.orfeomeTables.hgwdev = yes
         hg17.orfeomeTables.hgwbeta = yes
     # will need to enable for rr later.  In the future, this can just be enabled
     # as part the normal genbank build.  Change above to:
         hg18.orfeomeTables.default = yes
     
 ###########################################################################
 # Transcriptome Phase 3 tracks (Andy 2007-06-10)
 
 ssh hgwdev
 bash
 cd /san/sanVol1/scratch/andy
 mkdir transcriptome
 cd transcriptome/
 cp /var/ftp/encode/Affy_transcriptome_phase3.tar .
 tar xfv Affy_transcriptome_phase3.tar 
 find . -name '*.bz2' -exec bunzip2 '{}' \;
 cat > processWig.sh << "EOF"
 #!/bin/bash
 
 theDir=`dirname $1`;
 theFile=`basename $1`;
 table=affyTxnPhase3${theFile%.sig.wig};
 tmp=/scratch/tmp/trans3rdPhase.$$
 
 mkdir $tmp
 cp $1 $tmp
 pushd $tmp
 head -n1 $theFile > $table.sig.track.txt
 tail +2 $theFile > tmp; mv tmp $theFile
 wigEncode $theFile $table.wig $table.wib
 popd
 cp $tmp/${table}.* $theDir
 rm -rf $tmp
 EOF
 chmod +x processWig.sh
 cat > gsub << "EOF"
 #LOOP
 ./processWig.sh {check in line+ $(path1)} {check out exists $(dir1)/$(root1).track.txt}
 #ENDLOOP
 EOF
 find . -name '*.sig.wig' > wig.lst
 gensub2 wig.lst single gsub spec
 ssh pk
 cd /san/sanVol1/scratch/andy/transcriptome
 para create spec
 para push
 exit
 cd /cluster/data/hg17/bed
 mkdir transcriptome3rdPhase/{wig,wib,bed}
 cd transcriptome3rdPhase/wib/
 cp /san/sanVol1/scratch/andy/transcriptome/graphs/human_{long,short}_rna/affyTxnPhase3*.wib .
 pushd /gbdb/hg17/wib
 ln -s /cluster/data/hg17/bed/transcriptome3rdPhase/wib/* .
 popd
 cd ../wig/
 cp /san/sanVol1/scratch/andy/transcriptome/graphs/human_{long,short}_rna/affyTxnPhase3*.wig .
 for f in *; do
    hgLoadWiggle hg17 ${f%.wig} $f
 done
 cd ../bed
 for f in /san/sanVol1/scratch/andy/transcriptome/transfrags/human_{long,short}_rna/*; do
    newName=`basename $f`;
    newName=${newName%.bz2};
    bzcat $f | tail +2 > $newName;
    tab=affyTxnPhase3Frags${newName%.bed};
    hgLoadBed hg17 $tab $newName;
 done
 
 #######################################################################
 # CLEANUP OF DANRER1 BLASTZ SWAP (DONE, 2007-06-25, hartera)
    ssh kkstore02
    cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
    rm -r run1
    cd ../mafNet.new
    gzip *.maf
    # we don't tend to keep the Blastz PSLs anymore and this is an old 
    # zebrafish assembly so remove these.
    cd ../
    rm -r pslChrom
    # this removed 1.2 G of data.
 
 #######################################################################
 # CLEANUP OF ACEMBLY_050217 DATA (DONE, 2007-06-25, hartera)
    ssh kkstore02
    cd /cluster/store5/gs.18/build35/bed/acembly_050217
    rm GeneCheck.out GeneCheck2 acembly acembly.chk acembly.details \
       chrom.out genePred.tab hg16.name hg16Pep.name
    cd acembly_gene_lists
    rm test transcripts.names *.bak main_gene.list.IDsort mp.ids mp.sort ptest \
       maintest gid.tab gid.tab.sort genesGffs.ids genesGffs.ids.uniq 
    cd ../
    # remove fasta files as included in gzipped tar file
    rm -r acembly.ncbi_35.genes.proteins.fasta 
    cd acembly.ncbi_35.genes.gff
    gzip *.gff 
 
 #######################################################################
 # CLEANUP OF DANRER2 BLASTZ SWAP (DONE, 2007-06-25, hartera)
    ssh kkstore02
    cd /cluster/store5/gs.18/build35/bed/blastz.danRer2.2004-12-08
    # remove old axtChrom directory
    rm -r axtChrom.orig
    cd axtChain
    # chain directories can be recreated from all.chain files so remove
    rm -r chain chainAR
    # gzip net files
    gzip net/*.net
    # gzip .over files
    gzip over/*.over
    # removed ~1.3 G data 
 
 #############################################################################
 # Duke DNaseI HS (2007-06-26 kate)
 #
 #  Submitted by Terry Furey <terry.furey@duke.edu>
 #  in collaboration with Greg Crawford
 # Resubmitted 9/26/07 from FTP site
 # Resubmitted 10/25/07 from FTP site
 
     ssh kkstore02
     cd /cluster/data/hg17/bed
 
     # download 19GB archive from Duke site, password protected,
     # user=ucsc, password=dnase
     mkdir -p dukeDnase/2007-10-25/lab
     cd dukeDnase/2007-10-25/lab
 
     sftp ucsc@sftp.igsp.duke.edu
     mget *
     # dukeDnaseHsCd4.bed
     # dukeDnaseHsCd4Wiggle.tgz
 
     # unpack and load wiggle (signal) data
     nice tar xvfz dukeDnaseHsCd4Wiggle.tgz
     # packaged as chr*_dukeDnaseHsCd4Wiggle.out
     # fixedStep 1 files
 
     # create wiggle and load into database
     cd ..
     cat lab/chr*.out | nice wigEncode stdin \
         dukeDnaseCd4Signal.wig dukeDnaseCd4Signal.wib >&! wigencode.log &
         # upper limit 25.74, lower limit -0.66
 
     ssh hgwdev
     cd /cluster/data/hg17/bed/dukeDnase/2007-10-25
     rm -f /gbdb/hg17/wib/dukeDnaseCd4Signal.wib
     ln -s /cluster/data/hg17/bed/dukeDnase/2007-10-25/dukeDnaseCd4Signal.wib \
             /gbdb/hg17/wib
     nice hgLoadWiggle hg17 dukeDnaseCd4Signal -pathPrefix=/gbdb/hg17/wib \
                 dukeDnaseCd4Signal.wig
 
     # load bed file (sites)
     ssh hgwdev
     cd /cluster/data/hg17/bed/dukeDnase/2007-10-25/
     set table = dukeDnaseCd4Sites
     sed "s/bed5FloatScore/$table/" ~/kent/src/hg/lib/bed5FloatScore.sql > \
             $table.sql
     hgsql hg17 -e "DROP TABLE IF EXISTS $table"
     hgsql hg17 < $table.sql
     hgLoadBed -sqlTable=$table.sql hg17 $table lab/dukeDnaseHsCd4.bed
         # Loaded 95723 elements of size 6
         # min value:  0.000103164
         # max value: 25.7442
     #textHistogram -col=5 lab/dukeDnaseHsCd4.bed -binSize=50 
     300 ******************************** 11789
     350 ************************************************************ 22253
     400 ********************************************* 16854
     450 ********************************* 12333
     500 ************************* 9179
     550 ********************* 7870
     600 ************* 4987
     650 ********* 3271
     700 ******** 2789
     750 ****** 2153
     800 **** 1303
     850 ** 567
     900 * 219
     950  85
     1000  71
 
 
 ###########################################################################
 # Stanford ChIP-seq (Apr - July 2007, Heather)
 # Submitted 2007-03-14 by David Johnson <seasquirtdoctor@gmail.com>
 # 25bp tags (Solexa sequencing of IP fragments)
 # genome-wide, but funded by ENCODE, hence the location of the data
 
     ssh hgwdev
     cd /cluster/data/encode/stanford
     mkdir -p 2007-03-14/lab
     cd 2007-03-14/lab
     sort NRSF_chipseq_hg17.bed > data.bed
     sort NRSF_chipseq_control_hg17.bed > control.bed
     fix.pl < data.bed > fix.bed
     fix.pl < control.bed > control_fix.bed
     hgLoadBed hg17 stanfordNRSFEnriched fix.bed -tab
     hgLoadBed hg17 stanfordNRSFControl control_fix.bed -tab
 
 ############################################################################
 # Stanford ChIP/chip
 # Submitted 2007-07-11 by David Johnson (seasquirtdoctor@gmail.com)
 # Replaces submission from 2007-03-23
 # 12 subtracks
 # genome-wide, but funded by ENCODE, hence the location of the data
 
     ssh hgwdev
     cd /cluster/data/encode/stanford/2007-07-11/lab
     # Dave gave us bed 5, we need bed 4
     ./shrink.sh
     ./load.sh
 
 #########################################################################
 # REGULATORY POTENTIAL 7X UPDATED (DONE - 2007-08-01 - Hiram)
     #	download data from "James Taylor" <james@bx.psu.edu>
     ssh kkstore02
     mkdir /cluster/data/hg17/bed/regPotential7X.update
     cd /cluster/data/hg17/bed/regPotential7X.update
     ## In theory, only chr4, chr8, chr9 and chrY have updated, fetch them
     ## all and verify with ../regPotential7X
 for C in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y
 do
 wget --timestamping \
 "http://www.bx.psu.edu/~james/esperr_rp_7way_scores/genome_scores_hg17/chr${C}.scores.truncated.bz2"
 echo "DONE - chr${C}.scores.bz2"
 done
 
     #	create download gzip files from the bz2 files:
     time for F in chr*.scores.truncated.bz2
     do
 	C=`echo $F | awk -F'.' '{print $1}'`
 	echo -n "${C}.regPotential7X.hg17.gz working ... "
 	bzcat ${F} | gzip > ${C}.regPotential7X.hg17.gz
 	echo "done"
     done
 
     time for C in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y
     do
 	zcat chr${C}.regPotential7X.hg17.gz
     done | wigEncode -noOverlap  stdin regPotential7X.wig regPotential7X.wib
     #	Converted stdin, upper limit 1.00, lower limit -0.00
     #	real    16m51.215s
 
     #	Loading the table on hgwdev
     ssh hgwdev
     cd /cluster/data/hg17/bed/regPotential7X.update
     mkdir /gbdb/hg17/wib/061116
     ln -s /cluster/data/hg17/bed/regPotential7X.update/regPotential7X.wib \
 	/gbdb/hg17/wib/061116/regPotential7X.wib
     #	using the tmpDir is faster since it is on local disk and it will
     #	clean up any temporary .tab file it creates there
     time nice -n +19 hgLoadWiggle -tmpDir=/scratch/tmp \
 	-pathPrefix=/gbdb/hg17/wib/061116 hg17 regPotential7X regPotential7X.wig
     #	real    0m40.523s
 
     #	How about a histogram of the data.
     ssh kolossus
     cd /cluster/data/hg17/bed/regPotential7X.update
     time nice -n +19 hgWiggle -verbose=2 -doHistogram -hBinSize=0.01 \
       -hBinCount=100 -hMinVal=0.0 -db=hg17 regPotential7X > histogram.data 2>&1
     #	real    3m3.829s
     #	73 % of the data values are zero
 
 
     # renaming file directory -- kuhn 08-17-2007
     cd /gbdb/hg17/wib
     mv 061116 regPot061116
     hgsql -e " update regPotential7X SET file = \
       /gbdb/hg17/wib/regPot061116/regPotential7X.wib" hg17
     Query OK, 2366123 rows affected (31.46 sec)
     Rows matched: 2366123  Changed: 2366123  Warnings: 0
 
 ###########################################################################
 ## Create gc5Base download raw data file (DONE - 2007-08-29 - Hiram)
     ssh kkstore02
     cd /cluster/data/hg17/bed/gc5Base
     hgGcPercent -wigOut -doGaps -file=stdout -win=5 \
 	hg17 /cluster/data/hg17/hg17.2bit 2> /dev/null \
 	| gzip > hg17.gc5Base.txt.gz
     ssh hgwdev
     mkdir /usr/local/apache/htdocs/goldenPath/hg17/gc5Base
     cd /usr/local/apache/htdocs/goldenPath/hg17/gc5Base
     ln -s /cluster/data/hg17/bed/gc5Base/hg17.gc5Base.txt.gz .
 
 ############################################################################
 # INDEL-BASED CONSERVATION TRACK (DONE, 2007-10-02 - 2007-10-03, hartera)
 # Data from the Gerton Lunter (gerton.lunter@anat.ox.ac.uk), MRC 
 # Functional Genetics Unit, University of Oxford, United Kingdom.
 # Data is from the paper:
 # Lunter G, Ponting CP and Hein J Genome-wide identification of human
 # functional DNA using a neutral indel model. PLoS Comput Biol. 2006
 # Jan;2(1):e5.
     ssh kkstore02 
     mkdir -p /cluster/data/hg17/bed/consIndels/data
     cd /cluster/data/hg17/bed/consIndels/
     # Add a README.indels with the e-mail from Gerton Lunter, copy over 
     # from hg18 condIndels
     cp /cluster/data/hg18/bed/consIndels/README.indels .
     # get the data
     cd data
     wget --timestamping \
          http://wwwfgu.anat.ox.ac.uk/~gerton/IPS/IPSs.zip
     # 15 Mb zip file in GFF format. This contains data for hg17
     # comparing it to mm5 (NCBI Build 33) and 
     # canFam1 (Broad Institute, July 2004). The chr*.mm5.GFF data is old
     # data that can be removed.
     unzip IPSs.zip    
     cd /cluster/data/hg17/bed/consIndels
     rm ./data/*mm5.GFF
     foreach f (./data/*.GFF)
        set r = $f:r
        echo $r
        grep -v "track" $f > ${r}NoHeader.gff
     end
   
     # strip off the end of the name e.g. IGS0001:p=.26
     # so that the name displayed is short - IGS0001.1. The score field
     # is used to determine colouring and this is calculated from FDR 
     ssh kkstore02 
     cd /cluster/data/hg18/bed/consIndels
     perl -pi.bak -e \
          's/(IGS[0-9a-z]+\.?[0-9XY]*):p=?<?\.[0-9]+/$1/'  \
          ./data/chr*NoHeader.gff
     # check this looks ok then clean up
     rm *.bak
     # makes sense to store this as a BED5 table in order to use the score
     # for display.
     foreach f (./data/*NoHeader.gff)
        awk 'BEGIN {FS="\t"} {OFS="\t"} {print $1,$4,$5,$9,$6}' $f \
        >> consIndelsHg17Mm5CanFam1.bed
     end
  
     # load data
     ssh hgwdev
     cd /cluster/data/hg17/bed/consIndels
     hgLoadBed hg17 consIndelsHg17Mm5CanFam1 consIndelsHg17Mm5CanFam1.bed
     # Loaded 593298 elements of size 5
     
     # Get the IDs, posterior probabilities (p) for the segment being neutral, 
     # and the FDR from the original GFFs for a separate table. Some items 
     # have p<.001. Can not do Table Browser queries restricting 
     # p to <, =, or > a specified value unless all values are floats.
     # Contacted the data contributor, Gerton Lunter, and he said it would be 
     # ok to change all p<.001 to p=0.0005
     ssh kkstore02
     
     cd /cluster/data/hg17/bed/consIndels/
     awk '{if ($1 !~ /random/) print $1;}' /cluster/data/hg17/chrom.sizes \
         | sed -e 's/chr//' | sort -n > chrom.lst
     grep -v 'hap' chrom.lst > tmp2
     tail +4 tmp2 > tmp3
     echo "X\nY\n" >> chrom.lst
     rm tmp2 tmp3 
     # chrom.lst has a list of chroms 1-22, then X and Y
     foreach c (`cat chrom.lst`)
        echo $c
        foreach f (./data/chr${c}.GFF)
           echo $f
           awk 'BEGIN {FS="\t"} {OFS="\t"}{if ($9 ~ /IGS/) print $9,$6;}' $f \
               | sed -e 's/:/\t/' \
               | sed -e 's/p=\./0\./' | sed -e 's/p<\.001/0\.0005/' \
               >> consIndelsConf.txt
        end
     end
     # Add the FDR. 
     # For this set, there is no false discovery rate (FDR) field but it 
     # can be related to the score. If score is 999 then FDR is 1% (0.01) and
     # if score is 500 then FDR is 10% (0.10). Score is in column 6.
     # there are no GFF files for the haplotype chroms
    
     awk 'BEGIN {FS="\t"} {OFS="\t"} {if ($3 ~ /500/) print $1, $2, "0.10"; else if ($3 ~ /999/) print $1, $2, "0.01";}' consIndelsConf.txt > consIndelsHg17Mm5CanFam1Conf.txt 
 
     # Create a table definition for the table of identifier,  posterior 
     # probability and false discovery rate (FDR). Already created for hg18
     # track (see hg18.txt). It is $HOME/kent/src/hg/lib/itemConf.as.
    
     ssh hgwdev
     cd /cluster/data/hg17/bed/consIndels
     hgLoadSqlTab hg17 consIndelsHg17Mm5CanFam1Conf \
          $HOME/kent/src/hg/lib/itemConf.sql \
          consIndelsHg17Mm5CanFam1Conf.txt
     # check that all itesm are in this table.
     hgsql -N -e 'select distinct(name) from consIndelsHg17Mm5CanFam1;' hg17 \
          | sort > consIndels.names.sort
     hgsql -N -e 'select distinct(id) from consIndelsHg17Mm5CanFam1Conf;' hg17 \
          | sort > consIndels.idsfromConf.sort
     wc -l *.sort
     # 593298 consIndels.idsfromConf.sort
     # 593298 consIndels.names.sort
 
     comm -12 consIndels.names.sort consIndels.idsfromConf.sort | wc -l
     # 593298
     # so all element IDs are in both tables.
     # cleanup
     rm ./data/*.bak *.sort
     
     # add trackDb/human/hg17/trackDb.ra entry and add description that
     # was written by the data contributor. Add code to hgc.c to display
     # the posterior probability and the FDR on the details page for
     # track elements. Gerton Lunter provided a description for the data
     # on 2007-09-12.
     cd ~/kent/src/hg/makeDb/trackDb/human/hg17
     cp ../hg18/consIndelsHg18Mm8CanFam2.html consIndelsHg17Mm5CanFam1.html
     # check this is correct and add trackDb.ra track entry and search.  
 
 ##############################################################
 # NIMH Bipolar Genome Graphs built-in (DONE 2007-10-04 Galt)
 #
 #  See hg18.txt for details.
 #############################################################
 
 #############################################################
 # CCC Genome Graphs (DONE 2007-Sept Andy)
 # 
 # See hg18 make doc.
 
 ###############################################################
 # Affy Transcriptome Phase 3 chrY fix (DONE 2007-12-10, Andy)
 
 ssh kkstore05
 cd /cluster/store12/hg17/bed/affyTxnPhase3/raw
 zcat sRNA.affyTxnPhase3HeLaBottomStrand.wig.gz | grep -n chrY
 #256994657:variableStep chrom=chrY span=1
 zcat sRNA.affyTxnPhase3HeLaBottomStrand.wig.gz | head -n256994656 | gzip -c >tmp.wig.gz
 mv tmp.wig.gz sRNA.affyTxnPhase3HeLaBottomStrand.wig.gz
 zcat sRNA.affyTxnPhase3HeLaTopStrand.wig.gz | grep -n chrY 
 256994657:variableStep chrom=chrY span=1
 zcat sRNA.affyTxnPhase3HeLaTopStrand.wig.gz | head -n256994656 | gzip -c > tmp.wig.gz
 mv tmp.wig.gz sRNA.affyTxnPhase3HeLaTopStrand.wig.gz
 ssh kolossus
 cd /cluster/store12/hg17/bed/affyTxnPhase3/raw
 wigEncode sRNA.affyTxnPhase3HeLaBottomStrand.wig.gz affyTxnPhase3HeLaBottomStrand.{wig,wib}
 wigEncode sRNA.affyTxnPhase3HeLaTopStrand.wig.gz affyTxnPhase3HeLaTopStrand.{wig,wib}
 mv *.wig /cluster/data/hg17/bed/affyTxnPhase3/wig/
 mv *.wib /cluster/data/hg17/bed/affyTxnPhase3/wib/
 ssh hgwdev
 cd /cluster/data/hg17/bed/affyTxnPhase3/wig
 hgLoadWiggle hg17 affyTxnPhase3HeLaTopStrand{,.wig}
 hgLoadWiggle hg17 affyTxnPhase3HeLaBottomStrand{,.wig}
 
 ###########################################################################
 # Reload CCDS (2007-12-12 markd)
     # import ccds database as described in ccds.txt
     set db=hg17
     # create and load ccdsGene and ccdsInfo tables from imported database
     /cluster/data/genbank/bin/x86_64/ccdsMkTables -loadDb ccds ${db} ccdsInfo ccdsGene
 
     # ccdsKgMap
     /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap -db=${db} -loadDb ccdsGene knownGene ccdsKgMap
 
     # build initial version of ccdsMgcMap table, updated by nightly genbank update
     /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap -db=${db} -loadDb ccdsGene mgcGenes ccdsMgcMap
 
     checkTableCoords ${db} -verbose=2 ccdsGene
     # update all.jointer to include ${db} in ccdsDb
     joinerCheck -database=${db} -identifier=ccdsGeneId ~/compbio/genbank/kent/src/hg/makeDb/schema/all.joiner
     # request push of 
         ccdsGene
         ccdsInfo
         ccdsKgMap
         ccdsMgcMap
     # << emacs
 
 ############################################################################
 # ADD LINKS TO GENETESTS ON hgGene DETAILS PAGE (DONE 12/12/07 Fan)
 
 See hg18.txt for details.
 
 ############################################################################
 # Reload CCDS (2008-02-01 markd)
     # import ccds database as described in ccds.txt
     set db=hg17
     # create and load ccdsGene and ccdsInfo tables from imported database
     /cluster/data/genbank/bin/x86_64/ccdsMkTables -loadDb ccds ${db} ccdsInfo ccdsGene
 
     # ccdsKgMap
     /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap -db=${db} -loadDb ccdsGene knownGene ccdsKgMap
 
     checkTableCoords ${db} -verbose=2 ccdsGene
     # update all.jointer to include ${db} in ccdsDb
     joinerCheck -database=${db} -identifier=ccdsGeneId ~/compbio/genbank/kent/src/hg/makeDb/schema/all.joiner
     # request push of 
         ccdsGene
         ccdsInfo
         ccdsKgMap
     # << emacs
 
 ############################################################################
 # CREATE huge TABLE FOR HuGE LINK (DONE 3/6/08, Fan)
 
 # See the HuGE section in hg18.txt for details.
 ############################################################################
 
 
 ############################################################################
 # DGV V8 (DATABASE OF GENOMIC VARIANTS) (DONE 8/12/09 angie)
 # DGV V7 done 3/11/09
 # DGV V6 thin regions dropped 2/23/09
 # DGV V6 with useless thin regions done 11/12/08
 # DGV V5 done 8/11/08
 # DGV V4 done 5/9/08
     ssh hgwdev
     mkdir /hive/data/genomes/hg17/bed/dgv.v8
     cd /hive/data/genomes/hg17/bed/dgv.v8
     wget --timestamping \
       http://projects.tcag.ca/variation/downloads/variation.hg17.v8.aug.2009.txt
     wget --timestamping \
       http://projects.tcag.ca/variation/downloads/indel.hg17.v8.aug.2009.txt
     # Save previous version for comparison:
     hgsql hg17 -e 'rename table dgv to dgvV7'
     # shuffle fields into bed8+ (input has one start coord==0, but min 
     # nonzero size of 99 not 100 implies most coords are 1-based):
     foreach f (*.v8.*.txt)
       tail -n +2 $f \
       | perl -wpe 'chomp; \
         ($id, $landmark, $chr, $start, $end, $varType, \
          undef, undef, undef, $ref, $pmid, $method, \
          $gain, $loss, undef, undef, $sample) = split("\t"); \
         $id =~ s/^Variation_//; \
         $start-- unless ($start == 0); \
         $landmark = "" if ($landmark =~ /^chr.*\d\.\.\d/); \
         $rgb = ($varType =~ /^Inv/) ? "100,0,100" : "0,200,0"; \
         if ($gain ne "" || $loss ne "") { \
           $gain =~ s/^(NA)? ?$/0/;  $loss =~ s/^(NA)? ?$/0/; \
           $rgb = "200,0,0" if ($gain > 0 && $loss == 0); \
           $rgb = "0,0,200" if ($loss > 0 && $gain == 0); \
         } \
         $_ = join("\t", $chr, $start, $end, $id, 0, "+", \
                   $start, $start, $rgb, $landmark, $varType, \
                   $ref, $pmid, $method, $sample) . "\n";' \
           > $f:r.bed
     end
     hgLoadBed hg17 dgv *.bed \
       -onServer -sqlTable=$HOME/kent/src/hg/lib/dgv.sql -tab
 #Loaded 23424 elements of size 15
 
 
 ############################################################################
 # KIDD/EICHLER DISCORDANT CLONE ENDS (DONE 6/10/08 angie)
 # 8/11/08: Added kiddEichlerToNcbi (ID xref table).
     ssh kkstore02
     mkdir /cluster/data/hg17/bed/kiddEichlerDiscordant
     cd /cluster/data/hg17/bed/kiddEichlerDiscordant
     wget --user=uuuu  --password=ppppppp \
       http://eichlerlab.gs.washington.edu/kiddj/downloads/fosmids.hg17.tgz
     tar xvzf fosmids.hg17.tgz
     cd bd35
     # 8 clone-end linkedFeaturesSeries tracks and one bed custom track.
     # bed has illegal coords (maybe for unplaced ends?).
 
     # Load the tracks (translate bacEndPairs format to bed12):
     ssh hgwdev
     cd /cluster/data/hg17/bed/kiddEichlerDiscordant/bd35
     foreach f (abc*.txt)
       set track = `echo $f:r \
         | perl -wpe 's/^(G|abc)(\d+)discordant/kiddEichlerDisc\u$1$2/ || die;'`
       if ($status != 0) break
       perl -wpe 'next if s/^#.*\n$//; \
         ($c, $s, $e, $n, $sc, $st, undef, $bs, $bSt, $bSz)=split; \
         @bSts = split(",", $bSt);  @bSzs = split(",", $bSz); \
         $s--; \
         if ($n =~ /transchr/) { \
           $bs = 1; \
           $#bSts = 0;  $#bSzs = 0; \
           $bSts[0]--;  $e--; \
           $bSts[0] -= $s; \
         } elsif ($n =~ /OEA/) { \
           $bSts[0]--; \
           die "bSts[0] $bSts[0] != s $s\n" if ($bSts[0] != $s); \
           $bE = $bSts[0] + $bSzs[0]; \
           die "bE $bE != e $e\n" if ($bE != $e); \
           $bSts[0] -= $s; \
         } elsif ($bs == 2) { \
           $bSts[0]--;  $bSts[1]--; \
           if ($bSts[0] > $bSts[1]) { \
             # warn "Swapping $n ($bSts[0] > $bSts[1])\n"; \
             $tmp = $bSts[0];  $bSts[0] = $bSts[1];  $bSts[1] = $tmp; \
             $tmp = $bSzs[0];  $bSzs[0] = $bSzs[1];  $bSzs[1] = $tmp; \
           } \
           if ($bSts[0] != $s) { \
             # warn "Tweaking $n start from $s to $bSts[0]\n"; \
             $s = $bSts[0]; \
           } \
           $bE0 = $bSts[0] + $bSzs[0]; \
           $bE1 = $bSts[1] + $bSzs[1]; \
           $bE = $bE0 > $bE1 ? $bE0 : $bE1; \
           if ($bE != $e) { \
             # warn "Tweaking $n end from $e to $bE\n"; \
             $e = $bE; \
           } \
           $bSts[0] -= $s;  $bSts[1] -= $s; \
         } else { die "#blks is $bs for $n\n"; } \
         $bSt = join(",", @bSts) . ",";  $bSz = join(",", @bSzs) . ","; \
         $rgb = ($n =~ /deletion/) ? "224,0,0" : \
                ($n =~ /insertion/) ? "0,0,224" : \
                ($n =~ /inversion/) ? "0,224,0" : \
                ($n =~ /OEA/) ? "240,160,64" : "0,0,0"; \
         $_ = join("\t", $c, $s, $e, $n, $sc, $st, $s, $e, $rgb, \
                         $bs, $bSz, $bSt) . "\n";' $f \
       | hgLoadBed -tab hg17 $track stdin
     end
     perl -pe 'next if s/^track .*\n$//; \
         ($c, $s, $e, $n, $sc, $st, $tS, $tE, $r, $bs, $bSz, $bSt) = split; \
         @bSts = split(",", $bSt);  @bSzs = split(",", $bSz); \
         if ($n =~ /transchr/) { \
           $bs = 1; \
           $#bSts = 0;  $#bSzs = 0; \
         } elsif ($n =~ /OEA/) { \
           $s--; # weird that this is required only for OEA here \
           die "$n: bSts[0] $bSts[0] != 0\n" if ($bSts[0] != 0); \
           $bE = $s + $bSts[0] + $bSzs[0]; \
           die "$n: bE $bE != e $e\n" if ($bE != $e); \
         } elsif ($bs == 2) { \
           $bSts[0] += $s;  $bSts[1] += $s; \
           if ($bSts[0] > $bSts[1]) { \
             # warn "Swapping $n ($bSts[0] > $bSts[1])\n"; \
             $tmp = $bSts[0];  $bSts[0] = $bSts[1];  $bSts[1] = $tmp; \
             $tmp = $bSzs[0];  $bSzs[0] = $bSzs[1];  $bSzs[1] = $tmp; \
           } \
           if ($bSts[0] != $s) { \
             # warn "Tweaking $n start from $s to $bSts[0]\n"; \
             $s = $bSts[0]; \
           } \
           $bE0 = $bSts[0] + $bSzs[0]; \
           $bE1 = $bSts[1] + $bSzs[1]; \
           $bE = $bE0 > $bE1 ? $bE0 : $bE1; \
           if ($bE != $e) { \
             # warn "Tweaking $n end from $e to $bE\n"; \
             $e = $bE; \
           } \
          $bSts[0] -= $s;  $bSts[1] -= $s; \
         } else { die "#blks is $bs\n"; } \
         $bSt = join(",", @bSts) . ",";  $bSz = join(",", @bSzs) . ","; \
         $tS = $s;  $tE = $e; \
         $rgb = ($n =~ /deletion/) ? "224,0,0" : \
                ($n =~ /insertion/) ? "0,0,224" : \
                ($n =~ /inversion/) ? "0,224,0" : \
                ($n =~ /OEA/) ? "240,160,64" : "0,0,0"; \
         $_ = join("\t", $c, $s, $e, $n, $sc, $st, $tS, $tE, $rgb, \
                         $bs, $bSz, $bSt) . "\n";' G248discordant.txt \
     | hgLoadBed -tab hg17 kiddEichlerDiscG248 \
         stdin
 
     # 8/11/08: get clone ID -> NCBI acc mapping.
     ssh kkstore02
     mkdir /cluster/data/hg17/bed/kiddEichlerDiscordant/cloneIds
     cd /cluster/data/hg17/bed/kiddEichlerDiscordant/cloneIds
     # Saved off emailed file from Jeff Kidd to clones_used_3nov.txt.accessions;
     # get trace archive trace names for end reads:
     foreach n (7 9 10 11 12 13 14)
       wget http://hgsv.washington.edu/general/download/clone_mapping/ABC$n/ABC$n.conversion.gz
     end
     # ABC8 has _a and _b files:
     wget http://hgsv.washington.edu/general/download/clone_mapping/ABC8/ABC8_a.conversion.gz
     wget http://hgsv.washington.edu/general/download/clone_mapping/ABC8/ABC8_b.conversion.gz
     # That file is not available for G248.
     gunzip *.gz
     # Combine the relevant data from the .conversion files; keep only those
     # IDs that are used in the tracks.
     cut -f 4 ../bd35/*discordant.txt \
     | egrep -v '^(#chrom|track|name)' \
     | sed -e 's/,.*//' \
     | sort -u > discIds.txt
     perl -wpe 's/^OurClone.*\n// || s/^\d+_(HUMAN|\d+_).*\n$// || \
       s/^(\d+_)?(ABC|G)(\d+)(_\d\d?)?(_\d\d?)?_0*(\d+?_[A-Z]\d\d?)\.(F|FORWARD|R|REVERSE)\.\d+\t(\w+)$/$2$3_$6\t$7\t$8/ || \
       warn "Parse line $.:\n$_";' \
       *.conversion \
     | sort > allEnds.tab
     grep -wFf discIds.txt allEnds.tab > discEnds.txt
     wc -l discIds.txt allEnds.tab discEnds.txt 
 #   223051 discIds.txt
 # 17498527 allEnds.tab
 #   573974 discEnds.txt
     # discEnds.txt has 2 lines (forward & reverse) for most of its ids...
     # ideally we would see 2*(223051) lines in discEnds.txt.
     # Get a list of which discordant clone IDs don't have ends in *.conv*:
     cut -f 1 allEnds.tab | uniq > all.tmp
     comm -23 discIds.txt all.tmp > discNotInConv.txt
     wc -l discNotInConv.txt
 #16318 discNotInConv.txt
     cat > combine.pl <<'_EOF_'
 #!/usr/bin/perl -w
 use strict;
 my ($cloneFile, $endsFile) = @ARGV;
 open(CLONES, $cloneFile) || die "Can't open $cloneFile: $!\n";
 my %idInfo;
 while(<CLONES>) {
   (s/^(\d+_)?(ABC|G)(\d+)(_\d\d?)?(_\d\d?)?_0*(\d+?_[A-Z]\d\d?)\t(\w+)$/$2$3_$6\t$7/ && m/^(\w+)\t(\w+)/) || \
   m/^(G248\w+)\t(\w+)$/ || die "Parse line $.:$_";
   my ($id, $acc) = ($1, $2);
   $idInfo{$id}->[0] = $acc;
 }
 close(CLONES);
 open(ENDS, $endsFile) || die "Can't open $endsFile: $!\n";
 while (<ENDS>) {
   chomp; my ($id, $dir, $traceName) = split("\t");
   if ($dir =~ /^F/) {
     $idInfo{$id}->[1] = $traceName;
   } elsif ($dir =~ /^R/) {
     $idInfo{$id}->[2] = $traceName;
   } else { die "What is this \$dir: $dir ?\n"; }
 }
 close(ENDS);
 foreach my $id (sort keys %idInfo) {
   my $infoRef = $idInfo{$id};
   $infoRef->[0] = '' if (! defined $infoRef->[0]);
   $infoRef->[1] = 0 if (! defined $infoRef->[1]);
   $infoRef->[2] = 0 if (! defined $infoRef->[2]);
   print join("\t", $id, @{$infoRef}) . "\n";
 }
 '_EOF_'
     # << emacs
     chmod a+x combine.pl
     combine.pl clones_used_3nov.txt.accessions discEnds.txt \
     | sort > kiddEichlerToNcbi.txt
     # Load table:
     ssh hgwdev
     cd /cluster/data/hg17/bed/kiddEichlerDiscordant/cloneIds
     hgLoadSqlTab hg17 kiddEichlerToNcbi \
       $HOME/kent/src/hg/lib/kiddEichlerToNcbi.sql kiddEichlerToNcbi.txt
     # Add to makeDb/schema/all.joiner, then check:
     runJoiner.csh hg17 kiddEichlerToNcbi $HOME/kent/src/hg/makeDb/schema
 
 
 ############################################################################
 # TRANSMAP vertebrate.2008-05-20 build  (2008-05-24 markd)
 
 vertebrate-wide transMap alignments were built  Tracks are created and loaded
 by a single Makefile. This is available from:
    svn+ssh://hgwdev.cse.ucsc.edu/projects/compbio/usr/markd/svn/projs/transMap/tags/vertebrate.2008-05-20
 
 see doc/builds.txt for specific details.
 ############################################################################
 
 
 ############################################################################
 # KIDD/EICHLER VALIDATED SITES (DONE 6/11/08 angie)
     ssh hgwdev
     mkdir /cluster/data/hg17/bed/kiddEichlerValid
     cd /cluster/data/hg17/bed/kiddEichlerValid
     wget http://hgsv.washington.edu/general/download/validated_sites/Kidd_2008_sample_level_valided_sites.xls
     # Open in Excel, save as Kidd_2008_sample_level_valided_sites.txt,
     # move first 9 lines to Kidd_2008_sample_level_valided_sites.header.
     # Split into one file per individual:
     foreach id (Abc7 Abc8 Abc9 Abc10 Abc11 Abc12 Abc13 Abc14 G248)
       set ID = `echo $id | tr 'a-z' 'A-Z'`
       grep ${ID}_ Kidd_2008_sample_level_valided_sites.txt \
       | perl -wpe 'chomp; s/\r//; ($c, $s, $e, $n, $t) = split; \
         $rgb = ($n =~ /deletion/) ? "224,0,0" : \
                ($n =~ /insertion/) ? "0,0,224" : \
                ($n =~ /inversion/) ? "0,224,0" : "0,0,0"; \
         $t =~ s/:/,/g; \
         $n =~ s/^'$ID'_//;  $n = "$n,$t"; \
         $_ = join("\t", $c, $s, $e, $n, "0", "+", $s, $e, $rgb) . \
                   "\n";' \
       | hgLoadBed -tab hg17 kiddEichlerValid$id stdin
     end
 
 
 ################################################
 # SPLIT EXPRESSION & REGULATION GROUPS
 # (2008-09-09 kate)
 
 echo "insert into grp (name, label, priority, defaultIsClosed) values ('expression', 'Expression', 4.5, 1)" | hgsql hg17
 echo "update grp set label='Regulation' where name='regulation'" | hgsql hg17
 
 ################################################
 # AUTOMATE UPSTREAM FILE CREATION (2008-10-15 markd)
 update genbank.conf:
 hg17.upstreamGeneTbl = refGene
 hg17.upstreamMaf = multiz17way /hive/data/genomes/hg17/bed/multiz17way/species.lst
 
 
 #############################################################################
 # MAKE PCR TARGET FOR UCSC GENES (DONE 11/4/08 angie)
     ssh hgwdev
     mkdir /cluster/data/hg17/bed/mrnaPcr
     cd /cluster/data/hg17/bed/mrnaPcr
     # First, get consistent FA and PSL for UCSC Genes.
     genePredToBed /cluster/data/hg17/bed/kgHg17F/try3/kg3Try3.gp > ucscGenes.bed
     hgsql hg17 -NBe 'select kgId,geneSymbol from kgXref' \
     | perl -wpe 's/^(\S+)\t(\S+)/$1\t${1}__$2/ || die;' \
       > idSub.txt
     subColumn 4 ucscGenes.bed idSub.txt ucscGenesIdSubbed.bed
     sequenceForBed -keepName -db=hg17 -bedIn=ucscGenesIdSubbed.bed \
       -fastaOut=stdout \
     | faToTwoBit -ignoreDups stdin kgTargetSeq.2bit
     cut -f 1-10 /cluster/data/hg17/bed/kgHg17F/try3/kg3Try3.gp \
     | genePredToFakePsl hg17 stdin kgTargetAli.psl /dev/null
 
     # Load up the UCSC Genes target PSL table and put 2bit in /gbdb::
     cd /cluster/data/hg17/bed/mrnaPcr
     hgLoadPsl hg17 kgTargetAli.psl
     mkdir /gbdb/hg17/targetDb
     ln -s /cluster/data/hg17/bed/mrnaPcr/kgTargetSeq.2bit /gbdb/hg17/targetDb/
 
     # Ask cluster-admin to start an untranslated, -stepSize=5 gfServer on
     # /gbdb/hg17/targetDb/kgTargetSeq.2bit .
 
     ssh hgwdev
     # Add records to hgcentraltest blatServers and targetDb:
     hgsql hgcentraltest -e \
       'INSERT into blatServers values ("hg17Kg", "blat13", 17797, 0, 1);'
     hgsql hgcentraltest -e \
       'INSERT into targetDb values("hg17Kg", "UCSC Genes", \
          "hg17", "kgTargetAli", "", "", \
          "/gbdb/hg17/targetDb/kgTargetSeq.2bit", 1, now(), "");'
 
 
 #############################################################################
 # fox2ClipSeq from Gene Yeo (DONE - 2009-01-08 - Hiram)
     mkdir /hive/data/genomes/hg17/bed/fox2ClipSeq
     cd /hive/data/genomes/hg17/bed/fox2ClipSeq
     #	fetch data
     wget --timestamping \
 'http://www.snl.salk.edu/~geneyeo/stuff/FOX2.rmsk.BED.gz' \
         -O FOX2.rmsk.BED.gz
     #	remove track line and sort
     zcat FOX2.rmsk.BED.gz | grep -v "^track" | sort -k1,1 -k2,2n \
         | gzip > sorted.bed.gz
     #	separate strand data, and turn the positive into blue
     zcat sorted.bed.gz | awk '$6 == "+"' | sed -e "s/255,0,0/0,0,255/" \
 	| gzip > forwardStrand.bed.gz
     zcat sorted.bed.gz | awk '$6 == "-"' | gzip > reverseStrand.bed.gz
     #	turn into wiggle density plot
     zcat forwardStrand.bed.gz | bedItemOverlapCount hg17 stdin \
         | wigEncode stdin fox2ClipSeqDensityForwardStrand.wig \
 	fox2ClipSeqDensityForwardStrand.wib
     #	Converted stdin, upper limit 2401.00, lower limit 1.00
     zcat reverseStrand.bed.gz | bedItemOverlapCount hg17 stdin \
         | wigEncode stdin fox2ClipSeqDensityReverseStrand.wig \
 		fox2ClipSeqDensityReverseStrand.wib
     #	Converted stdin, upper limit 1406.00, lower limit 1.00
     #	and load tables
     zcat forwardStrand.bed.gz reverseStrand.bed.gz \
 	| hgLoadBed hg17 fox2ClipSeq stdin
     #	Loaded 4418298 elements of size 9
     ln -s `pwd`/*.wib /gbdb/hg17/wib
     hgLoadWiggle hg17 fox2ClipSeqDensityForwardStrand \
 	fox2ClipSeqDensityForwardStrand.wig
     hgLoadWiggle hg17 fox2ClipSeqDensityReverseStrand \
 	fox2ClipSeqDensityReverseStrand.wig
     #	add composite track definitions to makeDb/trackDb/human/trackDb.ra
 
 #############################################################################
 # LIFTOVER TO Hg19 (DONE - 2009-04-24 - Hiram )
     mkdir /hive/data/genomes/hg17/bed/blat.hg19.2009-04-24
     cd /hive/data/genomes/hg17/bed/blat.hg19.2009-04-24
     # -debug run to create run dir, preview scripts...
     doSameSpeciesLiftOver.pl -debug hg17 hg19
     # Real run:
     time nice -n +19 doSameSpeciesLiftOver.pl -verbose=2 \
 	-bigClusterHub=pk -dbHost=hgwdev -workhorse=hgwdev \
 	 hg17 hg19 > do.log 2>&1 &
     #	real    84m19.022s
 
 #############################################################################
+############################################################################
+# TRANSMAP vertebrate.2009-09-13 build  (2009-09-20 markd)
+
+vertebrate-wide transMap alignments were built  Tracks are created and loaded
+by a single Makefile. This is available from:
+   svn+ssh://hgwdev.cse.ucsc.edu/projects/compbio/usr/markd/svn/projs/transMap/tags/vertebrate.2009-09-13
+
+see doc/builds.txt for specific details.
+############################################################################

