f7577fc37cbcba25eac796f12e81ac0235f77c0c jeltje.van.baren Tue Jan 21 11:03:16 2025 -0800 fixing wrongly named copy of alphaMissense.txt diff --git src/hg/makeDb/doc/hg38/hg38.txt src/hg/makeDb/doc/hg38/hg38.txt index 20b6e2a715a..1f3dc1234db 100644 --- src/hg/makeDb/doc/hg38/hg38.txt +++ src/hg/makeDb/doc/hg38/hg38.txt @@ -1,14 +1,7390 @@ +# for emacs: -*- mode: sh; -*- + +# This file describes how we made the browser database on +# NCBI build 38 (December 2013 freeze) aka: +# GRCh38 - Genome Reference Consortium Human Reference 38 +# Assembly Accession: GCA_000001405.2 + +############################################################################# +## Download sequence - DONE - 2013-12-24 + mkdir /hive/data/genomes/hg38 + mkdir /hive/data/genomes/hg38/genbank + cd /hive/data/genomes/hg38/genbank + time rsync -a -P rsync://ftp.ncbi.nlm.nih.gov/genbank/genomes/Eukaryotes/vertebrates_mammals/Homo_sapiens/GRCh38/ ./ +# sent 19643 bytes received 4914689807 bytes 4490369.53 bytes/sec +# total size is 4914019581 speedup is 1.00 + +# real 18m14.497s + +############################################################################# +## convert to UCSC names - DONE - 2013-12-24 +# with this release, NCBI has adopted a naming convention that is similar +# to UCSC. The delivered sequence with these names can be found in: +# /hive/data/genomes/hg38/genbank/seqs_for_alignment_pipelines/ +# +# The following scripts reproduce this naming scheme from the separate +# files in the release +# + mkdir /hive/data/genomes/hg38/ucsc + cat << '_EOF_' > ucscCompositeAgp.pl +#!/bin/env perl + +use strict; +use warnings; + +my %accToChr; + +open (FH, "<../genbank/Primary_Assembly/assembled_chromosomes/chr2acc") or + die "can not read Primary_Assembly/assembled_chromosomes/chr2acc"; +while (my $line = <FH>) { + next if ($line =~ m/^#/); + chomp $line; + my ($chrN, $acc) = split('\s+', $line); + $accToChr{$acc} = $chrN; +} +close (FH); + +foreach my $acc (keys %accToChr) { + my $chrN = $accToChr{$acc}; + print "$acc $accToChr{$acc}\n"; + open (FH, "zcat ../genbank/Primary_Assembly/assembled_chromosomes/AGP/chr${chrN}.comp.agp.gz|") or die "can not read chr${chrN}.comp.agp.gz"; + open (UC, ">chr${chrN}.agp") or die "can not write to chr${chrN}.agp"; + while (my $line = <FH>) { + if ($line =~ m/^#/) { + print UC $line; + } else { + $line =~ s/^$acc/chr${chrN}/; + print UC $line; + } + } + close (FH); + close (UC); + open (FH, "zcat ../genbank/Primary_Assembly/assembled_chromosomes/FASTA/chr${chrN}.fa.gz|") or die "can not read chr${chrN}.fa.gz"; + open (UC, ">chr${chrN}.fa") or die "can not write to chr${chrN}.fa"; + while (my $line = <FH>) { + if ($line =~ m/^>/) { + printf UC ">chr${chrN}\n"; + } else { + print UC $line; + } + } + close (FH); + close (UC); +} +'_EOF_' + # << happy emacs + chmod +x ucscCompositeAgp.pl + + cat << '_EOF_' > unlocalized.pl +#!/bin/env perl + +use strict; +use warnings; + +my %accToChr; +my %chrNames; + +open (FH, "<../genbank/Primary_Assembly/unlocalized_scaffolds/unlocalized.chr2scaf") or + die "can not read Primary_Assembly/unlocalized_scaffolds/unlocalized.chr2scaf"; +while (my $line = <FH>) { + next if ($line =~ m/^#/); + chomp $line; + my ($chrN, $acc) = split('\s+', $line); + $acc =~ s/\./v/; + $accToChr{$acc} = $chrN; + $chrNames{$chrN} += 1; +} +close (FH); + +foreach my $chrN (keys %chrNames) { + my $agpFile = "../genbank/Primary_Assembly/unlocalized_scaffolds/AGP/chr$chrN.unlocalized.scaf.agp.gz"; + my $fastaFile = "../genbank/Primary_Assembly/unlocalized_scaffolds/FASTA/chr$chrN.unlocalized.scaf.fa.gz"; + open (FH, "zcat $agpFile|") or die "can not read $agpFile"; + open (UC, ">chr${chrN}_random.agp") or die "can not write to chr${chrN}_random.agp"; + while (my $line = <FH>) { + if ($line =~ m/^#/) { + print UC $line; + } else { + chomp $line; + my (@a) = split('\t', $line); + my $acc = $a[0]; + $acc =~ s/\./v/; + die "ERROR: chrN $chrN not correct for $acc" + if ($accToChr{$acc} ne $chrN); + my $ucscName = "chr${chrN}_${acc}_random"; + printf UC "%s", $ucscName; + for (my $i = 1; $i < scalar(@a); ++$i) { + printf UC "\t%s", $a[$i]; + } + printf UC "\n"; + } + } + close (FH); + close (UC); + printf "chr%s\n", $chrN; + open (FH, "zcat $fastaFile|") or die "can not read $fastaFile"; + open (UC, ">chr${chrN}_random.fa") or die "can not write to chr${chrN}_random.fa"; + while (my $line = <FH>) { + if ($line =~ m/^>/) { + chomp $line; + my $acc = $line; + $acc =~ s/.*gb\|//; + $acc =~ s/. Homo.*//; + $acc =~ s/\./v/; + die "ERROR: chrN $chrN not correct for $acc" + if ($accToChr{$acc} ne $chrN); + my $ucscName = "chr${chrN}_${acc}_random"; + printf UC ">$ucscName\n"; + } else { + print UC $line; + } + } + close (FH); + close (UC); +} +'_EOF_' + # << happy emacs + chmod +x unlocalized.pl + + cat << '_EOF_' > unplaced.pl +#!/bin/env perl + +use strict; +use warnings; + +my $agpFile = "../genbank/Primary_Assembly/unplaced_scaffolds/AGP/unplaced.scaf.agp.gz"; +my $fastaFile = "../genbank/Primary_Assembly/unplaced_scaffolds/FASTA/unplaced.scaf.fa.gz"; +open (FH, "zcat $agpFile|") or die "can not read $agpFile"; +open (UC, ">chrUn.agp") or die "can not write to chrUn.agp"; +while (my $line = <FH>) { + if ($line =~ m/^#/) { + print UC $line; + } else { + $line =~ s/\./v/; + printf UC "chrUn_%s", $line; + } +} +close (FH); +close (UC); + +open (FH, "zcat $fastaFile|") or die "can not read $fastaFile"; +open (UC, ">chrUn.fa") or die "can not write to chrUn.fa"; +while (my $line = <FH>) { + if ($line =~ m/^>/) { + chomp $line; + $line =~ s/.*gb\|//; + $line =~ s/. Homo.*//; + $line =~ s/\./v/; + printf UC ">chrUn_$line\n"; + } else { + print UC $line; + } +} +close (FH); +close (UC); +'_EOF_' + # << happy emacs + chmod +x unplaced.pl + + cat << '_EOF_' > altSequence.pl +#!/usr/bin/env perl + +use strict; +use warnings; +use File::Basename; + +open (AG, ">chrAlt.agp") or die "can not write to chrAlt.agp"; +open (FA, ">chrAlt.fa") or die "can not write to chrAlt.fa"; +open (FH, "find ../genbank/ALT* -type f | grep alt_scaffold_placement.txt|") or die "can not find alt_scaffold_placement.txt files"; +while (my $file = <FH>) { + chomp $file; + my $dirName = dirname($file); + my $agpFile = "$dirName/AGP/alt.scaf.agp.gz"; + my $fastaFile = "$dirName/FASTA/alt.scaf.fa.gz"; + # key is genbank acc name, value is UCSC chr name + my %nameDelta; +# printf STDERR "# %s\n", $file; + open (AL, "<$file") or die "can not read $file"; + while (my $line = <AL>) { + next if ($line =~ m/^#/); + chomp $line; + my ($alt_asm_name, $prim_asm_name, $alt_scaf_name, $alt_scaf_acc, + $parent_type, $parent_name, $parent_acc, $region_name, $ori, + $alt_scaf_start, $alt_scaf_stop, $parent_start, $parent_stop, + $alt_start_tail, $alt_stop_tail) = split('\t', $line); + my $ucscAcc = $alt_scaf_acc; + $ucscAcc =~ s/\./v/; + my $ucscName = sprintf("chr%s_%s_alt", $parent_name, $ucscAcc); + printf "%s %s\n", $alt_scaf_acc, $ucscName; + if (exists ($nameDelta{$alt_scaf_acc})) { + die "duplicate name incorrect ? $alt_scaf_acc $nameDelta{$alt_scaf_acc} ne $ucscName" if ($nameDelta{$alt_scaf_acc} ne $ucscName); + } else { + $nameDelta{$alt_scaf_acc} = $ucscName; + } + } + close (AL); + open (AL, "zcat $agpFile|") or die "can not read $agpFile"; + while (my $line = <AL>) { + if ($line =~ m/^#/) { + print AG "$line"; + } else { + my ($acc, $rest) = split('\t', $line, 2); + die "can not find ucsc name for $acc" if (!exists($nameDelta{$acc})); + printf AG "%s\t%s", $nameDelta{$acc}, $rest; + } + } + close (AL); + open (AL, "zcat $fastaFile|") or die "can not read $fastaFile"; + while (my $line = <AL>) { + chomp $line; + if ($line =~ m/^>/) { + $line =~ s/.*gb.//; + $line =~ s/. Homo.*//; + die "can not find ucsc name for $line" if (!exists($nameDelta{$line})); + printf FA ">%s\n", $nameDelta{$line}; + } else { + printf FA "%s\n", $line; + } + } + close (AL); +} +close (FH); +close (AG); +close (FA); +'_EOF_' + # << happy emacs + chmod +x altSequence.pl + + ./ucscCompositeAgp.pl + ./unlocalized.pl + ./unplaced.pl + ./altSequence.pl + + # temporarily verify the fasta and AGP are complete and compatible + faToTwoBit chr*.fa hg38.test.2bit + cat chr*.agp > hg38.agp + checkAgpAndFa hg38.agp hg38.test.2bit 2>&1 | tail -1 +# All AGP and FASTA entries agree - both files are valid + + rm -f hg38.agp hg38.test.2bit + + # comparing faCounts of this 2bit file and the sequences delivered + # in genbank/seqs_for_alignment_pipelines/ + # result in the exact same sequence + +############################################################################# +## initial db build - DONE - 2013-12-24 - Hiram + + cd /hive/data/genomes/hg38 + cat << '_EOF_' > hg38.config.ra +# Config parameters for makeGenomeDb.pl: +db hg38 +scientificName Homo sapiens +commonName Human +assemblyDate Dec. 2013 +assemblyLabel GRCh38 Genome Reference Consortium Human Reference 38 (GCA_000001405.2) +assemblyShortLabel GRCh38 +orderKey 13 +mitoAcc none +fastaFiles /hive/data/genomes/hg38/ucsc/chr*.fa +agpFiles /hive/data/genomes/hg38/ucsc/chr*.agp +# qualFiles /dev/null +dbDbSpeciesDir human +photoCreditURL http://www.cbse.ucsc.edu/ +photoCreditName Graphic courtesy of CBSE +ncbiGenomeId 51 +ncbiAssemblyId 883148 +ncbiAssemblyName GRCh38 +ncbiBioProject 31257 +genBankAccessionID GCA_000001305.2 +taxId 9606 +'_EOF_' + # << happy emacs + + # step wise to first verify AGP and Fasta files + time makeGenomeDb.pl -stop=agp hg38.config.ra > agp.log 2>&1 + + # looking good, continue: + time makeGenomeDb.pl -continue=db hg38.config.ra > db.log 2>&1 + + # add the files produced by the trackDb build to the source tree + + # this path is fixed in the makeGenomeDb.pl for next time + # honor new convention for bbi location files: + cd /gbdb/hg38/bbi + mkdir gc5BaseBw + mv gc5Base.bw gc5BaseBw + cd gc5BaseBw + # before + hgsql -e 'select * from gc5BaseBw;' hg38 +# +---------------------------+ +# | fileName | +# +---------------------------+ +# | /gbdb/hg38/bbi/gc5Base.bw | +# +---------------------------+ + # and fixed + hgBbiDbLink hg38 gc5BaseBw `pwd`/gc5Base.bw + hgsql -e 'select * from gc5BaseBw;' hg38 +# +-------------------------------------+ +# | fileName | +# +-------------------------------------+ +# | /gbdb/hg38/bbi/gc5BaseBw/gc5Base.bw | +# +-------------------------------------+ + +############################################################################# +## RepeatMasker with CrossMatch - DONE - 2013-12-24,27 - Hiram + mkdir /hive/data/genomes/hg38/bed/repeatMaskerCM + cd /hive/data/genomes/hg38/bed/repeatMaskerCM + # running this step wise so it can be loaded into its own table + time doRepeatMasker.pl -stop=mask -bigClusterHub=ku \ + -workhorse=hgwdev -dbHost=hgwdev -buildDir=`pwd` hg38 > mask.log 2>&1 + # real 3443m13.026s +# RepeatMasker version June 20 2013 open-4.0.3 +# Search Engine: cross-match version 1.090518 +# RepeatMasker Database: 20130422 + + # take the install script from this -debug run and alter it to load + # the table into rmskCM + time doRepeatMasker.pl -continue=install -stop=install -debug \ + -bigClusterHub=ku -workhorse=hgwdev -dbHost=hgwdev -buildDir=`pwd` hg38 + cat fb.hg38.rmskCM.txt + # 1586326530 bases of 3209286105 (49.429%) in intersection + + # profile of repeat elements: +# 1852545 rmskClass/SINE.tab +# 1570523 rmskClass/LINE.tab +# 748597 rmskClass/LTR.tab +# 703682 rmskClass/Simple_repeat.tab +# 499108 rmskClass/DNA.tab +# 102856 rmskClass/Low_complexity.tab +# 7962 rmskClass/Satellite.tab +# 5750 rmskClass/Retroposon.tab +# 5667 rmskClass/LTR?.tab +# 5622 rmskClass/Unknown.tab +# 4516 rmskClass/snRNA.tab +# 3294 rmskClass/DNA?.tab +# 2026 rmskClass/tRNA.tab +# 1840 rmskClass/rRNA.tab +# 1784 rmskClass/RC.tab +# 1672 rmskClass/srpRNA.tab +# 1420 rmskClass/scRNA.tab +# 704 rmskClass/RNA.tab +# 411 rmskClass/RC?.tab +# 38 rmskClass/SINE?.tab + + # using this RM result with trfMask for the final masked sequence + cd /hive/data/genomes/hg38 + twoBitMask hg38.rmskCM.2bit -add bed/simpleRepeat/trfMask.bed hg38.2bit + twoBitToFa hg38.2bit stdout | faSize stdin > faSize.hg38.2bit.txt +# 3209286105 bases (159970322 N's 3049315783 real 1460684798 upper 1588630985 lower) in 455 sequences in 1 files +# %49.50 masked total, %52.10 masked real + + featureBits -countGaps hg38 rmskCM '!rmskHmmer' -bed=crossMatchUnique.bed + # 24868153 bases of 3209286105 (0.775%) in intersection + hgLoadBed hg38 crossMatchUnique crossMatchUnique.bed + # Read 2352219 elements of size 4 from crossMatchUnique.bed + +############################################################################# +## repeating RepeatMasker Blastn run (DONE - 2014-01-07 - Hiram) + mkdir /hive/data/genomes/hg38/bed/rmskBlastn + cd /hive/data/genomes/hg38/bed/rmskBlastn + + time $HOME/kent/src/hg/utils/automation/doRepeatMasker.pl \ + -useRMBlastn -bigClusterHub=ku -workhorse=hgwdev -dbHost=hgwdev \ + -stop=mask -buildDir=`pwd` hg38 > mask.log + # real 203m33.670s + +# 3209286105 bases (159970322 N's 3049315783 real 1491207906 upper 1558107877 lower) in 455 sequences in 1 files +# %48.55 masked total, %51.10 masked real + + # install step with debug so the script can be altered to load into + # a specific rmskBlastn table: + + $HOME/kent/src/hg/utils/automation/doRepeatMasker.pl \ + -useRMBlastn -bigClusterHub=ku -workhorse=hgwdev -dbHost=hgwdev \ + -continue=install -debug -buildDir=`pwd` hg38 + +############################################################################# +## repeating RepeatMasker cross-match run (DONE - 2014-01-07 - Hiram) + mkdir /hive/data/genomes/hg38/bed/rmskCM + cd /hive/data/genomes/hg38/bed/rmskCM + + # missed recording stderr .... forgot the 2>&1 + time $HOME/kent/src/hg/utils/automation/doRepeatMasker.pl \ + -bigClusterHub=ku -workhorse=hgwdev -dbHost=hgwdev \ + -stop=mask -buildDir=`pwd` hg38 > mask.log + # real 1897m33.517s + # running from Tue Jan 7 16:10:33 PST 2014 thru 08 Jan 23:48 +# *** All done! (through the 'mask' step) - Elapsed time: 1897m34s +# *** Steps were performed in /hive/data/genomes/hg38/bed/rmskCM + # running install manually to allow edit of the script to load + # a specific rmskCm table + time $HOME/kent/src/hg/utils/automation/doRepeatMasker.pl \ + -bigClusterHub=ku -workhorse=hgwdev -dbHost=hgwdev \ + -continue=install -stop=install -buildDir=`pwd` hg38 -debug + +############################################################################# +## RepeatMasker with RM Blastn - DONE - 2013-12-24,25 - Hiram + mkdir /hive/data/genomes/hg38/bed/repeatMaskerBlastn + cd /hive/data/genomes/hg38/bed/repeatMaskerBlastn + # running this step wise so it can be loaded into its own table + time doRepeatMasker.pl -stop=mask -useRMBlastn -bigClusterHub=ku \ + -workhorse=hgwdev -dbHost=hgwdev -buildDir=`pwd` hg38 > mask.log 2>&1 + # real 354m55.842s + + # take the install script from this -debug run and alter it to load + # the table into rmskBlastn + doRepeatMasker.pl -useRMBlastn -bigClusterHub=ku -continue=install \ + -stop=install -debug -workhorse=hgwdev -dbHost=hgwdev -buildDir=`pwd` hg38 + # 1560264046 bases of 3209286105 (48.617%) in intersection + # profile of repeat elements: +# 1824560 rmskClass/SINE.tab +# 1552814 rmskClass/LINE.tab +# 738435 rmskClass/LTR.tab +# 715998 rmskClass/Simple_repeat.tab +# 486591 rmskClass/DNA.tab +# 105026 rmskClass/Low_complexity.tab +# 7712 rmskClass/Satellite.tab +# 5638 rmskClass/Retroposon.tab +# 5276 rmskClass/Unknown.tab +# 5100 rmskClass/LTR?.tab +# 4548 rmskClass/snRNA.tab +# 3033 rmskClass/DNA?.tab +# 1987 rmskClass/tRNA.tab +# 1809 rmskClass/rRNA.tab +# 1710 rmskClass/RC.tab +# 1633 rmskClass/srpRNA.tab +# 1428 rmskClass/scRNA.tab +# 614 rmskClass/RNA.tab +# 376 rmskClass/RC?.tab +# 38 rmskClass/SINE?.tab +# 3 rmskClass/Unspecified.tab +# 5464329 total + +############################################################################# +## repeating RepeatMasker run with HMMER - DONE - 2014-01-08 - Hiram + mkdir /hive/data/genomes/hg38/bed/rmskHmmer + cd /hive/data/genomes/hg38/bed/rmskHmmer + + # trying cpu=4 and ram=32g + time $HOME/kent/src/hg/utils/automation/doRepeatMasker.pl \ + -stop=mask -useHMMER -bigClusterHub=ku \ + -workhorse=hgwdev -dbHost=hgwdev -buildDir=`pwd` hg38 > mask.log 2>&1 + # 6 jobs required more than 32 Gb of memory to complete, ran them on + # hgwdev to complete, then continuing: + time $HOME/kent/src/hg/utils/automation/doRepeatMasker.pl \ + -continue=cat -stop=mask -useHMMER -bigClusterHub=ku \ + -workhorse=hgwdev -dbHost=hgwdev -buildDir=`pwd` hg38 > cat.log 2>&1 + # real 24m5.274s +# 3209286105 bases (159970322 N's 3049315783 real 1314916231 upper 1734399552 lower) in 455 sequences in 1 files +# %54.04 masked total, %56.88 masked real + + # running install manually to allow edit of the script to load + # a specific rmskHmmer table + time $HOME/kent/src/hg/utils/automation/doRepeatMasker.pl \ + -continue=install -debug -useHMMER -bigClusterHub=ku \ + -workhorse=hgwdev -dbHost=hgwdev -buildDir=`pwd` hg38 + + time ./doLoad_rmskHmmer.bash > load.log 2>&1 + # real 4m47.432s + + featureBits -countGaps hg38 rmskHmmer > fb.hg38.rmskHmmer.txt 2>&1 + # 1734398971 bases of 3209286105 (54.043%) in intersection + + grep rmskClass hg38.class.profile.txt \ + | sed -e 's#rmskClass/##; s/.tab//;' | sort -rn + # profile of repeat elements: +# 1884179 SINE +# 1702529 LINE +# 805427 LTR +# 636906 Simple_repeat +# 565171 DNA +# 95480 Low_complexity +# 11861 Retroposon +# 10852 Satellite +# 9181 LTR? +# 6783 scRNA +# 4582 DNA? +# 3914 Unknown +# 2059 RC +# 1517 srpRNA +# 1484 RNA +# 970 SINE? +# 806 RC? +# 464 rRNA +# 5744165 total + + featureBits -countGaps hg38 rmskHmmer '!rmskCM' -bed=hmmerUnique.bed + # 172940594 bases of 3209286105 (5.389%) in intersection + hgLoadBed hg38 hmmerUnique hmmerUnique.bed + # Read 3099505 elements of size 4 from hmmerUnique.bed + +############################################################################# +## RepeatMasker with HMMER - DONE - 2013-12-24,26 - Hiram + mkdir /hive/data/genomes/hg38/bed/repeatMaskerHMMER + cd /hive/data/genomes/hg38/bed/repeatMaskerHMMER + + time doRepeatMasker.pl -stop=mask -useHMMER -bigClusterHub=ku \ + -workhorse=hgwdev -dbHost=hgwdev -buildDir=`pwd` hg38 > mask.log 2>&1 + # take the install script from this -debug run and alter it to load + # the table into rmskHmmer + doRepeatMasker.pl -continue=install -stop=install -useHMMER \ + -bigClusterHub=ku -workhorse=hgwdev -dbHost=hgwdev \ + -buildDir=`pwd` hg38 > mask.log 2>&1 + # 1702017722 bases of 3209286105 (53.034%) in intersection + # profile of repeat elements: +# 1879864 rmskClass/SINE.tab +# 1678216 rmskClass/LINE.tab +# 794231 rmskClass/LTR.tab +# 651561 rmskClass/Simple_repeat.tab +# 551965 rmskClass/DNA.tab +# 97186 rmskClass/Low_complexity.tab +# 10756 rmskClass/Retroposon.tab +# 10448 rmskClass/Satellite.tab +# 8393 rmskClass/LTR?.tab +# 5849 rmskClass/scRNA.tab +# 4282 rmskClass/Unknown.tab +# 4276 rmskClass/DNA?.tab +# 2000 rmskClass/RC.tab +# 1573 rmskClass/srpRNA.tab +# 1291 rmskClass/RNA.tab +# 906 rmskClass/snRNA.tab +# 747 rmskClass/SINE?.tab +# 723 rmskClass/RC?.tab +# 722 rmskClass/rRNA.tab +# 468 rmskClass/tRNA.tab +# 5705457 total + +############################################################################# +# rmsk from genbank release (DONE - 2014-12-25 - Hiram) + mkdir /hive/data/genomes/hg38/bed/repeatMaskerGenbank + cd /hive/data/genomes/hg38/bed/repeatMaskerGenbank + + head -3 ../repeatMaskerBlastn/hg38.fa.out > genbank.rm.out +find ../../genbank -type f | grep rm.out | grep -v "/placed_scaffolds/" | while read F +do + headRest 3 $F +done | sort -k5,45 -k6,6n >> genbank.rm.out + grep -v "^#" ../../genbank/GCA_000001405.15_GRCh38_top-level.acc2name \ + | awk '{printf "s/%s/%s/g;\n", $1, $3}' > accessionToUcsc.sed.txt + + sed -e "`cat accessionToUcsc.sed.txt`" genbank.rm.out > ucscNames.rm.out + + head -3 ucscNames.rm.out > hg38.sorted.fa.out + tail -n +4 ucscNames.rm.out | sort -k5,5 -k6,6n >> hg38.sorted.fa.out + + hgLoadOut -table=rmskGenbank -nosplit hg38 hg38.sorted.fa.out + hgLoadOut -verbose=2 -tabFile=hg38.rmskGenbank.tab -table=rmskGenbank \ + -nosplit hg38 hg38.sorted.fa.out 2> bad.records.txt + # fixed up one of the masking scripts from the other runs to construct + # the bbi files + + # 1581568556 bases of 3209286105 (49.281%) in intersection + # profile of repeat elements: +# 1849444 rmskClass/SINE.tab +# 1586141 rmskClass/LINE.tab +# 759248 rmskClass/LTR.tab +# 502186 rmskClass/DNA.tab +# 433789 rmskClass/Simple_repeat.tab +# 396378 rmskClass/Low_complexity.tab +# 10198 rmskClass/Satellite.tab +# 5884 rmskClass/LTR?.tab +# 4595 rmskClass/snRNA.tab +# 4163 rmskClass/Retroposon.tab +# 2802 rmskClass/Unknown.tab +# 2157 rmskClass/DNA?.tab +# 2154 rmskClass/tRNA.tab +# 1915 rmskClass/rRNA.tab +# 1860 rmskClass/RC.tab +# 1784 rmskClass/srpRNA.tab +# 1397 rmskClass/scRNA.tab +# 822 rmskClass/RNA.tab +# 488 rmskClass/SINE?.tab +# 445 rmskClass/RC?.tab +# 5567850 total + +############################################################################# +## running TRF simple repeats - DONE - 2013-12-24,29 - Hiram + # this procedure ran into much trouble on this release. The new + # repeat sequences in the centromeres caused trf to run indefinitely. + # I tried different sizes of chunks, working down to 20 Mbase chunks. + # Even still, some jobs would not complete. Those broke down even + # more, eventually to the smallest bit of 30 Kbase that needed to + # run all the way down to 3,000 based chunks with 1,000 base overlaps. + + # this did not work: + screen # use screen to manage this day-long job + mkdir /hive/data/genomes/hg38/bed/simpleRepeat + cd /hive/data/genomes/hg38/bed/simpleRepeat + time doSimpleRepeat.pl -bigClusterHub=ku -workhorse=hgwdev \ + -smallClusterHub=ku -buildDir=`pwd` hg38 > do.log 2>&1 + cd /hive/data/genomes/hg38/bed + # move it aside: + mv simpleRepeat simpleRepeat.2013-12-24 + + # Instead, something like this: + mkdir /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/splitGap + cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/splitGap + mkdir -p noGap + + twoBitToFa ../../../hg38.unmasked.2bit stdout \ + | faSplit -lift=noGap.lift gap stdin 5000000 noGap/hg38_ + # make sure nothing has gone missing: + faCount noGap/*.fa > faCount.txt + tail -1 faCount.txt +# total 3068387174 898285419 623727342 626335137 900967885 19071391 30979734 + # compared to the full sequence, same numbers for ACGT: + twoBitToFa ../../../hg38.unmasked.2bit stdout | faCount stdin +# total 3209286105 898285419 623727342 626335137 900967885 159970322 30979743 + faToTwoBit noGap/*.fa hg38.nogap.2bit + twoBitInfo hg38.nogap.2bit stdout | sort -k2,2nr > hg38.nogap.sizes + + + mkdir /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M + cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M + rm -rf /hive/data/genomes/hg38/TrfPart20M + /cluster/bin/scripts/simplePartition.pl \ +/hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/splitGap/hg38.nogap.2bit \ + 20000000 /hive/data/genomes/hg38/TrfPart20M + rm -f /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/TrfPart20M + ln -s /hive/data/genomes/hg38/TrfPart20M \ + /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/TrfPart20M + ssh ku + cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M + gensub2 /hive/data/genomes/hg38/TrfPart20M/partitions.lst single gsub jobList + para create jobList + para push + # 20 jobs would not complete: +# Completed: 143 of 163 jobs +# Jobs currently running: 20 +# CPU time in finished jobs: 76994s 1283.24m 21.39h 0.89d 0.002 y +# IO & Wait Time: 1095s 18.24m 0.30h 0.01d 0.000 y +# Time in running jobs: 1807279s 30121.32m 502.02h 20.92d 0.057 y +# Average job time: 546s 9.10m 0.15h 0.01d +# Longest running job: 90422s 1507.03m 25.12h 1.05d +# Longest finished job: 43348s 722.47m 12.04h 0.50d +# Submission to last job: 43363s 722.72m 12.05h 0.50d + # determine which are the last jobs as individual bits: + para status | grep -v done | awk '{print $(NF-1),$NF}' | grep TrfRun \ + > not.done.list + awk '{print $NF}' not.done.list | sed -e 's/.bed//' | while read F +do + cat $F +done > seq.specs.not.done + + mkdir /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/lastJobs + cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/lastJobs + mkdir fasta + for seqSpec in `cat ../seq.specs.not.done` +do + fName=`echo $seqSpec | sed -e 's/.*://'` + echo $fName + twoBitToFa $seqSpec fasta/$fName.fa +done + ls -1S `pwd`/fasta > part.list + cat << '_EOF_' > template +#LOOP +./runTrf {check in line+ $(path1)} {check out line bed/$(root1).bed} +#ENDLOOP +'_EOF_' + # << happy emacs + + cat << '_EOF_' > runTrf +#!/bin/bash +set -beEu -o pipefail +export path1=$1 +export inputFN=`basename $1` +export outpath=$2 +export outputFN=`basename $2` +mkdir -p /dev/shm/$outputFN +cp -p $path1 /dev/shm/$outputFN +cd /dev/shm/$outputFN +/cluster/bin/x86_64/trfBig -trf=/cluster/bin/x86_64/trf \ + $inputFN /dev/null -bedAt=$outputFN -tempDir=/dev/shm +cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/lastJobs +rm -f $outpath +cp -p /dev/shm/$outputFN/$outputFN $outpath +rm -fr /dev/shm/$outputFN/* +rmdir --ignore-fail-on-non-empty /dev/shm/$outputFN +'_EOF_' + # << happy emacs + chmod +x runTrf + + gensub2 part.list single template jobList + para create jobList + para push + # not all of these jobs will finish either: +# Completed: 85 of 106 jobs +# Jobs currently running: 21 +# CPU time in finished jobs: 58076s 967.93m 16.13h 0.67d 0.002 y +# IO & Wait Time: 828s 13.81m 0.23h 0.01d 0.000 y +# Time in running jobs: 1988997s 33149.95m 552.50h 23.02d 0.063 y +# Average job time: 693s 11.55m 0.19h 0.01d +# Longest running job: 94730s 1578.83m 26.31h 1.10d +# Longest finished job: 34216s 570.27m 9.50h 0.40d +# Submission to last job: 34342s 572.37m 9.54h 0.40d + + # can use what we have here: + liftUp result.bed ../../splitGap/noGap.lift error bed/*.bed + # find jobs not done + para status | grep -v done | awk '{print $(NF-1),$NF}' | grep TrfRun \ + > not.done.list + # splitting up those last jobs: + mkdir /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/splitBits + cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/splitBits + mkdir noGap + awk '{print $2}' ../lastJobs/not.done.list | while read F +do + cp -p $F ./noGap/ +done + + # split into 1,000,000 chunks with 10,000 overlap: + mkdir -p 1M_10K + +for F in noGap/*.fa +do + B=`basename $F | sed -e 's/.fa//'` + echo "faSplit -lift=$B.lift -extra=10000 size $F 1000000 1M_10K/$B_" + faSplit -lift=$B.lift -extra=10000 size $F 1000000 1M_10K/${B}_ +done + + ls -1S `pwd`/1M_10K/*.fa > part.list + cat << '_EOF_' > runTrf +#!/bin/bash +set -beEu -o pipefail +export path1=$1 +export inputFN=`basename $1` +export outpath=$2 +export outputFN=`basename $2` +mkdir -p /dev/shm/$outputFN +cp -p $path1 /dev/shm/$outputFN +cd /dev/shm/$outputFN +/cluster/bin/x86_64/trfBig -trf=/cluster/bin/x86_64/trf \ + $inputFN /dev/null -bedAt=$outputFN -tempDir=/dev/shm +cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/splitBits +rm -f $outpath +cp -p /dev/shm/$outputFN/$outputFN $outpath +rm -fr /dev/shm/$outputFN/* +rmdir --ignore-fail-on-non-empty /dev/shm/$outputFN +'_EOF_' + # << happy emacs + + cat << '_EOF_' > template +#LOOP +./runTrf {check in line+ $(path1)} {check out line bed/$(root1).bed} +#ENDLOOP +'_EOF_' + # << happy emacs + + gensub2 part.list single template jobList + para create jobList + para push + # not all of these jobs will complete either: +# Completed: 53 of 96 jobs +# CPU time in finished jobs: 212403s 3540.05m 59.00h 2.46d 0.007 y +# IO & Wait Time: 1851s 30.85m 0.51h 0.02d 0.000 y +# Average job time: 4043s 67.38m 1.12h 0.05d +# Longest finished job: 68726s 1145.43m 19.09h 0.80d +# Submission to last job: 68890s 1148.17m 19.14h 0.80d + # use what results we have here: + cat *.lift | liftUp parts.bed stdin error bed/*.bed + liftUp -type=.bed stdout ../../splitGap/noGap.lift error parts.bed \ + | sort -u | sort -k1,1 -k2,2n > hg38.result.bed + + para status | grep -v -w done | awk '{print $(NF-1)}' > will.not.finish.txt + + # split those last bits: + mkdir /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/splitSplitBits + cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/splitSplitBits + mkdir splitBits + cat ../splitBits/will.not.finish.txt | while read F +do + cp -p $F splitBits +done + + # 100K chunks with 10K overlap + mkdir -p 100K_10K + +for F in splitBits/*.fa +do + B=`basename $F | sed -e 's/.fa//'` + echo "faSplit -lift=$B.lift -extra=10000 size $F 1000000 1M_10K/$B_" + faSplit -lift=$B.lift -extra=10000 size $F 100000 100K_10K/${B}_ +done + + cat << '_EOF_' > runTrf +#!/bin/bash +set -beEu -o pipefail +export path1=$1 +export inputFN=`basename $1` +export outpath=$2 +export outputFN=`basename $2` +mkdir -p /dev/shm/$outputFN +cp -p $path1 /dev/shm/$outputFN +cd /dev/shm/$outputFN +/cluster/bin/x86_64/trfBig -trf=/cluster/bin/x86_64/trf \ + $inputFN /dev/null -bedAt=$outputFN -tempDir=/dev/shm +cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/splitSplitBits +rm -f $outpath +cp -p /dev/shm/$outputFN/$outputFN $outpath +rm -fr /dev/shm/$outputFN/* +rmdir --ignore-fail-on-non-empty /dev/shm/$outputFN +'_EOF_' + # << happy emacs + chmod +x runTrf + + cat << '_EOF_' > template +#LOOP +./runTrf {check in line+ $(path1)} {check out line bed/$(root1).bed} +#ENDLOOP +'_EOF_' + # << happy emacs + + ls -1S `pwd`/100K_10K/*.fa > part.list + gensub2 part.list single template jobList + para create jobList + para push + # one last bit does not complete: +# Completed: 420 of 421 jobs +# CPU time in finished jobs: 19862s 331.04m 5.52h 0.23d 0.001 y +# IO & Wait Time: 2360s 39.33m 0.66h 0.03d 0.000 y +# Average job time: 53s 0.88m 0.01h 0.00d +# Longest finished job: 368s 6.13m 0.10h 0.00d +# Submission to last job: 448s 7.47m 0.12h 0.01d + + # can use the results obtained here: + cat *.lift | liftUp splitParts.bed stdin error bed/*.bed + cat ../splitBits/*.lift | liftUp parts.bed stdin error splitParts.bed + liftUp -type=.bed stdout ../../splitGap/noGap.lift error parts.bed | sort -u \ + | sort -k1,1 -k2,2n > hg38.result.bed + + para status | grep -v -w done | awk '{print $(NF-1)}' + # last chunk: 100K_10K/hg38_89_2_00.fa + + mkdir /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/last100K + cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/last100K + cp -p ../splitSplitBits/100K_10K/hg38_89_2_00.fa . + + # 20K chunks with 10K overlap: + mkdir -p 20K_10K + +for F in hg38_89_2_00.fa +do + B=`basename $F | sed -e 's/.fa//'` + echo "faSplit -lift=$B.lift -extra=10000 size $F 20000 20K_10K/$B_" + faSplit -lift=$B.lift -extra=10000 size $F 20000 20K_10K/${B}_ +done + + ls -1S `pwd`/20K_10K/*.fa > part.list + cat << '_EOF_' > runTrf +#!/bin/bash +set -beEu -o pipefail +export path1=$1 +export inputFN=`basename $1` +export outpath=$2 +export outputFN=`basename $2` +mkdir -p /dev/shm/$outputFN +cp -p $path1 /dev/shm/$outputFN +cd /dev/shm/$outputFN +/cluster/bin/x86_64/trfBig -trf=/cluster/bin/x86_64/trf \ + $inputFN /dev/null -bedAt=$outputFN -tempDir=/dev/shm +cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/last100K +rm -f $outpath +cp -p /dev/shm/$outputFN/$outputFN $outpath +rm -fr /dev/shm/$outputFN/* +rmdir --ignore-fail-on-non-empty /dev/shm/$outputFN +'_EOF_' + # << happy emacs + chmod +s runTrf + cat << '_EOF_' > template +#LOOP +./runTrf {check in line+ $(path1)} {check out line bed/$(root1).bed} +#ENDLOOP +'_EOF_' + # << happy emacs + + gensub2 part.list single template jobList + para create jobList + para push + # one of these jobs will not finish: +# Completed: 4 of 5 jobs +# CPU time in finished jobs: 10s 0.17m 0.00h 0.00d 0.000 y +# IO & Wait Time: 16s 0.26m 0.00h 0.00d 0.000 y +# Average job time: 7s 0.11m 0.00h 0.00d +# Longest finished job: 8s 0.13m 0.00h 0.00d +# Submission to last job: 16s 0.27m 0.00h 0.00d + + # can use the results we have here: + cat *.lift | liftUp 20Kparts.bed stdin error bed/*.bed + cat ../splitSplitBits/*.lift | liftUp 100Kpart.bed stdin error 20Kparts.bed + cat ../splitBits/*.lift | liftUp parts.bed stdin error 100Kpart.bed + liftUp -type=.bed stdout ../../splitGap/noGap.lift error parts.bed | sort -u \ + | sort -k1,1 -k2,2n > hg38.result.bed + + # finally, what turns out to be the last batch: + mkdir /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/last30K + cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/last30K + cp -p ../last100K/20K_10K/hg38_89_2_00_3.fa . + + # 2K chunks with 1K overlap + mkdir -p 2K_1K + +for F in hg38_89_2_00_3.fa +do + B=`basename $F | sed -e 's/.fa//'` + echo "faSplit -lift=$B.lift -extra=1000 size $F 2000 2K_1K/$B_" + faSplit -lift=$B.lift -extra=1000 size $F 2000 2K_1K/${B}_ +done + + ls -1S `pwd`/2K_1K/*.fa > part.list + cat << '_EOF_' > runTrf +#!/bin/bash +set -beEu -o pipefail +export path1=$1 +export inputFN=`basename $1` +export outpath=$2 +export outputFN=`basename $2` +mkdir -p /dev/shm/$outputFN +cp -p $path1 /dev/shm/$outputFN +cd /dev/shm/$outputFN +/cluster/bin/x86_64/trfBig -trf=/cluster/bin/x86_64/trf \ + $inputFN /dev/null -bedAt=$outputFN -tempDir=/dev/shm +cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M/last30K +rm -f $outpath +cp -p /dev/shm/$outputFN/$outputFN $outpath +rm -fr /dev/shm/$outputFN/* +rmdir --ignore-fail-on-non-empty /dev/shm/$outputFN +'_EOF_' + # << happy emacs + chmod +x runTrf + cat << '_EOF_' > template +#LOOP +./runTrf {check in line+ $(path1)} {check out line bed/$(root1).bed} +#ENDLOOP +'_EOF_' + # << happy emacs + + gensub2 part.list single template jobList + para create + para push +# Completed: 15 of 15 jobs +# CPU time in finished jobs: 1s 0.02m 0.00h 0.00d 0.000 y +# IO & Wait Time: 26s 0.43m 0.01h 0.00d 0.000 y +# Average job time: 2s 0.03m 0.00h 0.00d +# Longest finished job: 4s 0.07m 0.00h 0.00d +# Submission to last job: 14s 0.23m 0.00h 0.00d + + cat *.lift | liftUp 2Kparts.bed stdin error bed/*.bed + cat ../last100K/*.lift | liftUp 20Kpart.bed stdin error 2Kparts.bed + cat ../splitSplitBits/*.lift | liftUp 100Kpart.bed stdin error 20Kpart.bed + cat ../splitBits/*.lift | liftUp parts.bed stdin error 100Kpart.bed + liftUp -type=.bed stdout ../../splitGap/noGap.lift error parts.bed | sort -u \ + | sort -k1,1 -k2,2n > hg38.result.bed + + ## To put it all together: + cd /hive/data/genomes/hg38/bed/simpleRepeat.2013-12-27/run20M + cat /hive/data/genomes/hg38/TrfPart20M/???/*.bed lastJobs/bed/*.bed \ + splitBits/parts.bed splitSplitBits/parts.bed last100K/parts.bed \ + last30K/parts.bed > beforeLift.simpleRepeat.bed + liftUp -type=.bed stdout ../splitGap/noGap.lift error \ + beforeLift.simpleRepeat.bed | sort -u \ + | sort -k1,1 -k2,2n > simpleRepeat.bed + + awk '{if ($5 <= 12) print;}' simpleRepeat.bed > trfMask.bed + + hgLoadBed hg38 simpleRepeat simpleRepeat.bed \ + -sqlTable=$HOME/kent/src/hg/lib/simpleRepeat.sql + featureBits hg38 simpleRepeat > fb.simpleRepeat 2>&1 + cat fb.simpleRepeat +# 146785521 bases of 3049335806 (4.814%) in intersection + + cd /hive/data/genomes/hg38/bed + ln -s simpleRepeat.2013-12-27/run20M simpleRepeat + +############################################################################ + + # WINDOWMASKER - DONE - 2013-12-24 - Hiram + mkdir /hive/data/genomes/hg38/bed/windowMasker + cd /hive/data/genomes/hg38/bed/windowMasker + time nice -n +19 doWindowMasker.pl -buildDir=`pwd` -workhorse=hgwdev \ + -dbHost=hgwdev hg38 > do.log 2>&1 & + +############################################################################ +# Verify all gaps are marked - DONE - 2013-12-24 - Hiram + mkdir /hive/data/genomes/hg38/bed/gap + cd /hive/data/genomes/hg38/bed/gap + time nice -n +19 findMotif -motif=gattaca -verbose=4 \ + -strand=+ ../../hg38.unmasked.2bit > findMotif.txt 2>&1 + # real 0m28.634s + grep "^#GAP " findMotif.txt | sed -e "s/^#GAP //" > allGaps.bed + featureBits hg38 -not gap -bed=notGap.bed + # 3049335806 bases of 3049335806 (100.000%) in intersection + time featureBits hg38 allGaps.bed notGap.bed -bed=new.gaps.bed + # 20023 bases of 3049335806 (0.001%) in intersection + # real 0m20.427s + # this indicates that 20,023 bases are not marked as N's + # with this element size profile: + awk '{print $3-$2}' new.gaps.bed | ave stdin +# Q1 1.000000 +# median 1.000000 +# Q3 100.000000 +# average 44.894619 +# min 1.000000 +# max 1000.000000 +# count 446 +# total 20023.000000 +# standard deviation 81.743447 + + # the four largest ones: +# 1000 chr2 32916625 32917625 chr2.7 +# 1000 chr2 32867130 32868130 chr2.6 +# 348 chr20 36314371 36314719 chr20.36 +# 200 chr12 123443533 123443733 chr12.10 + +######################################################################### +## CYTOBAND - fixing the ideogram track (DONE - 2014-06-11 - Hiram) + ## the file we used before was broken + mkdir -p /hive/data/outside/ncbi/ideogram/2014-06 + cd /hive/data/outside/ncbi/ideogram/2014-06 + # fetch all the ideogram files: + rsync -a -P rsync://ftp.ncbi.nlm.nih.gov/pub/gdp/ ./ + mkdir /hive/data/genomes/hg38/bed/cytoBandUpdate + cd /hive/data/genomes/hg38/bed/cytoBandUpdate + + # Create bed file + $HOME/kent/src/utils/ncbi/createNcbiCytoBand.pl \ +/hive/data/outside/ncbi/ideogram/2014-06/ideogram_9606_GCF_000001305.14_850_V1 + + # add in the other genome data: + hgsql -N -e 'select * from cytoBand;' hg38 \ + | egrep "chrU|chrM|_alt|_random" >> cytoBand.bed + + $HOME/kent/src/utils/ncbi/cytoBandVerify.pl + # everything checks out OK on 455 chroms + + # Load the bed file + hgLoadBed -tab -noBin -sqlTable=$HOME/kent/src/hg/lib/cytoBand.sql \ + hg38 cytoBand cytoBand.bed + cut -f1 cytoBand.bed | sort -u | awk '{print length($1)}' | sort -rn | head + # 23 + sed -e 's/12/23/' $HOME/kent/src/hg/lib/cytoBand.sql > cytoBand.sql + sort -k1,1 -k2,2n cytoBand.bed \ + | hgLoadSqlTab hg38 cytoBand cytoBand.sql stdin + + # Make cytoBandIdeo track for ideogram gif on hgTracks page. + # cytoBandIdeo is just a replicate of the cytoBand track. + hgsql -e "drop table cytoBandIdeo;" hg38 + hgsql hg38 -e "create table cytoBandIdeo (index(chrom(23),chromStart)) as select * from cytoBand;" + +######################################################################### +## CYTOBAND - ideogram track (DONE - 2014-03-04 - Hiram) + ssh hgwdev + mkdir -p /hive/data/outside/ncbi/ideogram/2014-03 + cd /hive/data/outside/ncbi/ideogram/2014-03 + + # fetch all the ideogram files: + rsync -a -P rsync://ftp.ncbi.nlm.nih.gov/pub/gdp/ ./ + + mkdir /hive/data/genomes/hg38/bed/cytoBand + cd /hive/data/genomes/hg38/bed/cytoBand + + # Create bed file + $HOME/kent/src/utils/ncbi/createNcbiCytoBand.pl \ +/hive/data/outside/ncbi/ideogram/2014-03/ideogram_9606_GCF_000001305.14_850_V1 + + # add in the other genome data: + hgsql -N -e 'select * from cytoBand;' hg38 > bobTable.bed + + egrep "chrU|chrM|_alt|_random" bobTable.bed >> cytoBand.bed + + ## can now verify before load: + $HOME/kent/src/utils/ncbi/cytoBandVerify.pl + # everything checks out OK on 455 chroms + + # Load the bed file + hgLoadBed -tab -noBin -sqlTable=$HOME/kent/src/hg/lib/cytoBand.sql \ + hg38 cytoBand cytoBand.bed + cut -f1 cytoBand.bed | sort -u | awk '{print length($1)}' | sort -rn | head + # 23 + sed -e 's/12/23/' $HOME/kent/src/hg/lib/cytoBand.sql > cytoBand.sql + sort -k1,1 -k2,2n cytoBand.bed \ + | hgLoadSqlTab hg38 cytoBand cytoBand.sql stdin + + # Make cytoBandIdeo track for ideogram gif on hgTracks page. + # cytoBandIdeo is just a replicate of the cytoBand track. + hgsql -e "drop table cytoBandIdeo;" hg38 + hgsql hg38 -e "create table cytoBandIdeo (index(chrom(23),chromStart)) as select * from cytoBand;" + +########################################################################## +# cytoBandIdeo - (DONE - 2013-12-26 - Hiram) + mkdir /hive/data/genomes/hg38/bed/cytoBand + cd /hive/data/genomes/hg38/bed/cytoBand + makeCytoBandIdeo.csh hg38 + +#making temporary liftover of items from hg19 +liftOver /hive/data/genomes/hg19/bed/ncbiCytoBand/cytobands.bed \ + /hive/data/gbdb/hg19/liftOver/hg19ToHg38.over.chain.gz \ + cytobands.bed unMapped + +liftOver -minBlocks=0.5 /hive/data/genomes/hg19/bed/ncbiCytoBand/cytobands.bed \ + /hive/data/gbdb/hg19/liftOver/hg19ToHg38.over.chain.gz \ + cytobands.0.5.bed unMapped0.5 + +############################### ###################### +# cytoBandIdeo - (reDONE - 2014-02-25 - kuhn) + +# adding centromeres to generic cytonBandIdeo tavle as it exists. +# (lifted track is already gone) + +# get the cen values for hg38 +hgsql -Ne "SELECT DISTINCT chrom FROM centromeres" hg38 | sort > hg38.chroms +rm -f hg38.cens +foreach chrom (`cat hg38.chroms`) + set cenStart="" + set cenEnd="" + set cenStart=`hgsql -Ne 'SELECT MIN(chromStart) FROM centromeres WHERE chrom = "'$chrom'"' hg38` + set cenEnd=`hgsql -Ne 'SELECT MAX(chromEnd) FROM centromeres WHERE chrom = "'$chrom'"' hg38` + echo "$chrom $cenStart $cenEnd" >> hg38.cens +end + +# Modified makeCytoBandIdeo.csh to use this file instead of looking +# for centromeres in a gap table. +# Replaced existing cytoBandIdeo table, which was really only a copy +# of chromInfo. + +########################################################################## +# hg19 <-> hg38 difference tracks (DONE - 2013-12-28 - Hiram) + mkdir /hive/data/genomes/hg19/bed/liftOverHg38 + cd /hive/data/genomes/hg19/bed/liftOverHg38 + + # not needed, but interesting, collect all the fragment + # definitions from the gold tables: + hgsql -N -e "select frag,fragStart,fragEnd,strand from gold;" hg19 \ + | sort > hg19.gold.frags.tab + + hgsql -N -e "select frag,fragStart,fragEnd,strand from gold;" hg38 \ + | sort > hg38.gold.frags.tab + + # construct common and difference listings + comm -12 hg19.gold.frags.tab hg38.gold.frags.tab \ + > identical.hg19.hg38.frags.tab + comm -23 hg19.gold.frags.tab hg38.gold.frags.tab \ + > unique.hg19Only.frags.tab + comm -13 hg19.gold.frags.tab hg38.gold.frags.tab \ + > unique.hg38Only.frags.tab + + # better yet, get full information about each fragment + hgsql -N -e "select chrom,chromStart,chromEnd,ix,type,frag,fragStart,fragEnd,strand from gold;" hg19 \ + | sort -k6 > hg19.gold.tab + + hgsql -N -e "select chrom,chromStart,chromEnd,ix,type,frag,fragStart,fragEnd,strand from gold;" hg38 \ + | sort -k6 > hg38.gold.tab + + # construct a single key for each fragment for joining. + # the key is frag,fragStart,fragEnd,strand + awk '{printf "%s,%d,%d,%s\t%s\t%s\t%s\t%d\t%d\t%d\t%s\n", + $6,$7,$8,$9,$6,$9,$1,$2,$3,$4,$5}' hg19.gold.tab | sort \ + > hg19.fragKey.tab + awk '{printf "%s,%d,%d,%s\t%s\t%s\t%s\t%d\t%d\t%d\t%s\n", + $6,$7,$8,$9,$6,$9,$1,$2,$3,$4,$5}' hg38.gold.tab | sort \ + > hg38.fragKey.tab + + # now, by joining those keys, we can get exact identicals, and + # the only-in listings as bed files to load as tracks: + join hg19.fragKey.tab hg38.fragKey.tab \ + | awk '{printf "%s\t%d\t%d\t%s\t1000\t%s\t%d\t%d\t0,0,128\n", $4,$5,$6,$2,$3,$5,$6}' \ + | sort -k1,1 -k2,2n > hg19.hg38.identical.bed + + join hg19.fragKey.tab hg38.fragKey.tab \ + | awk '{printf "%s\t%d\t%d\t%s\t1000\t%s\t%d\t%d\t0,0,128\n", $11,$12,$13,$9,$10,$12,$13}' \ + | sort -k1,1 -k2,2n > hg38.hg19.identical.bed + + join -v 1 hg19.fragKey.tab hg38.fragKey.tab \ + | awk '{printf "%s\t%d\t%d\t%s\t0\t%s\n", $4,$5,$6,$2,$3}' \ + | sort -k1,1 -k2,2n > hg19.only.bed + + join -v 2 hg19.fragKey.tab hg38.fragKey.tab \ + | awk '{printf "%s\t%d\t%d\t%s\t0\t%s\n", $4,$5,$6,$2,$3}' \ + | sort -k1,1 -k2,2n > hg38.only.bed + + hgLoadBed hg19 hg38ContigDiff hg19.only.bed + hgLoadBed hg38 hg19ContigDiff hg38.only.bed + + wc -l hg??.only.bed + # 6097 hg19.only.bed + # 23632 hg38.only.bed + + # this leaves the outstanding question of "why" they might be in + # the only-in listings. Some contigs may be different versions, + # sometimes different sections of the same contig are used, + # and contigs are dropped from hg19 to hg38, or new contigs added + # to hg38 to fill in gaps from hg19 + # Let's see if we can measure some of this: + awk '{print $4}' hg19.only.bed | sort -u > hg19.only.ids.list + awk '{print $4}' hg38.only.bed | sort -u > hg38.only.ids.list + + # Looks like 5405 idential contigs with different parts used: + comm -12 hg19.only.ids.list hg38.only.ids.list > differentPortions.list + wc -l differentPortions.list + # 5405 + + # and perhaps 63 = 5468-5405 of different versions of same contig: + sed -e "s/\.[0-9]*$//" hg19.only.ids.list | sort -u \ + > hg19.noVersions.ids.list + sed -e "s/\.[0-9]*$//" hg38.only.ids.list | sort -u \ + > hg38.noVersions.ids.list + comm -12 hg19.noVersions.ids.list hg38.noVersions.ids.list | wc -l + # 5468 + sed -e "s/\.[0-9]*$//" differentPortions.list | sort -u \ + > differentPortions.noVersions.list + comm -12 hg19.noVersions.ids.list hg38.noVersions.ids.list | sort -u \ + > noVersions.common.list + # indeed, 63 contigs of different versions: + comm -23 noVersions.common.list differentPortions.noVersions.list \ + | sort -u > differentVersions.list + wc -l differentVersions.list + # 63 + + # dividing up these items: + cat << '_EOF_' > identifyPortions.pl +#!/usr/bin/env perl + +use strict; +use warnings; + +my %differentVersions; +my %differentPortions; + +open (FH, "<differentVersions.list" ) or + die "can not read differentVersions.list"; +while (my $line = <FH>) { + chomp $line; + $differentVersions{$line} = 1; +} +close (FH); + +open (FH, "differentPortions.list" ) or + die "can not read differentPortions.list"; +while (my $line = <FH>) { + chomp $line; + $differentPortions{$line} = 1; +} +close (FH); + +my %hg19Done; +open (DP, ">hg19.differentPortions.bed") or die "can not write to hg19.differentPortions.bed"; +open (DV, ">hg19.differentVersions.bed") or die "can not write to hg19.differentVersions.bed"; +open (FH, "<hg19.only.bed" ) or die "can not read hg19.only.bed"; +while (my $line = <FH>) { + chomp $line; + my ($chr, $start, $end, $acc, $score, $strand) = split('\s+', $line); + # assume done while $acc is still complete + $hg19Done{$acc} = 1; + if (exists($differentPortions{$acc})) { + printf DP "%s\n", $line; + } else { + my $trimAcc = $acc; + $trimAcc =~ s/\.[0-9]+$//; + if (exists($differentVersions{$trimAcc})) { + printf DV "%s\n", $line; + } else { + # this one does not match + $hg19Done{$acc} = 0; + } + } +} +close (FH); +close (DV); +close (DP); +open (DR, ">hg19.dropped.bed") or die "can not write to hg19.dropped.bed"; +open (FH, "<hg19.only.bed" ) or die "can not read hg19.only.bed"; +while (my $line = <FH>) { + chomp $line; + my ($chr, $start, $end, $acc, $score, $strand) = split('\s+', $line); + if (0 == $hg19Done{$acc}) { + printf DR "%s\n", $line; + } +} +close (FH); +close (DR); + +my %hg38Done; +open (DP, ">hg38.differentPortions.bed") or die "can not write to hg38.differentPortions.bed"; +open (DV, ">hg38.differentVersions.bed") or die "can not write to hg38.differentVersions.bed"; +open (FH, "<hg38.only.bed" ) or die "can not read hg38.only.bed"; +while (my $line = <FH>) { + chomp $line; + my ($chr, $start, $end, $acc, $score, $strand) = split('\s+', $line); + # assume done while $acc is still complete + $hg38Done{$acc} = 1; + if (exists($differentPortions{$acc})) { + printf DP "%s\n", $line; + } else { + my $trimAcc = $acc; + $trimAcc =~ s/\.[0-9]+$//; + if (exists($differentVersions{$trimAcc})) { + printf DV "%s\n", $line; + } else { + # this one does not match + $hg38Done{$acc} = 0; + } + } +} +close (FH); +close (DV); +close (DP); +open (DR, ">hg38.newTo19.bed") or die "can not write to hg38.newTo19.bed"; +open (FH, "<hg38.only.bed" ) or die "can not read hg38.only.bed"; +while (my $line = <FH>) { + chomp $line; + my ($chr, $start, $end, $acc, $score, $strand) = split('\s+', $line); + if (0 == $hg38Done{$acc}) { + printf DR "%s\n", $line; + } +} +close (FH); +close (DR); +'_EOF_' + # << happy emacs + chmod +x identifyPortions.pl + ./identifyPortions.pl + # make sure nothing was lost + sort hg19.differentVersions.bed hg19.differentPortions.bed \ + hg19.dropped.bed | sum + # 43711 233 + sort hg19.only.bed | sum + # 43711 233 + sort hg38.differentVersions.bed hg38.differentPortions.bed \ + hg38.newTo19.bed | sum + # 00502 911 + sort hg38.only.bed | sum + # 00502 911 + + sort -k1,1 -k2,2n hg38.differentVersions.bed hg38.differentPortions.bed \ + hg38.newTo19.bed > hg38.itemRgb.bed + sort -k1,1 -k2,2n hg19.differentVersions.bed hg19.differentPortions.bed \ + hg19.dropped.bed > hg19.itemRgb.bed + + hgLoadBed hg19 hg38ContigDiff hg19.itemRgb.bed + # if you wanted to load the identicals in this track too: + sort -k1,1 -k2,2n hg38.hg19.identical.bed hg38.itemRgb.bed \ + | hgLoadBed hg38 hg38ContigDiff stdin + # but we don't, we deliver only the differences + hgLoadBed hg38 hg38ContigDiff hg38.itemRgb.bed + +######################################################################### +# construct ooc file to be used in blat operations +# DONE - 2012-12-30 - Hiram +# can be done on unmasked sequence the same result as masked: + cd /hive/data/genomes/hg38 + time blat hg38.unmasked.2bit /dev/null /dev/null \ + -tileSize=11 -makeOoc=jkStuff/hg38.11.ooc -repMatch=1024 + + # been confirmed, the 100-base non-bridged gaps are really non-bridged + gapToLift -minGap=100 -bedFile=jkStuff/nonBridgedGaps.bed hg38 \ + jkStuff/hg38.nonBridged.lft + +############################################################################## +# cpgIslands - (DONE - 2014-01-07 - Hiram) + # run on the Hmmer + trfMask sequence + mkdir /hive/data/genomes/hg38/bed/cpgIslands + cd /hive/data/genomes/hg38/bed/cpgIslands + time $HOME/kent/src/hg/utils/automation/doCpgIslands.pl \ + -dbHost=hgwdev -bigClusterHub=ku -buildDir=`pwd` \ + -workhorse=hgwdev -smallClusterHub=ku hg38 > do.log 2>&1 + # real 3m31.684s + # wc -l cpgIsland.bed -> 30456 cpgIsland.bed + cat fb.hg38.cpgIslandExt.txt + # 23654068 bases of 3049335806 (0.776%) in intersection + + # Previously in hg19: + featureBits -countGaps hg19 cpgIslandExt + # 21842742 bases of 3137161264 (0.696%) in intersection + + # when run on Hmmer and Trf masked sequence: + # wc -l cpgIsland.bed -> 30416 cpgIsland.bed + # 23635946 bases of 3049335806 (0.775%) in intersection + + # when run on unmasked sequence: + # wc -l cpgIsland.bed -> 55149 cpgIsland.bed + # 33637531 bases of 3049335806 (1.103%) in intersection +############################################################################## +# rerun cpgIslands on contig sequence (DONE - 2014-01-07 - Hiram) + # this is a test of the contig sequence file, + # should get a very similar answer to the above + mkdir /hive/data/genomes/hg38/bed/cpgIslandsContigs + cd /hive/data/genomes/hg38/bed/cpgIslandsContigs + + # run stepwise so the lift can be done on the result before loading + time $HOME/kent/src/hg/utils/automation/doCpgIslands.pl \ + -dbHost=hgwdev -bigClusterHub=ku -buildDir=`pwd` \ + -stop=makeBed -maskedSeq=/hive/data/genomes/hg38/hg38.contigs.2bit \ + -workhorse=hgwdev -smallClusterHub=ku hg38 > makeBed.log 2>&1 + # real 9m31.502s + # fails on the bedToBigBed creation since this isn't the actual + # hg38 sequence. + mv cpgIsland.bed cpgIsland.beforeLift.bed + liftUp -type=.bed stdout ../../jkStuff/hg38.contigs.lift carry \ + cpgIsland.beforeLift.bed | sort -k1,1 -k2,2n > cpgIsland.bed + bedToBigBed -tab -type=bed4+6 -as=$HOME/kent/src/hg/lib/cpgIslandExt.as \ + cpgIsland.bed ../../chrom.sizes hg38.cpgIslandExt.bb + zcat ../cpgIslands/cpgIsland.bed.gz | sort -k1,1 -k2,2n > t.bed + # Surprisingly, a few more are detected, perhaps due to the different + # masking since this contig run is on the final corrected cross-match rmsk + # plus TRF, the above was on the corrupted HMMER+TRF mask: + wc -l cpgIsland.bed t.bed +# 30477 cpgIsland.bed +# 30456 t.bed + # 2,835 different items between the two: + sort t.bed cpgIsland.bed | uniq -c | awk '$1 < 2' | wc -l + # 2835 + # 29.049 identical items + sort t.bed cpgIsland.bed | uniq -c | awk '$1 == 2' | wc -l + # 29049 + cut -f1-3 cpgIsland.bed | sort > contigs.bed + cut -f1-3 t.bed | sort > fullSequence.bed + # 29,339 identical locations: + comm -12 contigs.bed fullSequence.bed | wc -l + # 29339 + + time $HOME/kent/src/hg/utils/automation/doCpgIslands.pl \ + -dbHost=hgwdev -bigClusterHub=ku -buildDir=`pwd` \ + -continue=load -maskedSeq=/hive/data/genomes/hg38/hg38.contigs.2bit \ + -workhorse=hgwdev -smallClusterHub=ku hg38 > load.log 2>&1 + # real 0m12.056s + + cat fb.hg38.cpgIslandExt.txt + # 23610399 bases of 3049335806 (0.774%) in intersection + +############################################################################## +# rerun cpgIslands on contig UNMASKED sequence (DONE - 2014-01-07 - Hiram) + mkdir /hive/data/genomes/hg38/bed/cpgIslandsContigsUnmasked + cd /hive/data/genomes/hg38/bed/cpgIslandsContigsUnmasked + + twoBitToFa -noMask ../../hg38.contigs.2bit stdout \ + | faToTwoBit stdin hg38.contigsUnmasked.2bit + + # verify sequence is OK: + twoBitToFa hg38.contigsUnmasked.2bit stdout | faSize stdin +# 3061688741 bases (12372958 N's 3049315783 real 3049315783 upper 0 lower) +# in 733 sequences in 1 files +# %0.00 masked total, %0.00 masked real + twoBitToFa hg38.contigsUnmasked.2bit stdout | faCount stdin | tail -1 +# total 3061688741 898285419 623727342 626335137 900967885 12372958 30979743 + # ACGT CpG same as original hg38.2bit except for the missing N's: +# total 3209286105 898285419 623727342 626335137 900967885 159970322 30979743 + + # run stepwise so the lift can be done on the result before loading + time $HOME/kent/src/hg/utils/automation/doCpgIslands.pl \ + -dbHost=hgwdev -bigClusterHub=ku -buildDir=`pwd` \ + -stop=makeBed -maskedSeq=`pwd`/hg38.contigsUnmasked.2bit \ + -workhorse=hgwdev -smallClusterHub=ku hg38 > makeBed.log 2>&1 + # real 11m0.690s + # as above, failed on the bedToBigBed step since this isn't the full hg38 + # sequence + mv cpgIsland.bed cpgIsland.beforeLift.bed + liftUp -type=.bed stdout ../../jkStuff/hg38.contigs.lift carry \ + cpgIsland.beforeLift.bed | sort -k1,1 -k2,2n > cpgIsland.bed + bedToBigBed -tab -type=bed4+6 -as=$HOME/kent/src/hg/lib/cpgIslandExt.as \ + cpgIsland.bed ../../chrom.sizes hg38.cpgIslandExt.bb + # a lot more here that for masked sequence: + wc -l cpgIsland.bed ../cpgIslandsContigs/cpgIsland.bed + # 55149 cpgIsland.bed + # 30477 ../cpgIslandsContigs/cpgIsland.bed + featureBits -countGaps hg38 cpgIsland.bed + # 33637531 bases of 3209286105 (1.048%) in intersection + featureBits -countGaps hg38 ../cpgIslandsContigs/cpgIsland.bed + # 23610399 bases of 3209286105 (0.736%) in intersection + + # debug load step so it can be loaded into a separate table: + $HOME/kent/src/hg/utils/automation/doCpgIslands.pl \ + -dbHost=hgwdev -bigClusterHub=ku -buildDir=`pwd` \ + -debug -continue=load -maskedSeq=`pwd`/hg38.contigsUnmasked.2bit \ + -workhorse=hgwdev -smallClusterHub=ku hg38 + + time ./doLoadCpg.csh > load.log 2>&1 + # real 0m2.179s + # 33637531 bases of 3049335806 (1.103%) in intersection + +######################################################################### +# construct liftOver to hg19 (DONE - 2013-12-31 - Hiram) + # it turns out it doesn't matter if the query or target 2bit files + # are masked. This procedure can be done on completely unmasked sequences + # for both, same result masked or not masked + screen -S hg38 # manage this longish running job in a screen + mkdir /hive/data/genomes/hg38/bed/blat.hg19.2013-12-31 + cd /hive/data/genomes/hg38/bed/blat.hg19.2013-06-10 + # this was run in manual steps as experiments were done about the masking + # check it with -debug first to see if it is going to work: + doSameSpeciesLiftOver.pl -stop=net -buildDir=`pwd` -bigClusterHub=ku \ + -dbHost=hgwdev -workhorse=hgwdev -debug \ + -ooc=/hive/data/genomes/hg38/jkStuff/hg38.11.ooc hg38 hg19 + # the debug step doesn't actually construct enough files to run the + # steps manually. The chaining has an extra procedure that is performed + # while not in 'debug' mode + # the run.blat was operated manually, then chaining: + time doSameSpeciesLiftOver.pl -continue=chain -stop=net -buildDir=`pwd` \ + -bigClusterHub=ku \ + -dbHost=hgwdev -workhorse=hgwdev \ + -ooc=/hive/data/genomes/hg38/jkStuff/hg38.11.ooc \ + hg38 hg19 > chain.log 2>&1 + # real 22m31.635s + # loading is only a few seconds: + doSameSpeciesLiftOver.pl -continue=load -buildDir=`pwd` \ + -bigClusterHub=ku \ + -dbHost=hgwdev -workhorse=hgwdev \ + -ooc=/hive/data/genomes/hg38/jkStuff/hg38.11.ooc \ + hg38 hg19 > load.log 2>&1 + + # verify this file exists: + # /gbdb/hg38/liftOver/hg38ToHg19.over.chain.gz + # and try out the conversion on genome-test from hg38 to hg19 + # same file should exist for downloads: + # /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/liftOver/hg38ToHg19.over.chain.gz + +############################################################################ +# marking the PAR regions: (DONE - 2014-01-09 - Hiram) + # after much experimentation with the AGP files and the given NCBI + # files in hg38/genbank/Primary_Assembly/pseudoautosomal_region + # the PAR region definitions can be seen in the par_align.gff file: +# CM000685.2 10001 2781479 -> CM000686.2 10001 2781479 +# CM000685.2 155701383 156030895 -> CM000686.2 56887903 57217415 + # equivalent to: +# chrX 10001 2781479 -> chrY 10001 2781479 +# chrX 155701383 156030895 -> chrY 56887903 57217415 + + # subtract one for the chromStart position: + cat << '_EOF_' > hg38Par.bed4 +chrX 10000 2781479 PAR1 +chrX 155701382 156030895 PAR2 +chrY 10000 2781479 PAR1 +chrY 56887902 57217415 PAR2 +'_EOF_' + # << happy emacs + + hgLoadBed hg38 par hg38Par.bed4 + checkTableCoords hg38 + + # hg19 had: ++-------+------------+-----------+------+ +| chrom | chromStart | chromEnd | name | ++-------+------------+-----------+------+ +| chrX | 60000 | 2699520 | PAR1 | +| chrX | 154931043 | 155260560 | PAR2 | +| chrY | 10000 | 2649520 | PAR1 | +| chrY | 59034049 | 59363566 | PAR2 | ++-------+------------+-----------+------+ + + # The AGP files come close to definining the location, but not + # precisely. The first region uses different bits of AC006209.25: +zcat ../../genbank/Primary_Assembly/assembled_chromosomes/AGP/chrX.comp.agp.gz\ + | grep AC006209.25 +CM000685.2 2665048 2677319 56 F AC006209.25 127483 139754 - +CM000685.2 2677869 2804801 58 F AC006209.25 1 126933 - +zcat ../../genbank/Primary_Assembly/assembled_chromosomes/AGP/chrY.comp.agp.gz\ + | grep AC006209.25 +CM000686.2 2665048 2677319 56 F AC006209.25 127483 139754 - +CM000686.2 2677869 2781479 58 F AC006209.25 23323 126933 - + + # and the second region uses different bits of AJ271735.1: +zcat ../../genbank/Primary_Assembly/assembled_chromosomes/AGP/chrX.comp.agp.gz\ + | grep AJ271735.1 | head -1 +CM000685.2 155676925 155719966 3096 O AJ271735.1 44687 87728 + +zcat ../../genbank/Primary_Assembly/assembled_chromosomes/AGP/chrY.comp.agp.gz\ + | grep AJ271735.1 | head -1 +CM000686.2 56887903 56906486 356 O AJ271735.1 69145 87728 + + + # combining all the contig definitions from each will find all the + # exact identical contig bits: +zcat ../../genbank/Primary_Assembly/assembled_chromosomes/AGP/chrY.comp.agp.gz\ + | grep -v "^#" | awk '$5 != "N"' \ + | awk '{printf "%s_%d_%d\t%s\t%d\t%d\n", $6,$7,$8,$1,$2,$3}' \ + | sort > chrY.comp.agp.txt +zcat ../../genbank/Primary_Assembly/assembled_chromosomes/AGP/chrX.comp.agp.gz\ + | grep -v "^#" | awk '$5 != "N"' \ + | awk '{printf "%s_%d_%d\t%s\t%d\t%d\n", $6,$7,$8,$1,$2,$3}' \ + | sort > chrX.comp.agp.txt + join -t'^I' chrY.comp.agp.txt chrX.comp.agp.txt | head + +CM000685.2 10001 44821 CM000686.2 10001 44821 +... +CM000685.2 2677320 2677868 CM000686.2 2677320 2677868 + +CM000685.2 155719967 155720351 CM000686.2 56906487 56906871 +... +CM000685.2 155964490 156030895 CM000686.2 57151010 57217415 + +############################################################################ +## altLocations track (DONE - 2014-01-02 - Hiram) + # indicate corresponding locations between haplotypes and reference + mkdir /hive/data/genomes/hg38/bed/altLocations + cd /hive/data/genomes/hg38/bed/altLocations + + find ../../genbank/ALT_* -type f | grep alt_scaffold_placement.txt \ + | while read F +do + grep -v "^#" ${F} | sed -e 's/\./v/;' | awk -F'\t' '{printf "chr%s\t%d\t%d\tchr%s_%s_alt\n", $6,$12-1,$13,$6, $4}' +done | sort -k1,1 -k2,2n > chrToAlt.bed + + # note silent hidden <tab> character in the join -t argument + # explicit as written here + +find ../../genbank/ALT_* -type f | grep alt_scaffold_placement.txt \ + | while read F +do + grep -v "^#" ${F} | sed -e 's/\./v/;' | awk -F'\t' '{printf "chr%s_%s_alt\tchr%s:%d-%d\n", $6,$4,$6,$12,$13}' +done | sort > altToChr.tab +sort ../../chrom.sizes | join -t'^I' - altToChr.tab \ + | awk '{printf "%s\t0\t%d\t%s\n", $1,$2,$3}' > altToChr.bed + + + hgLoadBed hg38 altLocations chrToAlt.bed altToChr.bed + featureBits -countGaps hg38 altLocations + # 170113652 bases of 3209286105 (5.301%) in intersection + +############################################################################ +## genscan (DONE - 2014-01-07 - Hiram) + mkdir /hive/data/genomes/hg38/bed/genscan + cd /hive/data/genomes/hg38/bed/genscan + + # using the contig sequence + # running stepwise to allow the lifting of the final result + time $HOME/kent/src/hg/utils/automation/doGenscan.pl hg38 -buildDir=`pwd` \ + -maskedSeq=/hive/data/genomes/hg38/hg38.contigs.2bit \ + -stop=makeBed -bigClusterHub=ku -dbHost=hgwdev -workhorse=hgwdev \ + > do.log 2>&1 + # three jobs did not finish due to almost all N's in the sequence, + # just a couple of bases in each piece. Their empty result is good enough. + time $HOME/kent/src/hg/utils/automation/doGenscan.pl hg38 -buildDir=`pwd` \ + -maskedSeq=/hive/data/genomes/hg38/hg38.contigs.2bit \ + -continue=makeBed -stop=makeBed -bigClusterHub=ku -dbHost=hgwdev \ + -workhorse=hgwdev > makeBed.log 2>&1 + # real 0m48.161s + + cd lifted + mkdir -p gtf subopt nameFixed/gtf nameFixed/pep newNames pep + for F in ../gtf/000/*.gtf +do + B=`basename $F` + liftUp gtf/${B} ../../../jkStuff/hg38.contigs.lift carry $F + echo $B +done + for F in ../subopt/000/*.bed +do + B=`basename $F` + liftUp subopt/${B} ../../../jkStuff/hg38.contigs.lift carry $F + echo $B +done + + ls gtf/chr*_[0-9][0-9].gtf \ + | sed -e 's/_[0-9][0-9]//; s#gtf/##; s/.gtf//;' | sort -u | while read C +do + cat ../pep/000/${C}_[0-9][0-9].pep > pep/${C}.pep + cat gtf/${C}_[0-9][0-9].gtf | ./gtfFixId.pl ${C} > nameFixed/gtf/${C}.gtf + ./pepNameFix.pl ${C} > nameFixed/pep/${C}.pep +done + + cat nameFixed/gtf/*.gtf > ../hg38.genscan.gtf + ls gtf | egrep -v '^chr[0-9XY][0-9]*_[0-9][0-9].gtf' | while read C +do + cat gtf/${C} +done >> ../hg38.genscan.gtf + + cat nameFixed/pep/*.pep > ../hg38.genscan.pep + ls gtf | egrep -v '^chr[0-9XY][0-9]*_[0-9][0-9].gtf' \ + | sed -e 's/.gtf/.pep/' | while read C +do + cat ../pep/000/${C} +done >> ../hg38.genscan.pep + + cd /hive/data/genomes/hg38/bed/genscan + cat lifted/subopt/*.bed | sort -k1,1 -k2,2n > hg38.genscanSubopt.bed + + gtfToGenePred hg38.genscan.gtf hg38.genscan.gp + genePredCheck -db=hg38 hg38.genscan.gp + # checked: 44149 failed: 0 + genePredToBed hg38.genscan.gp hg38.genscan.bed + bedToBigBed hg38.genscan.bed ../../chrom.sizes hg38.genscan.bb + bedToBigBed hg38.genscanSubopt.bed ../../chrom.sizes hg38.genscanSubopt.bb + ldHgGene -gtf hg38 genscan hg38.genscan.gtf +# Read 44149 transcripts in 339212 lines in 1 files +# 44149 groups 345 seqs 1 sources 1 feature types + + cat fb.hg38.genscan.txt + # 58278346 bases of 3049335806 (1.911%) in intersection + cat fb.hg38.genscanSubopt.txt + # 55020514 bases of 3049335806 (1.804%) in intersection + + # oddly, we are getting half of what hg19 had ? + featureBits hg19 genscan + # 106433874 bases of 2897316137 (3.674%) in intersection + + # This is because hg19 was run on soft-masked sequence and not + # on hard masked sequence + +############################################################################ +## genscan on unmasked sequence experiment (DONE - 2013-12-03 - Hiram) + ## instead, working on unmasked sequence: + mkdir /hive/data/genomes/hg38/bed/genscan/unmaskedRun + cd /hive/data/genomes/hg38/bed/genscan/unmaskedRun + + mkdir liftSpecs + split -a 3 -d -l 1 ../../../jkStuff/hg38.nonBridged.lift liftSpecs/hg38_ + + mkdir fasta +for F in liftSpecs/hg38_* +do + L=`cut -f2 $F` + echo $L + /cluster/home/hiram/kent/src/hg/utils/lft2BitToFa.pl \ + ../../../hg38.unmasked.2bit $F > fasta/${L}.fa +done + + + cat << '_EOF_' > template +#LOOP +./runGsBig.bash $(path1) {check out exists gtf/$(root1).gtf} {check out exists pep/$(root1).pep} {check out exists subopt/$(root1).bed} +#ENDLOOP +'_EOF_' + # << happy emacs + cat << '_EOF_' > runGsBig.bash +#!/bin/bash + +set -beEu -o pipefail + +export seqFile=$1 +export resultGtf=$2 +export resultPep=$3 +export resultSubopt=$4 +/cluster/bin/x86_64/gsBig $seqFile $resultGtf -trans=$resultPep -subopt=$resultSubopt -exe=/scratch/data/genscan/genscan -par=/scratch/data/genscan/HumanIso.smat -tmp=/dev/shm -window=2400000 +'_EOF_' + # << happy emacs + + ls -1S `pwd`/fasta/*.fa > part.list + gensub2 part.list single template jobList + para create jobList + para push + # several jobs crashed: +# Completed: 726 of 733 jobs +# Crashed: 7 jobs +# CPU time in finished jobs: 62501s 1041.68m 17.36h 0.72d 0.002 y +# IO & Wait Time: 2563s 42.72m 0.71h 0.03d 0.000 y +# Average job time: 90s 1.49m 0.02h 0.00d +# Longest finished job: 3288s 54.80m 0.91h 0.04d +# Submission to last job: 3294s 54.90m 0.92h 0.04d + + para status | grep -v -w done | awk '{print $(NF-3)}' > crashed.job.list + + mkdir /hive/data/genomes/hg38/bed/genscan/unmaskedRun/crashedJobs + cd /hive/data/genomes/hg38/bed/genscan/unmaskedRun/crashedJobs + mkdir splitBits + + for F in chr2.06 chr1.03 chr3.05 chr12.07 chr10.05 chr17.08 chr11.04 +do + faSplit -lift=${F}.lift gap ../fasta/${F}.fa 2000000 splitBits/${F}_ +done + + ls -1S `pwd`/splitBits/*.fa > part.list + cat << '_EOF_' > runGsBig.bash +#!/bin/bash + +set -beEu -o pipefail + +export seqFile=$1 +export resultGtf=$2 +export resultPep=$3 +export resultSubopt=$4 +/cluster/bin/x86_64/gsBig $seqFile $resultGtf -trans=$resultPep -subopt=$resultSubopt -exe=/scratch/data/genscan/genscan -par=/scratch/data/genscan/HumanIso.smat -tmp=/dev/shm -window=2400000 +'_EOF_' + # << happy emacs + chmod +x runGsBig.bash + + cat << '_EOF_' > template +#LOOP +./runGsBig.bash $(path1) {check out exists gtf/$(root1).gtf} {check out exists pep/$(root1).pep} {check out exists subopt/$(root1).bed} +#ENDLOOP +'_EOF_' + # << happy emacs + + gensub2 part.list single template jobList + para create jobList + para push +# Completed: 331 of 334 jobs +# Crashed: 3 jobs +# CPU time in finished jobs: 18097s 301.62m 5.03h 0.21d 0.001 y +# IO & Wait Time: 1085s 18.08m 0.30h 0.01d 0.000 y +# Average job time: 58s 0.97m 0.02h 0.00d +# Longest finished job: 79s 1.32m 0.02h 0.00d +# Submission to last job: 249s 4.15m 0.07h 0.00d + # the last three completed with -window=1600000 + + # lifting results: + cat << '_EOF_' > fixIds.pl +#!/usr/bin/env perl + +use strict; +use warnings; + +my $argc = scalar(@ARGV); + +if ($argc != 1) { + printf STDERR "usage: cat chrN.M.lifted | ./fixIds.pl chrN.M\n"; + exit 255; +} + +my $F=shift; +my $C = $F; +$C =~ s/\.[0-9][0-9]//; + +my $id = 0; +my $prevId = ""; +open (GT, ">${F}.gtf") or die "can not write to ${F}.gtf"; +while (my $line=<>) { + chomp $line; + my $geneId = $line; + $geneId =~ s/^${C}.*gene_id "${C}//; + $geneId =~ s/";.*//; + $id += 1 if ( $prevId ne $geneId); + $line =~ s/${C}[0-9]+.[0-9]+/${F}.$id/g; + printf GT "%s\n", $line; + $prevId = $geneId; +} +close (GT); +'_EOF_' + # << happy emacs + chmod +x fixIds.pl + for F in chr1.03 chr10.05 chr11.04 chr12.07 chr17.08 chr2.06 chr3.05 +do + echo "${F}" 1>&2 + cut -f2 ${F}.lift | while read P + do + liftUp -type=.gtf stdout ${F}.lift error gtf/${P}.gtf + done > ${F}.lifted.gtf + cat ${F}.lifted.gtf | ./fixIds.pl ${F} +done + # copied these results to ../gtf/ to get into the final result +# -rw-rw-r-- 1 3349959 Jan 2 15:33 chr1.03.gtf +# -rw-rw-r-- 1 2439182 Jan 2 15:33 chr10.05.gtf +# -rw-rw-r-- 1 1068097 Jan 2 15:33 chr11.04.gtf +# -rw-rw-r-- 1 2392548 Jan 2 15:33 chr12.07.gtf +# -rw-rw-r-- 1 1831336 Jan 2 15:33 chr17.08.gtf +# -rw-rw-r-- 1 3539694 Jan 2 15:33 chr2.06.gtf +# -rw-rw-r-- 1 2309903 Jan 2 15:33 chr3.05.gtf + + for F in chr1.03 chr10.05 chr11.04 chr12.07 chr17.08 chr2.06 chr3.05 +do + echo "${F}" 1>&2 + cut -f2 ${F}.lift | while read P + do + liftUp -type=.bed stdout ${F}.lift error subopt/${P}.bed + done > ${F}.lifted.subopt.bed +done + # copied these results to ../subopt/ to get into the final result +# -rw-rw-r-- 1 3349959 Jan 2 15:33 chr1.03.gtf +# -rw-rw-r-- 1 2439182 Jan 2 15:33 chr10.05.gtf +# -rw-rw-r-- 1 1068097 Jan 2 15:33 chr11.04.gtf +# -rw-rw-r-- 1 2392548 Jan 2 15:33 chr12.07.gtf +# -rw-rw-r-- 1 1831336 Jan 2 15:33 chr17.08.gtf +# -rw-rw-r-- 1 3539694 Jan 2 15:33 chr2.06.gtf +# -rw-rw-r-- 1 2309903 Jan 2 15:33 chr3.05.gtf + + + cat << '_EOF_' > pepNameFix.pl +#!/usr/bin/env perl + +use strict; +use warnings; + +# BIG ASSUMPTION ! ! ! - the peptides are in the same order as +# they are in the GTF file ! ! ! + +my $argc = scalar(@ARGV); + +if ($argc != 1) { + printf STDERR "usage: cat chrN.M.needNameFix.pep | ./pepNameFix.pl chrN.M > chrN.M.pep\n"; + exit 255; +} + +my $C=shift; + +my $id = 1; + +while (my $line = <>) { + if ($line =~ m/^>/) { + printf ">%s.%d\n", $C, $id++; + } else { + print $line; + } +} +'_EOF_' + # << happy emacs + chmod +x pepNameFix.pl + +for F in chr1.03 chr10.05 chr11.04 chr12.07 chr17.08 chr2.06 chr3.05 +do + echo "${F}" 1>&2 + cut -f2 ${F}.lift | while read P + do + cat pep/${P}.pep + done > ${F}.needNameFix.pep + cat ${F}.needNameFix.pep | ./pepNameFix.pl ${F} > ${F}.pep +done + # copied these results to ../pep/ to get into the final result: +# -rw-rw-r-- 1 1592655 Jan 2 15:55 chr1.03.pep +# -rw-rw-r-- 1 1169168 Jan 2 15:55 chr10.05.pep +# -rw-rw-r-- 1 519106 Jan 2 15:55 chr11.04.pep +# -rw-rw-r-- 1 1152111 Jan 2 15:55 chr12.07.pep +# -rw-rw-r-- 1 775052 Jan 2 15:55 chr17.08.pep +# -rw-rw-r-- 1 1799546 Jan 2 15:55 chr2.06.pep +# -rw-rw-r-- 1 1248762 Jan 2 15:55 chr3.05.pep + + # and then, adding in all the results together + + cd /hive/data/genomes/hg38/bed/genscan/unmaskedRun + cat << '_EOF_' > gtfIdFix.pl +#!/usr/bin/env perl + +use strict; +use warnings; + +my $argc = scalar(@ARGV); + +if ($argc != 1) { + printf STDERR "usage: cat lifted/gtf/chrN.gtf | ./gtfIdFix.pl chrN\n"; + exit 255; +} + +my $C=shift; + +my $id = 0; +my $prevId = ""; +open (NM, ">nameFixed/newNames/${C}.tab") or die "can not write to nameFixed/newNames/${C}.tab"; +open (GT, ">nameFixed/gtf/${C}.gtf") or die "can not write to nameFixed/gtf/${C}.gtf"; +while (my $line=<>) { + chomp $line; + my $geneId = $line; + $geneId =~ s/^${C}.*gene_id "//; + $geneId =~ s/";.*//; + if ( $prevId ne $geneId) { + $id += 1; + printf NM "%s\t%s.%d\n", $geneId, $C, $id; + } + $line =~ s/${C}.[0-9]+.[0-9]+/${C}.$id/g; + printf GT "%s\n", $line; + $prevId = $geneId; +} +close (GT); +close (NM); +'_EOF_' + # << happy emacs + chmod +x gtfIdFix.pl + + rm -fr lifted + rm -fr nameFix + mkdir -p lifted + mkdir -p lifted/gtf + mkdir -p lifted/pep + mkdir -p lifted/subopt + mkdir -p nameFix + mkdir -p nameFix/gtf + mkdir -p nameFix/newNames + + for F in liftSpecs/hg38_* +do + L=`cut -f2 $F` + C=`cut -f4 $F` + liftUp -type=.gtf stdout ${F} error gtf/${L}.gtf >> lifted/gtf/${C}.gtf + cat pep/${L}.pep >> lifted/pep/${C}.pep + liftUp -type=.bed stdout ${F} error subopt/${L}.bed >> lifted/subopt/${C}.bed +done + + for F in lifted/gtf/*.gtf +do + C=`basename $F | sed -e 's/.gtf//'` + cat $F | ./gtfIdFix.pl $C +done + +mkdir -p nameFixed/pep + + cat << '_EOF_' > pepNameFix.pl +#!/usr/bin/env perl + +use strict; +use warnings; + +my $argc = scalar(@ARGV); +if ($argc != 1) { + printf STDERR "usage: ./pepNameFix.pl chrN > chrN.pep\n"; + exit 255 +} + +my $C = shift; +my %newName; + +open (FH, "<lifted/pep/$C.pep") or die "can not read <lifted/pep/$C.pep"; +open (NM, "<nameFixed/newNames/$C.tab") or die "can not read nameFixed/newNames/$C.tab"; +while (my $line = <NM>) { + chomp $line; + my ($needFix, $fixedName) = split('\t', $line); + $newName{$needFix} = $fixedName; +} +close (NM); + +while (my $line = <FH>) { + if ($line =~m /^>/) { + chomp $line; + $line =~ s/^>//; + die "can not find name to fix $line" if (!exists($newName{$line})); + printf ">%s\n", $newName{$line}; + } else { + print $line; + } +} +close (FH); +'_EOF_' + # << happy emacs + chmod +x pepNameFix.pl + + for F in lifted/pep/*.pep +do + C=`basename $F | sed -e 's/.pep//'` + echo $C + ./pepNameFix.pl $C > nameFixed/pep/$C.pep +done + +############################################################################# +# Mark the new centromere regions (DONE - 2014-01-09 - Hiram) + mkdir /hive/data/genomes/hg38/bed/centromere + cd /hive/data/genomes/hg38/bed/centromere + grep GJ ../../hg38.agp > hg38.centContigs.agp + + awk '{printf "%s\t%d\t%d\t%s\n", $1, $2-1, $3, $6}' hg38.centContigs.agp \ + > hg38.centContigs.bed4 + + hgLoadBed hg38 centromeres hg38.centContigs.bed4 + checkTableCoords hg38 centromeres + +############################################################################# +## alternate sequence/haplotype alignments (DONE - 2014-01-23 - Hiram) + mkdir /hive/data/genomes/hg38/bed/lastzAltSequences + cd /hive/data/genomes/hg38/bed/lastzAltSequences + +rm -fr hg38.haplotypes.lift temp.lift targetFa queryFa +mkdir targetFa +mkdir queryFa +touch temp.lift + +cat ../altLocations/chrToAlt.bed | while read L +do + chrName=`echo $L | awk '{print $1}'` + chromSize=`egrep "^$chrName " ../../chrom.sizes | cut -f2` + chrStart=`echo $L | awk '{if (($2-10000)>=0) {printf "%d", $2-10000} else {printf "0"}}'` + chrEnd=`echo $L | awk -v chromSize=$chromSize '{if (($3+10000)<=chromSize) {printf "%d", $3+10000} else {printf "%d", chromSize}}'` + chrSize=`echo $chrEnd $chrStart | awk '{print $1-$3}'` + queryName=`echo $L | awk '{print $4}'` + partName="${chrName}_${chrStart}_${chrEnd}" + echo $chrName $chrStart $chrEnd $queryName $partName $chromSize + echo -e "$chrStart\t${partName}\t$chrSize\t$chrName\t$chromSize" >> temp.lift + twoBitToFa ../../hg38.unmasked.2bit:$chrName:$chrStart-$chrEnd stdout | sed -e "s/^>.*/>$partName/;" > targetFa/$queryName.fa + twoBitToFa ../../hg38.unmasked.2bit:$queryName queryFa/$queryName.fa +done + +sort -u temp.lift | sort -k4,4 -k1,1n > hg38.haplotypes.lift + + # these were run serially on hgwdev, they could be a cluster run: + ssh ku + mkdir /hive/data/genomes/hg38/bed/lastzAltSequences/run.blastz + cd /hive/data/genomes/hg38/bed/lastzAltSequences/run.blastz + mkdir ../lav ../psl + + # construct the jobList + ls ../targetFa | sed -e 's/.fa//;' | while read partName +do + echo "./runJob.sh ${partName}" +done > jobList + + cat << '_EOF_' > runJob +#!/bin/sh + +export partName=$1 +export target="../targetFa/$partName.fa" +export query="../queryFa/$partName.fa" +export lav="../lav/$partName.lav" +export psl="../psl/$partName.psl" + +/cluster/bin/penn/lastz-distrib-1.03.46/bin/lastz \ + $target $query \ + Y=15000 T=2 M=254 O=600 H=2000 O=600 E=150 K=10000 L=10000 \ + Q=/scratch/data/blastz/human_chimp.v2.q > $lav +lavToPsl $lav stdout | liftUp $psl ../hg38.haplotypes.lift error stdin +'_EOF_' + # << happy emacs + + # these were run serially on hgwdev, they could be a cluster run: + time ./jobList > do.log + # real 61m35.898s + + # chaining lastz results: + mkdir -p /hive/data/genomes/hg38/bed/lastzAltSequences/axtChain/run/chain + cd /hive/data/genomes/hg38/bed/lastzAltSequences/axtChain/run + + ls ../../psl/*.psl | while read P +do + B=`basename $P | sed -e 's/.psl//'` + echo $B $P + ls -og $P ../../targetFa/${B}.fa ../../queryFa/${B}.fa + /cluster/home/hiram/kent/src/hg/mouseStuff/axtChain/axtChain \ + -psl -scoreScheme=/scratch/data/blastz/human_chimp.v2.q \ + -minScore=1000 -linearGap=medium $P \ + ../../../../hg38.unmasked.2bit \ + ../../../../hg38.unmasked.2bit stdout \ + | chainAntiRepeat ../../../../hg38.unmasked.2bit \ + ../../../../hg38.unmasked.2bit stdin chain/${B}.chain +done + + # real 7m54.677s + + cd /hive/data/genomes/hg38/bed/lastzAltSequences/axtChain + find ./run/chain -name "*.chain" | chainMergeSort -inputList=stdin \ + | nice gzip -c > hg38.haplotypes.all.chain.gz + chainPreNet hg38.haplotypes.all.chain.gz ../../../chrom.sizes \ + /hive/data/genomes/hg38/chrom.sizes stdout \ + | chainNet stdin -minSpace=1 ../../../chrom.sizes \ + ../../../chrom.sizes stdout /dev/null \ + | netSyntenic stdin noClass.net + + # Make liftOver chains from chroms to alternates: + netChainSubset -verbose=0 noClass.net hg38.haplotypes.all.chain.gz stdout \ + | chainStitchId stdin stdout | gzip -c > hg38.haplotypes.over.chain.gz + # swap the alignments to get the alternates to chrom mappings: + chainSwap hg38.haplotypes.over.chain.gz stdout \ + | gzip -c > hg38.reference.over.chain.gz + # and put them all together so mappings go both directions + chainMergeSort hg38.haplotypes.over.chain.gz hg38.reference.over.chain.gz \ + | gzip -c > hg38.haploReference.over.chain.gz + + hgLoadChain -tIndex hg38 chainAltSequence hg38.haploReference.over.chain.gz + netClass -verbose=0 -noAr noClass.net hg38 hg38 hg38.hg38AltSequence.net + netFilter -minGap=10 hg38.hg38AltSequence.net \ + | hgLoadNet -verbose=0 hg38 netAltSequence stdin + + chainToPsl hg38.haploReference.over.chain.gz ../../../chrom.sizes \ + ../../../chrom.sizes \ + /hive/data/genomes/hg38/hg38.unmasked.2bit \ + /hive/data/genomes/hg38/hg38.unmasked.2bit \ + hg38.beforeRecalc.haploReference.over.psl + + pslCheck -targetSizes=../../../chrom.sizes \ + -querySizes=../../../chrom.sizes \ + hg38.beforeRecalc.haploReference.over.psl 2>&1 | tail -1 + # checked: 3092 failed: 57 errors: 57 + + pslRecalcMatch hg38.beforeRecalc.haploReference.over.psl \ + ../../../hg38.unmasked.2bit ../../../hg38.unmasked.2bit \ + hg38.haploReference.over.psl + + pslCheck -targetSizes=../../../chrom.sizes \ + -querySizes=../../../chrom.sizes \ + hg38.haploReference.over.psl 2>&1 | tail -1 + # checked: 3092 failed: 0 errors: 0 + + hgLoadPsl hg38 -table=altSequenceLiftOver hg38.haploReference.over.psl + +############################################################################# +## construct non-bridged contig sequence (DONE - 2014-01-10 - Hiram) + mkdir /hive/data/genomes/hg38/bed/nonBridgedContigs + cd /hive/data/genomes/hg38/bed/nonBridgedContigs + + # only need the actual split chroms in this lift, and the + # _nn name is a bit more convenient than the .nn: + gapToLift -minGap=100 hg38 stdout | sed -e 's/\./_/;' \ + | awk '$1 != 0' > hg38.contigs.lift + # the warnings gapToLift issues are about gaps defined in the table + # that are abutting to each other. teleomere gaps are next to contig gaps + # those lifts in the format of a bed file: + awk '{printf "%s\t%d\t%d\t%s\n", $4, $1, $1+$3, $2}' hg38.contigs.lift \ + > hg38.contigs.bed + # the negation of that is the gaps between the contigs + # fixup the .N to _nn with the awk: + featureBits -not -countGaps hg38 hg38.contigs.bed -bed=stdout \ +| awk '{split($4,a,"."); printf "%s\t%d\t%d\t%s_%02d\n", $1,$2,$3,a[1],a[2]}' \ + > hg38.gaps.bed + # 268613637 bases of 3209286105 (8.370%) in intersection + + # together, those two should be %100 of the genome exactly: + featureBits -countGaps -or hg38 hg38.contigs.bed hg38.gaps.bed + # 3209286105 bases of 3209286105 (100.000%) in intersection + + # the list of all those other bits not in the split chroms: + egrep "_alt|chrUn|chrM|_random" hg38.gaps.bed | cut -f1 \ + | sort > other.bits.list + + # extract those chrom pieces and the other bits from the masked sequence: + (twoBitToFa -bed=hg38.contigs.bed ../../hg38.2bit stdout; \ + twoBitToFa -seqList=other.bits.list ../../hg38.2bit stdout) \ + | faToTwoBit stdin hg38.contigs.2bit + twoBitInfo hg38.contigs.2bit stdout | sort -k2nr > hg38.contigs.chrom.sizes + # verify nothing has been lost: + twoBitToFa ../../hg38.2bit stdout | faCount stdin | tail -1 +# total 3209286105 898285419 623727342 626335137 900967885 159970322 30979743 + twoBitToFa hg38.contigs.2bit stdout | faCount stdin | tail -1 +# total 3061688741 898285419 623727342 626335137 900967885 12372958 30979743 + # the ACGT and CPG counts remain the same, only N's have been lost + + # make a copy of this at the top: + cp -p hg38.contigs.2bit ../.. + cp -p hg38.contigs.lift ../../jkStuff + + # load as a track to be able to see where they are: + egrep "chrUn|chrM|_alt|_random" hg38.contigs.chrom.sizes \ + | awk '{printf "%s\t0\t%d\t%s\n", $1, $2, $1}' \ + > fullCoverage.hg38Contigs.bed + cat hg38.contigs.bed >> fullCoverage.hg38Contigs.bed + featureBits -or -countGaps hg38 fullCoverage.hg38Contigs.bed gap + # 3209286105 bases of 3209286105 (100.000%) in intersection + + hgLoadBed hg38 contigAlignmentSegments fullCoverage.hg38Contigs.bed + +############################################################################# +## analysis of repeat elements from each RM run +## (DONE - 2014-01-10 - Hiram) + mkdir /hive/data/genomes/hg38/bed/repeatElementCount + cd /hive/data/genomes/hg38/bed/repeatElementCount + for F in ../rmsk*/hg38.class.profile.txt \ + ../repeatMaskerGenbank/hg38.class.profile.txt +do + D=`dirname $F` + B=`basename $D | sed -e 's/repeatMaskerGenbank/NCBI/; s/rmsk//;'` + echo "==== $B ====" + grep rmskClass $F | sed -e 's#rmskClass/##; s/.tab//;' \ + | awk '{printf "%s\t%d\n", $2, $1}' | sort > ${B}.tab +done + + # Hmmer does not have snRNA and tRNA ? + echo -e "snRNA\t0" >> Hmmer.tab + echo -e "tRNA\t0" >> Hmmer.tab + sort Hmmer.tab > t.tab + mv t.tab Hmmer.tab + + echo "# Repeat Masker item counts" > table.result.txt + echo "# class NCBI cross-match rmblastn HMMER" >> table.result.txt + join NCBI.tab CM.tab | join - Blastn.tab | join - Hmmer.tab \ + | awk '{printf "%-15s\t%7d\t%7d\t%7d\t%7d\n", $1,$2,$3,$4,$5}' \ + | sort -k2,2nr >> table.result.txt + + cat table.result.txt +# Repeat Masker item counts +# class NCBI cross-match rmblastn HMMER +SINE 1849444 1852545 1822406 1884179 +LINE 1586141 1570523 1551012 1702529 +LTR 759248 748597 737799 805427 +DNA 502186 499108 485558 565171 +Simple_repeat 433789 703682 716968 636906 +Low_complexity 396378 102856 105181 95480 +Satellite 10198 7962 7703 10852 +LTR? 5884 5667 5068 9181 +snRNA 4595 4516 4548 0 +Retroposon 4163 5750 5630 11861 +Unknown 2802 5622 5263 3914 +DNA? 2157 3294 3018 4582 +tRNA 2154 2026 1983 0 +rRNA 1915 1840 1810 464 +RC 1860 1784 1706 2059 +srpRNA 1784 1672 1633 1517 +scRNA 1397 1420 1426 6783 +RNA 822 704 611 1484 +SINE? 488 38 38 970 +RC? 445 411 374 806 + +total 5567850 5520017 5459735 5744165 + +############################################################################# +## blat server turned on (DONE - 2014-01-13 - Hiram) +# After getting a blat server assigned by the Blat Server Gods, + ssh hgwdev + + hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \ + VALUES ("hg38", "blat4c", "17780", "1", "0"); \ + INSERT INTO blatServers (db, host, port, isTrans, canPcr) \ + VALUES ("hg38", "blat4c", "17781", "0", "1");' \ + hgcentraltest + # test it with some sequence + +############################################################################ +## reset default position to ABO gene (DONE - 2014-01-13 - Hiram) + ssh hgwdev + hgsql -e 'update dbDb set defaultPos="chr9:133252000-133280861" + where name="hg38";' hgcentraltest + +######################################################################### +## update grp table with new set of standard rows (DONE - 2014-01-29 - Hiram) + hgsql -e 'alter table grp rename grpOriginal;' hg38 + hgsql -e 'drop table grp;' hg38 + hgsql -e "create table grp (PRIMARY KEY(NAME)) select * from hg19.grp" hg38 + hgsql -e 'delete from grp where name="denisova";' hg38 + hgsql -e 'delete from grp where name="pub";' hg38 + hgsql -e 'delete from grp where name="neandertal";' hg38 + hgsql -e 'update grp set defaultIsClosed=0 where name="map";' hg38 + + hgsql -e 'drop table grpOriginal;' hg38 + +############################################################################ +# PREPARE LINEAGE SPECIFIC REPEAT FILES FOR LASTZ (DONE - 2014-01-21 - Hiram) + ssh ku + mkdir /hive/data/genomes/hg38/bed/linSpecRep + cd /hive/data/genomes/hg38/bed/linSpecRep + # create individual .out files from the master record in ../repeatMasker + mkdir splitOut + cat << '_EOF_' > split.csh +#!/bin/csh -fe +set C = $1 +head -3 ../repeatMasker/hg38.sorted.fa.out > splitOut/${C}.out +grep "${C} " ../repeatMasker/hg38.sorted.fa.out >> splitOut/${C}.out +'_EOF_' + # << happy emacs + chmod +x split.csh + + cat << '_EOF_' > template +#LOOP +split.csh $(root1) {check out line+ splitOut/$(root1).out} +#ENDLOOP +'_EOF_' + # << happy emacs + + # small ones first: + cut -f1 ../../chrom.sizes | tac > chrom.list + gensub2 chrom.list single template jobList + para create jobList + para try ... check ... push ... etc... +# Completed: 93 of 93 jobs +# CPU time in finished jobs: 127s 2.12m 0.04h 0.00d 0.000 y +# IO & Wait Time: 17154s 285.90m 4.76h 0.20d 0.001 y +# Average job time: 186s 3.10m 0.05h 0.00d +# Longest finished job: 224s 3.73m 0.06h 0.00d +# Submission to last job: 280s 4.67m 0.08h 0.00d + + # now, we can date and process each of those .out files + # constructing the humanSpecific set of repeats + # this means repeats found in human, and not in others + # using mouse here for 'others' is good enough, a variety + # of other species could be used (rat dog cow) where they all + # produce the same result + mkdir dateRepeats + cd dateRepeats + cat << '_EOF_' > mkLSR +#!/bin/bash +set -beEu -o pipefail +rm -f $1.out_mus-musculus +ln -s ../splitOut/$1.out . +/scratch/data/RepeatMasker/DateRepeats $1.out -query human -comp mouse +rm $1.out +mkdir -p ../humanSpecific +/cluster/bin/scripts/extractRepeats 1 $1.out_mus-musculus \ + > ../humanSpecific/$1.out.spec +'_EOF_' + # << happy emacs + chmod +x mkLSR + + cat << '_EOF_' > template +#LOOP +./mkLSR $(path1) {check out line+ ../humanSpecific/$(path1).out.spec} +#ENDLOOP +'_EOF_' + # << happy emacs + + gensub2 ../chrom.list single template jobList + para try ... check ... push ... etc... + para time +# Completed: 455 of 455 jobs +# CPU time in finished jobs: 13985s 233.08m 3.88h 0.16d 0.000 y +# IO & Wait Time: 1470s 24.50m 0.41h 0.02d 0.000 y +# Average job time: 34s 0.57m 0.01h 0.00d +# Longest finished job: 111s 1.85m 0.03h 0.00d +# Submission to last job: 1427s 23.78m 0.40h 0.02d + + + # We also need the nibs for blastz runs with lineage specific repeats + mkdir /hive/data/genomes/hg38/bed/nibs + cd /hive/data/genomes/hg38/bed/nibs + cut -f1 ../../chrom.sizes | while read C +do + twoBitToFa -seq=${C} ../../hg38.2bit stdout \ + | faToNib -softMask stdin ${C}.nib + echo "${C} done" +done + + # verify nothing lost + cat ../../chrom.sizes \ + | awk '{printf "nibFrag -masked %s.nib 0 %d + stdout\n", $1, $2}' \ + | sh | faSize stdin +# 3209286105 bases (159970322 N's 3049315783 real 1460684798 upper +# 1588630985 lower) in 455 sequences in 1 files +# Total size: mean 7053376.1 sd 31548372.6 +# min 970 (chrUn_KI270394v1.nib:0-970) +# max 248956422 (chr1.nib:0-248956422) median 161218 +# %49.50 masked total, %52.10 masked real + + mkdir /hive/data/staging/data/hg38/nib + rsync -a --progress ./ /hive/data/staging/data/hg38/nib + +############################################################################# +## GRC Contigs/ctgPos2 track (DONE - 2014-12-25 - Hiram) + # provide mapping of UCSC chrom names to GRC names + mkdir /hive/data/genomes/hg38/bed/ctgPos2 + cd /hive/data/genomes/hg38/bed/ctgPos2 + grep -v "^#" ../../genbank/GCA_000001405.15_GRCh38_top-level.acc2name \ + | awk '{printf "s/^%s/%s/g;\n", $1, $3}' > accessionToUcsc.sed.txt + + find ../../genbank -type f | grep "/assembled_chromosomes/AGP/" | sed -e 's/.comp//' | while read F +do + if [ -s $F ]; then + zcat $F | grep -v "^#" + fi +done | sed -e "`cat accessionToUcsc.sed.txt`" > ucsc.grch38.agp + + awk '$5 != "N"' ucsc.grch38.agp \ +| awk '{printf "%s\t%d\t%s\t%d\t%d\t%s\n", $6, $3-$2+1, $1, $2-1, $3, $5}' \ + | sort -u | sort -k3,3 -k4,4n > ctgPos2.tab + + + export ctgSize=`awk '{print length($1)}' ctgPos2.tab | sort -n | tail -1` + export chrSize=`awk '{print length($3)}' ctgPos2.tab | sort -n | tail -1` + + sed -e "s/20/$ctgSize/; s/16/$chrSize/;" \ + /cluster/home/hiram/kent/src/hg/lib/ctgPos2.sql > hg38.ctgPos2.sql + + hgLoadSqlTab hg38 ctgPos2 hg38.ctgPos2.sql ctgPos2.tab + +############################################################################ +# constructing download files (WORKING - 2014-01-15 - Hiram) + # add hg38 to all.joiner and verify it is clean: + joinerCheck -database=hg38 -keys all.joiner +# Checking keys on database hg38 +# hg38.ucscToINSDC.chrom - hits 455 of 455 (100.000%) ok + # and all table coordinates are OK: + checkTableCoords hg38 + + cd /hive/data/genomes/hg38 + time $HOME/kent/src/hg/utils/automation/makeDownloads.pl \ + -workhorse=hgwdev hg38 + # makeDownloads.pl has made a preliminary set of files + + # need to fixup these names and add chromFa.tar.gz files + cd /hive/data/genomes/hg38/goldenPath/bigZips + + mkdir chroms + mkdir maskedChroms + + faSplit byname hg38.fa.gz chroms/ + faSplit byname hg38.fa.masked.gz maskedChroms/ + + tar cvzf ./hg38.chromFa.tar.gz ./chroms/ + tar cvzf ./hg38.chromFaMasked.tar.gz ./maskedChroms/ + + cd /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/bigZips + ln -s /hive/data/genomes/hg38/goldenPath/bigZips/hg38.chromFa.tar.gz hg38.chromFa.tar.gz + ln -s /hive/data/genomes/hg38/goldenPath/bigZips/hg38.chromFaMasked.tar.gz hg38.chromFaMasked.tar.gz + + #also added entries for above to md5sum.txt and README.txt + +############################################################################ +# LASTZ MOUSE Mm10 (DONE - 2014-01-23,31 - Hiram) + # can no longer use the lineage specific repeats with the new lastz + # use a screen to manage this longish job: + screen -S hg38Mm10 + + mkdir /hive/data/genomes/hg38/bed/lastzMm10.2014-01-23 + cd /hive/data/genomes/hg38/bed/lastzMm10.2014-01-23 + + # best to always specify an exact path to lastz so we know which one is used + # lastz default parameters are human-mouse parameters + + cat << '_EOF_' > DEF +# human vs mouse +BLASTZ=/cluster/bin/penn/lastz-distrib-1.03.52/bin/lastz + +# TARGET: Human Hg38 +SEQ1_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ1_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ1_CTGDIR=/hive/data/genomes/hg38/hg38.contigs.2bit +SEQ1_CTGLEN=/hive/data/genomes/hg38/hg38.contigs.chrom.sizes +SEQ1_LIFT=/hive/data/genomes/hg38/jkStuff/hg38.contigs.lift +SEQ1_CHUNK=40000000 +SEQ1_LAP=10000 + +# QUERY: Mouse Mm10 +SEQ2_DIR=/scratch/data/mm10/mm10.2bit +SEQ2_LEN=/scratch/data/mm10/chrom.sizes +SEQ2_CHUNK=20000000 +SEQ2_LAP=0 + +BASE=/hive/data/genomes/hg38/bed/lastzMm10.2014-01-23 +TMPDIR=/dev/shm +'_EOF_' + # << happy emacs + + time $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \ + -verbose=2 \ + -stop=net `pwd`/DEF \ + -syntenicNet -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -fileServer=hgwdev \ + -chainMinScore=3000 -chainLinearGap=medium > net.log 2>&1 + # real 1494m26.135s ---- busy cluster + time $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \ + -verbose=2 \ + -continue=load `pwd`/DEF \ + -syntenicNet -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -fileServer=hgwdev \ + -chainMinScore=3000 -chainLinearGap=medium > load.log 2>&1 + # Elapsed time: 43m11s + cat fb.hg38.chainMm10Link.txt + # 964465044 bases of 3049335806 (31.629%) in intersection + + # and the swap + mkdir /hive/data/genomes/mm10/bed/blastz.hg38.swap + cd /hive/data/genomes/mm10/bed/blastz.hg38.swap + time nice -n +19 doBlastzChainNet.pl -verbose=2 \ + /hive/data/genomes/hg38/bed/lastzMm10.2014-01-23/DEF \ + -swap -syntenicNet \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 + # real 83m28.397s + + cat fb.mm10.chainHg38Link.txt + # 937030766 bases of 2652783500 (35.323%) in intersection + +######################################################################### +# LASTZ Dog CanFam3 (DONE - 2014-01-26 - Hiram) + mkdir /hive/data/genomes/hg38/bed/lastzCanFam3.2014-01-26 + cd /hive/data/genomes/hg38/bed/lastzCanFam3.2014-01-26 + + cat << '_EOF_' > DEF +# human vs dog +BLASTZ=/cluster/bin/penn/lastz-distrib-1.03.52/bin/lastz + +# TARGET: Human Hg38 +SEQ1_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ1_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ1_CTGDIR=/hive/data/genomes/hg38/hg38.contigs.2bit +SEQ1_CTGLEN=/hive/data/genomes/hg38/hg38.contigs.chrom.sizes +SEQ1_LIFT=/hive/data/genomes/hg38/jkStuff/hg38.contigs.lift +SEQ1_CHUNK=20000000 +SEQ1_LAP=10000 + +# QUERY: Dog CanFam3 +SEQ2_DIR=/hive/data/genomes/canFam3/canFam3.2bit +SEQ2_LEN=/hive/data/genomes/canFam3/chrom.sizes +SEQ2_CHUNK=20000000 +SEQ2_LAP=0 + +BASE=/hive/data/genomes/hg38/bed/lastzCanFam3.2014-01-26 +TMPDIR=/dev/shm +'_EOF_' + # << happy emacs + + # establish a screen to control this job + screen hg38CanFam3 + time $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl -verbose=2 \ + `pwd`/DEF \ + -syntenicNet \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -chainMinScore=3000 -chainLinearGap=medium > do.log 2>&1 + # Elapsed time: 1396m22s - busy cluster + cat fb.hg38.chainCanFam3Link.txt + # 1523987456 bases of 3049335806 (49.978%) in intersection + + # running the swap + mkdir /hive/data/genomes/canFam3/bed/blastz.hg38.swap + cd /hive/data/genomes/canFam3/bed/blastz.hg38.swap + time nice -n +19 doBlastzChainNet.pl -verbose=2 \ + /hive/data/genomes/hg38/bed/lastzCanFam3.2014-01-26/DEF \ + -syntenicNet -swap \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 + # real 107m57.787s + + cat fb.canFam3.chainHg38Link.txt + # 1437624815 bases of 2392715236 (60.083%) in intersection + +######################################################################### +# LASTZ Macaca Mulatta RheMac3 (DONE - 2014-01-27,02-10 - Hiram) + mkdir /hive/data/genomes/hg38/bed/lastzRheMac3.2014-01-27 + cd /hive/data/genomes/hg38/bed/lastzRheMac3.2014-01-27 + + # best to always specify an exact path to lastz so we know which one is used + # lastz default parameters are human-mouse parameters + + cat << '_EOF_' > DEF +# human vs macaca mulatta +BLASTZ=/cluster/bin/penn/lastz-distrib-1.03.52/bin/lastz +# maximum M allowed with lastz is only 254 +BLASTZ_M=254 +BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q +BLASTZ_O=600 +BLASTZ_E=150 +# other parameters from panTro2 vs hg18 lastz on advice from Webb +BLASTZ_K=4500 +BLASTZ_Y=15000 +BLASTZ_T=2 + +# TARGET: Human Hg38 +SEQ1_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ1_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ1_CTGDIR=/hive/data/genomes/hg38/hg38.contigs.2bit +SEQ1_CTGLEN=/hive/data/genomes/hg38/hg38.contigs.chrom.sizes +SEQ1_LIFT=/hive/data/genomes/hg38/jkStuff/hg38.contigs.lift +SEQ1_CHUNK=20000000 +SEQ1_LAP=10000 + +# QUERY: Macaca Mulatta RheMac3 +SEQ2_DIR=/scratch/data/rheMac3/rheMac3.2bit +SEQ2_LEN=/scratch/data/rheMac3/chrom.sizes +SEQ2_CHUNK=20000000 +SEQ2_LAP=0 +SEQ2_IN_CONTIGS=0 + +BASE=/hive/data/genomes/hg38/bed/lastzRheMac3.2014-01-27 +TMPDIR=/dev/shm +'_EOF_' + # << happy emacs + time $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \ + `pwd`/DEF \ + -syntenicNet -fileServer=hgwdev \ + -chainMinScore=5000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku > do.log 2>&1 + # Elapsed time: 1426m43s - busy cluster + cat fb.hg38.chainRheMac3Link.txt + # 2431208700 bases of 3049335806 (79.729%) in intersection + + # running the swap + mkdir /hive/data/genomes/rheMac3/bed/blastz.hg38.swap + cd /hive/data/genomes/rheMac3/bed/blastz.hg38.swap + time nice -n +19 doBlastzChainNet.pl -verbose=2 \ + /hive/data/genomes/hg38/bed/lastzRheMac3.2014-01-27/DEF \ + -swap -syntenicNet -chainMinScore=5000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku > swap.log 2>&1 + # 82m32.329s + cat fb.rheMac3.chainHg38Link.txt + # 2288533769 bases of 2639145830 (86.715%) in intersection + +######################################################################### +## construct analysis set (DONE - 2014-01-27 - Hiram) + mkdir /hive/data/genomes/hg38/bed/analysisSet + cd /hive/data/genomes/hg38/bed/analysisSet + mkdir -p splitFa + + faToTwoBit \ +../../genbank/seqs_for_alignment_pipelines/GCA_000001405.15_GRCh38_no_alt_analysis_set.fna.gz \ + hg38.unmasked.analysisSet.2bit + + faCount splitFa/c*.fa > splitFa.faCount.txt + + egrep -v "chr[0-9_BGHIJKLXv]*_alt" ../rmskCM/hg38.sorted.fa.out \ + > hg38.analysisSet.out + + twoBitMask hg38.unmasked.analysisSet.2bit hg38.analysisSet.out \ + hg38.rmsk.analysisSet.2bit + + egrep -v "chr[0-9_BGHIJKLXv]*_alt" ../simpleRepeat/trfMask.bed \ + > trfMask.analysisSet.bed + + twoBitMask hg38.rmsk.analysisSet.2bit -add trfMask.analysisSet.bed \ + hg38.analysisSet.2bit + + twoBitToFa hg38.unmasked.analysisSet.2bit stdout | faSize stdin +# 3099922541 bases (165046090 N's 2934876451 real 2934876451 upper 0 lower) +# in 195 sequences in 1 files +# Total size: mean 15897038.7 sd 46804464.6 min 970 (chrUn_KI270394v1) +# max 248956422 (chr1) median 32032 +# %0.00 masked total, %0.00 masked real + + twoBitToFa hg38.analysisSet.2bit stdout | faSize stdin +# 3099922541 bases (165046090 N's 2934876451 real 1409378896 upper 1525497555 +# lower) in 195 sequences in 1 files +# Total size: mean 15897038.7 sd 46804464.6 min 970 (chrUn_KI270394v1) +# max 248956422 (chr1) median 32032 +# %49.21 masked total, %51.98 masked real + + mkdir hg38.analysisSet.chroms + twoBitToFa hg38.analysisSet.2bit stdout \ + | faSplit byname stdin hg38.analysisSet.chroms/ + + tar cvzf ./hg38.analysisSet.chroms.tar.gz ./hg38.analysisSet.chroms + + ln -s `pwd`/hg38.analysisSet.2bit \ + /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/bigZips + ln -s `pwd`/hg38.analysisSet.chroms.tar.gz \ + /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/bigZips + # add these md5 sums to md5sum.txt + md5sum hg38.analysisSet.2bit hg38.analysisSet.chroms.tar.gz >> \ + /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/bigZips/md5sum.txt + + cp ../../genbank/README_ANALYSIS_SETS README.analysisSet.txt + # add note at the top of README: + ###################################################################### + UCSC copy of the file from: + + ftp://ftp.ncbi.nlm.nih.gov/genbank/genomes/Eukaryotes/vertebrates_mammals/Homo_sapiens/GRCh38/seqs_for_alignment_pipelines/README_ANALYSIS_SETS + + ln -s `pwd`/README.analysisSet.txt \ + /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/bigZips + +######################################################################### +# the FULL analysis set (DONE - 2014-03-18 - Hiram + mkdir /hive/data/genomes/hg38/bed/fullAnalysisSet + cd /hive/data/genomes/hg38/bed/fullAnalysisSet + + mkdir hg38.fullAnalysisSet.chroms + twoBitToFa ../analysisSet/hg38.analysisSet.2bit stdout \ + | faSplit byname stdin hg38.fullAnalysisSet.chroms/ + + grep _alt ../../chrom.sizes | cut -f 1 > alt.list + + twoBitToFa -seqList=alt.list ../../hg38.2bit stdout \ + | faSplit byname stdin hg38.fullAnalysisSet.chroms/ + + faCount hg38.fullAnalysisSet.chroms/chr*.fa > faCount.fullAnalysisSet.txt + + faToTwoBit hg38.fullAnalysisSet.chroms/chr*.fa hg38.fullAnalysisSet.2bit + twoBitInfo hg38.fullAnalysisSet.2bit stdout | sort -k2nr > chrom.sizes + + tar cvzf ./hg38.fullAnalysisSet.chroms.tar.gz ./hg38.fullAnalysisSet.chroms + +######################################################################### +# LASTZ Self/hg38 (DONE - 2014-01-25,02-10 - Hiram) + # can no longer use the lineage specific repeats with the new lastz + # use a screen to manage this longish job: + screen -S hg38Self + + mkdir /hive/data/genomes/hg38/bed/lastzSelf.2014-01-25 + cd /hive/data/genomes/hg38/bed/lastzSelf.2014-01-25 + # construct the non-bridged contigs sequence to use: + (twoBitToFa ../nonBridgedContigs/hg38.chroms.contigs.2bit stdout; + twoBitToFa ../../hg38.2bit:chrM stdout) | faToTwoBit stdin hg38.self.2bit + twoBitInfo hg38.self.2bit stdout | sort -k2nr > hg38.self.chrom.sizes + + # best to always specify an exact path to lastz so we know which one is used + # lastz default parameters are human-mouse parameters + + cat << '_EOF_' > DEF +# human vs human with mouse defaults +BLASTZ=/cluster/bin/penn/lastz-distrib-1.03.52/bin/lastz + +# TARGET: Human Hg38 +SEQ1_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ1_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ1_CTGDIR=/hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/hg38.self.2bit +SEQ1_CTGLEN=/hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/hg38.self.chrom.sizes +SEQ1_LIFT=/hive/data/genomes/hg38/jkStuff/hg38.contigs.lift +SEQ1_CHUNK=20000000 +SEQ1_LAP=10000 + +# QUERY: Human Hg38 +SEQ2_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ2_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ2_CTGDIR=/hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/hg38.self.2bit +SEQ2_CTGLEN=/hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/hg38.self.chrom.sizes +SEQ2_LIFT=/hive/data/genomes/hg38/jkStuff/hg38.contigs.lift +SEQ2_CHUNK=20000000 +SEQ2_LAP=0 + +BASE=/hive/data/genomes/hg38/bed/lastzSelf.2014-01-25 +TMPDIR=/dev/shm +'_EOF_' +_EOF_ + + time $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \ + -verbose=2 \ + -stop=net `pwd`/DEF \ + -syntenicNet -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -fileServer=hgwdev \ + -chainMinScore=3000 -chainLinearGap=medium > net.log 2>&1 + # real 1518m15.817s -- problems + # there was a problem in the 'part014' batch. running that manually: + mkdir /hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/run.blastz/lastJob + cd /hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/run.blastz/lastJob + # make 100 jobs out of the 10 parts: + mkdir -p psl + cp ../tParts/part014.lst ./xpart014.lst + split -l 1 xpart014.lst -d -a 3 part + for F in part0* +do + mv $F $F.lst +done + +for T in part0*.lst +do + for Q in part0*.lst + do + mkdir -p psl/${T} + echo /cluster/home/hiram/kent/src/hg/utils/automation/blastz-run-ucsc -outFormat psl -dropSelf ${T} ${Q} ../../DEF \{check out exists psl/${T}/${T}.${Q}.psl\} + done +done > jobList + para -ram=32g create jobList + para push + # one last failing job: +# Completed: 99 of 100 jobs +# CPU time in finished jobs: 2836s 47.27m 0.79h 0.03d 0.000 y +# IO & Wait Time: 279s 4.65m 0.08h 0.00d 0.000 y +# Average job time: 31s 0.52m 0.01h 0.00d +# Longest finished job: 586s 9.77m 0.16h 0.01d +# Submission to last job: 620s 10.33m 0.17h 0.01d + + mkdir /hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/run.blastz/lastJob/split010 + cd /hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/run.blastz/lastJob/split010 + mkdir psl + + twoBitToFa /hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/hg38.self.2bit:chr16_03:0-1689648 part010.fa + + faSplit -lift=split010.lift size part010.fa 169000 split010_ +TOP="/hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/run.blastz/lastJob/split010" + +for T in split*.fa +do + mkdir -p psl/${T} + echo "${TOP}/${T}" > ${T}.lst + faToTwoBit ${T} ${T}.2bit + for Q in split*.fa + do + echo "/cluster/home/hiram/kent/src/hg/utils/automation/blastz-run-ucsc -outFormat psl -dropSelf ${T}.lst ${Q}.lst DEF {check out exists psl/${T}/${T}.${Q}.psl}" + done +done > jobList + para -ram=32g create jobList + +# Completed: 100 of 100 jobs +# CPU time in finished jobs: 176579s 2942.99m 49.05h 2.04d 0.006 y +# IO & Wait Time: 1239s 20.64m 0.34h 0.01d 0.000 y +# Average job time: 1778s 29.64m 0.49h 0.02d +# Longest finished job: 29343s 489.05m 8.15h 0.34d +# Submission to last job: 29348s 489.13m 8.15h 0.34d + + catDir psl/* | grep -v "^#" > raw.psl + + liftUp -type=.psl stdout split010.lift error raw.psl \ + | liftUp -pslQ -type=.psl chr16_03.psl split010.lift error stdin + + # this combination allowed psl headers to sneak in the middle, + # had to be cleaned: + catDir psl/* | grep -v "^#" > part014.psl + cat split010/chr16_03.psl >> part014.psl + cp -p part014.psl ../../psl/part014.lst/part014.lst_part014.lst.psl + + time $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \ + -verbose=2 \ + -continue=cat -stop=net `pwd`/DEF \ + -syntenicNet -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -fileServer=hgwdev \ + -chainMinScore=3000 -chainLinearGap=medium > cat.log 2>&1 + # real 43m11.340s + # failed in chaining, running manually on hgwdev + time ./bigJobs.sh > bigJobs.log 2>&1 + # real 468m59.648s + + time ./part014.sh > part014.log 2>&1 + + # real 1319m57.911s + # -rw-rw-r-- 1 3581498246 Feb 8 14:37 part014.lst.chain + time $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \ + -verbose=2 \ + -continue=chainMerge -stop=net `pwd`/DEF \ + -syntenicNet -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -fileServer=hgwdev \ + -chainMinScore=3000 -chainLinearGap=medium > chainMerge.log 2>&1 + + time $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \ + -verbose=2 \ + -continue=load -stop=load `pwd`/DEF \ + -syntenicNet -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -fileServer=hgwdev \ + -chainMinScore=3000 -chainLinearGap=medium > load.log 2>&1 + + hgLoadChain -normScore -tIndex hg38 chainSelf hg38.hg38.all.chain.gz + # Loading 104815249 chains into hg38.chainSelf + + cat fb.hg38.chainSelfLink.txt + # 392419010 bases of 3049335806 (12.869%) in intersection + cd /hive/data/genomes/hg38/bed + ln -s lastzSelf.2014-01-25 lastz.self + ln -s lastzSelf.2014-01-25 lastz.hg38 + +######################################################################### +## 4-Way Multiz for UCSC Genes construction (DONE - 2014-02-11 - Hiram) + ssh hgwdev + mkdir /hive/data/genomes/hg38/bed/multiz4way + cd /hive/data/genomes/hg38/bed/multiz4way + + # extract our 4 organisms from the 44-way on hg18: + ln -s /hive/data/genomes/hg18/bed/multiz44way/44way.4d.nh ./44way.nh + + /cluster/bin/phast/tree_doctor \ + --prune-all-but hg19,mm10,canFam3,rheMac3 $HOME/kent/src/hg/utils/phyloTrees/120way.nh \ + | sed -e "s/hg19/hg38/" > 4way.nh + + # this looks like: + cat 4way.nh +(((hg38:0.033974,rheMac3:0.037601):0.109934,mm10:0.356483):0.020593,canFam3:0.165928); + + + # Use this specification in the phyloGif tool: + # http://genome.ucsc.edu/cgi-bin/phyloGif + # to obtain a gif image for htdocs/images/phylo/hg38_4way.gif + + /cluster/bin/phast/all_dists 4way.nh > 4way.distances.txt + # Use this output to create the table below + grep -y hg38 4way.distances.txt | sort -k3,3n +# +# If you can fill in all the numbers in this table, you are ready for +# the multiple alignment procedure +# +# featureBits chainLink measures +# chainHg38Link chain linearGap +# distance on hg38 on other minScore +# 1 0.071575 - rhesus rheMac3 (% 79.729) (% 86.715) 5000 medium +# 2 0.330429 - dog canFam3 (% 49.978) (% 60.083) 3000 medium +# 3 0.500391 - mouse mm10 (% 31.629) (% 35.323) 3000 medium + + # using the syntenic nets + cd /cluster/data/hg38/bed/multiz4way + mkdir mafLinks + cd mafLinks + mkdir rheMac3 canFam3 mm10 + + for D in mm10 canFam3 rheMac3 +do + ln -s ../../../lastz.${D}/axtChain/hg38.${D}.synNet.maf.gz ./${D}/ +done + + mkdir /hive/data/genomes/hg38/bed/multiz4way/mafSplit + cd /hive/data/genomes/hg38/bed/multiz4way/mafSplit + for D in mm10 canFam3 rheMac3 +do + echo "working: ${D}" + zcat ../mafLinks/${D}/hg38.${D}.synNet.maf.gz > ${D}.maf + mkdir -p ${D} + mafSplit -byTarget -useFullSequenceName /dev/null ${D}/${D}_ ${D}.maf + rm -f ${D}.maf +done + + # determine what is the newest version of multiz and use that + cd /hive/data/genomes/hg38/bed/multiz4way + mkdir penn + cp -p /cluster/bin/penn/multiz.2009-01-21_patched/multiz penn + cp -p /cluster/bin/penn/multiz.2009-01-21_patched/maf_project penn + cp -p /cluster/bin/penn/multiz.2009-01-21_patched/autoMZ penn + + # the autoMultiz cluster run + ssh ku + cd /hive/data/genomes/hg38/bed/multiz4way + + # create species list and stripped down tree for autoMZ + sed 's/[a-z][a-z]*_//g; s/:[0-9\.][0-9\.]*//g; s/;//; /^ *$/d' \ + 4way.nh > tmp.nh + echo `cat tmp.nh` | sed 's/ //g; s/,/ /g' > tree.nh + sed 's/[()]//g; s/,/ /g' tree.nh > species.lst + + mkdir run maf + cd run + + # NOTE: you need to set the db and multiz dirname properly in this script + cat > autoMultiz << '_EOF_' +#!/bin/csh -ef +set db = hg38 +set c = $1 +set maf = $2 +set binDir = /hive/data/genomes/hg38/bed/multiz4way/penn +set tmp = /dev/shm/$db/multiz.$c +set pairs = /hive/data/genomes/hg38/bed/multiz4way/mafSplit +rm -fr $tmp +mkdir -p $tmp +cp ../{tree.nh,species.lst} $tmp +pushd $tmp +foreach s (`cat species.lst`) + set in = $pairs/$s/${s}_$c.maf + set out = $db.$s.sing.maf + if ($s == $db) then + continue + endif + if (-e $in.gz) then + zcat $in.gz > $out + else if (-e $in) then + cp $in $out + else + echo "##maf version=1 scoring=autoMZ" > $out + endif +end +set path = ($binDir $path); rehash +$binDir/autoMZ + T=$tmp E=$db "`cat tree.nh`" $db.*.sing.maf $c.maf +popd +cp $tmp/$c.maf $maf +rm -fr $tmp +'_EOF_' + # << happy emacs + chmod +x autoMultiz + +cat << '_EOF_' > template +#LOOP +./autoMultiz $(root1) {check out line+ /hive/data/genomes/hg38/bed/multiz4way/maf/$(root1).maf} +#ENDLOOP +'_EOF_' + # << happy emacs + + cut -f1 /cluster/data/hg38/chrom.sizes > chrom.lst + gensub2 chrom.lst single template jobList + para create jobList + # 455 jobs + para try ... check ... push ... etc ... +# Completed: 455 of 455 jobs +# CPU time in finished jobs: 50111s 835.18m 13.92h 0.58d 0.002 y +# IO & Wait Time: 5574s 92.91m 1.55h 0.06d 0.000 y +# Average job time: 122s 2.04m 0.03h 0.00d +# Longest finished job: 4717s 78.62m 1.31h 0.05d +# Submission to last job: 4722s 78.70m 1.31h 0.05d + + # combine results into a single file for loading and gbdb reference + cd /hive/data/genomes/hg38/bed/multiz4way + grep "^#" maf/chr19_GL949749v2_alt.maf | grep -v "eof maf" > multiz4way.maf + grep -h -v "^#" maf/*.maf >> multiz4way.maf + grep "^#" maf/chr19_GL949749v2_alt.maf | grep "eof maf" >> multiz4way.maf + # real 3m27.561s + + # makes a 8.5 Gb file: + # -rw-rw-r-- 1 9044143788 Feb 11 12:51 multiz4way.maf + + # Load into database + ssh hgwdev + cd /hive/data/genomes/hg38/bed/multiz4way + mkdir /gbdb/hg38/multiz4way + ln -s /hive/data/genomes/hg38/bed/multiz4way/multiz4way.maf \ + /gbdb/hg38/multiz4way + # the hgLoadMaf generates huge tmp files, locate them in /dev/shm + cd /dev/shm + time nice -n +19 hgLoadMaf hg38 multiz4way + # Loaded 6141667 mafs in 1 files from /gbdb/hg38/multiz4way + # real 2m2.812s + + cd /hive/data/genomes/hg38/bed/multiz4way + time (cat /gbdb/hg38/multiz4way/*.maf \ + | hgLoadMafSummary -verbose=2 -minSize=10000 \ + -mergeGap=500 -maxSize=50000 hg38 multiz4waySummary stdin) + # Created 1266559 summary blocks from 11780291 components and 6141667 mafs + # real 3m0.791s +# -rw-rw-r-- 1 311246327 Feb 11 12:54 multiz4way.tab +# -rw-rw-r-- 1 58730176 Feb 11 12:58 multiz4waySummary.tab + wc -l multiz4way* + # 6141667 multiz4way.tab + # 1266559 multiz4waySummary.tab + # 7408226 total + +######################################################################### +## RE-load alternate sequence for PSL display (DONE - 2016-01-15 - Hiram) +## The procedure below +## "load alternate sequence for PSL display (DONE - #2014-02-24 - Hiram) +## produced an illegal psl Table altSeqLiftOverPsl: + pslCheck -db=hg38 altSeqLiftOverPsl + checked: 266 failed: 264 errors: 1046 + +## Since then, the gff3ToPsl command has been updated to be a bit more +## robust, so, the following sequence produces the new alignment file: + mkdir -p /hive/data/genomes/hg38/bed/altAlignments/redo2016 + cd /hive/data/genomes/hg38/bed/altAlignments/redo2016 + +mkdir -p ucscPsl + +awk -F'/' '{printf "s/^%s\t/%s\t/g;\n", $3,$2}' ../accessionToUcsc.sed.txt \ + > ucscToNcbi.sed.txt + +sed -f ucscToNcbi.sed.txt ../../../chrom.sizes > ncbi.chrom.sizes + +paste ncbi.chrom.sizes ../../../chrom.sizes \ + | awk -F'\t' '{printf "0\t%s\t%d\t%s\t%d\n", $1,$2,$3,$4}' \ + > ncbiToUcsc.lift + +find ../../../genbank -type f | grep alt_scaffolds | grep "\.gff$" \ + | while read gff +do + name=`basename $gff | sed -e 's/_.*//;'` + fasta=`dirname $gff | sed -e 's#alignments#FASTA/alt.scaf.fa.gz#;'` + size=`faCount $fasta | grep -w total | cut -f2` + printf "%s\t%d\n" "$name" "$size" > target.sizes + gff3ToPsl ncbi.chrom.sizes target.sizes $gff $name.psl + pslCheck ${name}.psl + liftUp -type=.psl stdout ncbiToUcsc.lift error ${name}.psl \ + | liftUp -type=.psl -pslQ ucscPsl/${name}.psl ncbiToUcsc.lift error stdin + pslCheck ucscPsl/${name}.psl +done + + pslSort dirs altSeqLiftOverPsl.psl ./tmp ucscPsl + pslCheck -db=hg38 altSeqLiftOverPsl.psl + + hgLoadPsl hg38 altSeqLiftOverPsl.psl + pslCheck -db=hg38 altSeqLiftOverPsl + # checked: 266 failed: 0 errors: 0 + +######################################################################### +## load alternate sequence for PSL display (DONE - 2014-02-24 - Hiram) + mkdir /hive/data/genomes/hg38/bed/altAlignments/sequence + cd /hive/data/genomes/hg38/bed/altAlignments/sequence + + rm -fr hg38.haplotypes.lift temp.lift targetFa queryFa + mkdir targetFa + mkdir queryFa + touch temp.lift + + cat ../../altLocations/chrToAlt.bed | while read L +do + chrName=`echo $L | awk '{print $1}'` + chromSize=`egrep "^$chrName " ../../../chrom.sizes | cut -f2` + chrStart=`echo $L | awk '{printf "%d", $2}'` + chrEnd=`echo $L | awk '{printf "%d", $3}'` + chrSize=`echo $chrEnd $chrStart | awk '{print $1-$3}'` + queryName=`echo $L | awk '{print $4}'` + partName="${chrName}_${chrStart}_${chrEnd}" + echo $chrName $chrStart $chrEnd $queryName $partName $chromSize + echo -e "$chrStart\t${partName}\t$chrSize\t$chrName\t$chromSize" >> temp.lift + twoBitToFa ../../../hg38.2bit:$chrName:$chrStart-$chrEnd stdout | sed -e "s/^>.*/>$partName/;" > targetFa/$queryName.fa + twoBitToFa ../../../hg38.2bit:$queryName queryFa/$queryName.fa +done + +sort -u temp.lift | sort -k4,4 -k1,1n > hg38.haplotypes.lift + + mkdir /gbdb/hg38/ncbiAltMappings + cd /hive/data/genomes/hg38/bed/altAlignments/sequence/queryFa + ln -s `pwd`/*.fa /gbdb/hg38/ncbiAltMappings + cd /hive/data/genomes/hg38/bed/altAlignments/sequence + hgLoadSeq -drop -seqTbl=seqNcbiAltSequence -extFileTbl=extNcbiAltSequence \ + hg38 /gbdb/hg38/ncbiAltMappings/*.fa + + pslSwap ../altAlignments.psl stdout \ + | pslRecalcMatch stdin ../../../hg38.2bit ../../../hg38.2bit \ + hg38.referenceTarget.psl + + # the table name altSeqLiftOverPsl is recognized in hgc to allow display + # of the details of the alignments + hgLoadPsl hg38 -table=altSeqLiftOverPsl hg38.referenceTarget.psl + +######################################################################### +## alternate sequence alignments EXPERIMENT (DONE - 2014-01-17 - Hiram) + # the lastzAltSequences.2014-01-23 alignment was used for this instead + # of this procedure + mkdir /hive/data/genomes/hg38/bed/altAlignments + cd /hive/data/genomes/hg38/bed/altAlignments + + grep -v "^#" ../../genbank/GCA_000001405.15_GRCh38_top-level.acc2name \ + | awk '{printf "s/%s/%s/g;\n", $1, $3}' > accessionToUcsc.sed.txt + + find ../../genbank -type f | grep alt_scaffolds | grep "\.gff$" \ + | while read F +do + cat $F | sed -f accessionToUcsc.sed.txt \ + | gff3ToPsl ../../chrom.sizes stdin stdout +done > altAlignments.psl + | xargs cat | sed -f accessionToUcsc.sed.txt \ + | gff3ToPsl ../../chrom.sizes stdin altAlignments.psl + + time pslRecalcMatch altAlignments.psl ../../hg38.2bit ../../hg38.2bit \ + altRecalcMatch.psl + # real 0m51.122s + + # just to see what they look like in different formats: + pslToChain altRecalcMatch.psl altAlignments.chain + chainToAxt altAlignments.chain ../../hg38.2bit ../../hg38.2bit \ + altAlignments.axt + axtToMaf -score altAlignments.axt ../../chrom.sizes ../../chrom.sizes \ + altAlignments.maf + + mkdir mafSplits + mafSplit /dev/null mafSplits/ altAlignments.maf + # doesn't work: +# Can't find chrom in MAF component src: chr6_GL000250v2_alt + + mkdir splits psl + find ../../genbank -type f | grep alt_scaffolds | grep "\.gff$" \ + | while read F +do + chrAlt=`basename $F | sed -e 's/_.*//' | sed -f accessionToUcsc.sed.txt` + echo $chrAlt + cat $F | sed -f accessionToUcsc.sed.txt \ + | gff3ToPsl ../../chrom.sizes stdin splits/${chrAlt}.psl + pslRecalcMatch splits/${chrAlt}.psl ../../hg38.2bit ../../hg38.2bit \ + psl/${chrAlt}.psl +done + + mkdir swap + mkdir swap/psl swap/chain swap/axt swap/maf swap/anno + for F in psl/*.psl +do + B=`basename $F | sed -e 's/.psl//'` + echo $B + pslSwap $F stdout | pslRecalcMatch stdin ../../hg38.2bit ../../hg38.2bit \ + swap/psl/${B}.psl + pslToChain swap/psl/${B}.psl swap/chain/${B}.chain + chainToAxt swap/chain/${B}.chain ../../hg38.2bit ../../hg38.2bit \ + swap/axt/${B}.axt + axtToMaf -score swap/axt/${B}.axt ../../chrom.sizes ../../chrom.sizes stdout \ + | sed -e 's/^s chr\([0-9XYM][0-9]* \)/s ref38.chr\1/; s/^s chr\([0-9XYM][0-9]*_\)/s alt38.chr\1/;' > swap/maf/${B}.maf + mafAddIRows -nBeds=nBeds swap/maf/${B}.maf ../../hg38.2bit swap/anno/${B}.maf +done +# axtToMaf -score swap/axt/${B}.axt ../../chrom.sizes ../../chrom.sizes stdout \ +# | sed -e 's/^s chr/s hg38.chr/' > swap/maf/${B}.maf + + twoBitInfo -nBed ../../hg38.2bit ../../hg38.N.bed + ln -s ../../hg38.N.bed hg38.bed + ln -s ../../hg38.N.bed ref38.bed + ln -s ../../hg38.N.bed alt38.bed + echo hg38.bed > nBeds + echo ref38.bed >> nBeds + echo alt38.bed >> nBeds + ln -s ../../chrom.sizes hg38.len + ln -s ../../chrom.sizes ref38.len + ln -s ../../chrom.sizes alt38.len + echo hg38.len > sizes + echo ref38.len >> sizes + echo alt38.len >> sizes + + mkdir chain axt maf anno + for F in psl/*.psl +do + B=`basename $F | sed -e 's/.psl//'` + echo $B + pslToChain $F chain/${B}.chain + chainToAxt chain/${B}.chain ../../hg38.2bit ../../hg38.2bit axt/${B}.axt + axtToMaf -score axt/${B}.axt ../../chrom.sizes ../../chrom.sizes stdout \ + | sed -e 's/^s chr\([0-9XYM][0-9]* \)/s ref38.chr\1/; s/^s chr\([0-9XYM][0-9]*_\)/s alt38.chr\1/;' > maf/${B}.maf + mafAddIRows -nBeds=nBeds maf/${B}.maf ../../hg38.2bit anno/${B}.maf +done + +# axtToMaf -score axt/${B}.axt ../../chrom.sizes ../../chrom.sizes stdout \ +# | sed -e 's/^s chr/s hg38.chr/' > maf/${B}.maf + +############################################################################ +# Liftover Gencode V19 from hg19 (DONE braney 2014-02-14) + +mkdir /cluster/data/hg38/bed/liftOverGencodeV19 +cd /cluster/data/hg38/bed/liftOverGencodeV19 + +echo "show tables like 'wgEncodeGencode%19'" | hgsql hg19 | tail -n +2 > all.gencode.tables +echo " select tableName from trackDb where tableName like 'wgEncodeGencode_%V19';" | hgsql hg19 --skip-column-names > genePred.gencode.tables + +# load the non-genepred table as is. This isn't quite the right thing to do +# with exon support, but it's good enough for our purposes at the moment +join -v 1 *.gencode.tables | while read t; do echo "create table $t select * from hg19.$t" | hgsql hg38; echo $t; done + +for i in `cat genePredExt.gencode.tables`; +do + echo "select name,score,name2 from $i" | hgsql hg19 | sort > $i.name2Score.txt; + genePredToFakePsl hg19 $i $i.psl $i.cds; + pslMap -chainMapFile -swapMap $i.psl /gbdb/hg19/liftOver/hg19ToHg38.over.chain.gz stdout | pslCDnaFilter -uniqueMapped stdin stdout | sort -k 14,14 -k 16,16n | pslToBed -cds=$i.cds stdin stdout | bedToGenePred stdin stdout | sort | join /dev/stdin $i.name2Score.txt| tr ' ' '\t' | hgLoadGenePred -genePredExt hg38 $i stdin; + echo $i; +done + +for i in `cat genePred.gencode.tables`; +do + genePredToFakePsl hg19 $i $i.psl $i.cds; + pslMap -chainMapFile -swapMap $i.psl /gbdb/hg19/liftOver/hg19ToHg38.over.chain.gz stdout | pslCDnaFilter -uniqueMapped stdin stdout | sort -k 14,14 -k 16,16n | pslToBed -cds=$i.cds stdin stdout | bedToGenePred stdin stdout | tr ' ' '\t' | hgLoadGenePred hg38 $i stdin; + echo $i; +done + +##################################################################### +## tRNAs track ( 2014-02-18 braney DONE) +## this is a preliminary version for UCSC build. NOT FOR RELEASE! +ssh hgwdev +cd /hive/data/genomes/hg38/bed +mkdir tRNAs +cd tRNAs + +cp /hive/users/pchan/tRNAs/Eukaryota/hg38/hg38-tRNAs.bed . + +hgLoadBed -tab hg38 tRNAs hg38-tRNAs.bed -sqlTable=$HOME/kent/src/hg/lib/tRNAs.sql + +## tRNAs track (2015-10-04, Chris FINISHING BUILD FOR RELEASE) + cd /hive/data/genomes/hg38/bed/tRNAs + cat /hive/users/pchan/gtrnadb2/Eukaryota/hg38/hg38-tRNAs.bed | sed 's^</BLOCKQUOTE>^^g' | > hg38-tRNAs2.bed + hgsql hg38 -e 'drop table if exists tRNAs' + hgLoadBed -tab hg38 tRNAs hg38-tRNAs2.bed -sqlTable=$HOME/kent/src/hg/lib/tRNAs.sql + mkdir gif + cp -p /hive/users/pchan/gtrnadb2/Eukaryota/hg38/images/* gif + cd /hive/data/gbdb/hg38 + ln -s /hive/data/genomes/hg38/bed/tRNAs/gif RNA-img + cd /usr/local/apache/htdocs-ceisenhart/RNA-img + ln -s /gbdb/hg38/RNA-img hg38 + +############################################################################ +# EXONIPHY , lifted from hg19 (DONE - braney 2014-02-19) +# needed for ucscGenes building + # exoniphyHg19.gp is prepared as follows + mkdir /cluster/data/hg38/bed/exoniphy + cd /cluster/data/hg38/bed/exoniphy + hgsql hg19 -e "select * from exoniphy" -N | cut -f 2-16 > exoniphyHg19.gp + time nice -n +19 liftOver -genePred exoniphyHg19.gp \ + /cluster/data/hg19/bed/liftOver/hg19ToHg38.over.chain.gz \ + exoniphyHg38.gp unmapped + # real 0m2.015s + # user 0m1.894s + # sys 0m0.076s + + wc -l * + # 186601 exoniphyHg19.gp + # 186533 exoniphyHg38.gp + # 136 unmapped + # 373270 total + + cd /cluster/data/hg38/bed/exoniphy + nice -n +19 hgLoadGenePred -genePredExt hg38 exoniphy exoniphyHg38.gp + nice -n +19 featureBits hg38 exoniphy + # 28807039 bases of 3049335806 (0.945%) in intersection + nice -n +19 featureBits hg19 exoniphy + # 28661160 bases of 2897316137 (0.989%) in intersection + +######################################################################### +# LASTZ Rat Rn5 (DONE - 2014-02-27 - Hiram) + # establish a screen to control this job + screen -S hg38Rn5 + mkdir /hive/data/genomes/hg38/bed/lastzRn5.2014-02-27 + cd /hive/data/genomes/hg38/bed/lastzRn5.2014-02-27 + + # XXX don't forget to specify the BLASTZ binary: + cat << '_EOF_' > DEF +# human vs rat +BLASTZ=/cluster/bin/penn/lastz-distrib-1.03.52/bin/lastz + +# TARGET: Human Hg38 +SEQ1_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ1_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ1_CTGDIR=/hive/data/genomes/hg38/hg38.contigs.2bit +SEQ1_CTGLEN=/hive/data/genomes/hg38/hg38.contigs.chrom.sizes +SEQ1_LIFT=/hive/data/genomes/hg38/jkStuff/hg38.contigs.lift +SEQ1_CHUNK=20000000 +SEQ1_LAP=10000 + +# QUERY: Rat Rn5 +SEQ2_DIR=/hive/data/genomes/rn5/rn5.2bit +SEQ2_LEN=/hive/data/genomes/rn5/chrom.sizes +SEQ2_CHUNK=10000000 +SEQ2_LIMIT=100 +SEQ2_LAP=0 + +BASE=/hive/data/genomes/hg38/bed/lastzRn5.2014-02-27 +TMPDIR=/scratch/tmp +'_EOF_' + # << happy emacs + + time doBlastzChainNet.pl -verbose=2 \ + `pwd`/DEF \ + -syntenicNet \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -chainMinScore=3000 -chainLinearGap=medium > do.log 2>&1 + + # real 658m53.984s + cat fb.hg38.chainRn5Link.txt + # 938823407 bases of 3049335806 (30.788%) in intersection + + # running the swap + mkdir /hive/data/genomes/rn5/bed/blastz.hg38.swap + cd /hive/data/genomes/rn5/bed/blastz.hg38.swap + time nice -n +19 doBlastzChainNet.pl -verbose=2 \ + /hive/data/genomes/hg38/bed/lastzRn5.2014-02-27/DEF \ + -swap \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 + # real 66m53.095s + cat fb.rn5.chainHg38Link.txt + # 934256475 bases of 2572853723 (36.312%) in intersection + + # syntenic net for 14-way use 2014-04-02 - Hiram + cd /hive/data/genomes/rn5/bed/blastz.hg38.swap + time nice -n +19 doBlastzChainNet.pl -verbose=2 \ + /hive/data/genomes/hg38/bed/lastzRn5.2014-02-27/DEF \ + -continue=syntenicNet -syntenicNet -swap \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -chainMinScore=3000 -chainLinearGap=medium > synNet.log 2>&1 + # real 16m54.489s + +############################################################################## +# LASTZ Rat Rn4 (DONE - 2014-02-27 - Hiram) + # establish a screen to control this job + screen -S hg38Rn4 + mkdir /hive/data/genomes/hg38/bed/lastzRn4.2014-02-27 + cd /hive/data/genomes/hg38/bed/lastzRn4.2014-02-27 + + # XXX don't forget to specify the BLASTZ binary: + cat << '_EOF_' > DEF +# human vs rat +BLASTZ=/cluster/bin/penn/lastz-distrib-1.03.52/bin/lastz + +# TARGET: Human Hg38 +SEQ1_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ1_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ1_CTGDIR=/hive/data/genomes/hg38/hg38.contigs.2bit +SEQ1_CTGLEN=/hive/data/genomes/hg38/hg38.contigs.chrom.sizes +SEQ1_LIFT=/hive/data/genomes/hg38/jkStuff/hg38.contigs.lift +SEQ1_CHUNK=20000000 +SEQ1_LAP=10000 + +# QUERY: Rat Rn4 +SEQ2_DIR=/hive/data/genomes/rn4/rn4.2bit +SEQ2_LEN=/hive/data/genomes/rn4/chrom.sizes +SEQ2_CHUNK=10000000 +SEQ2_LIMIT=100 +SEQ2_LAP=0 + +BASE=/hive/data/genomes/hg38/bed/lastzRn4.2014-02-27 +TMPDIR=/scratch/tmp +'_EOF_' + # << happy emacs + + doBlastzChainNet.pl -verbose=2 \ + `pwd`/DEF \ + -syntenicNet \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -chainMinScore=3000 -chainLinearGap=medium > do.log 2>&1 + # real 658m53.984s + + cat fb.hg38.chainRn4Link.txt + # 913992768 bases of 3049335806 (29.974%) in intersection + + # running the swap + mkdir /hive/data/genomes/rn4/bed/blastz.hg38.swap + cd /hive/data/genomes/rn4/bed/blastz.hg38.swap + time nice -n +19 doBlastzChainNet.pl -verbose=2 \ + /hive/data/genomes/hg38/bed/lastzRn4.2014-02-27/DEF \ + -swap \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 & + # real 73m5.666s + + cat fb.rn4.chainHg38Link.txt + # 889613774 bases of 2571531505 (34.595%) in intersection + +############################################################################## +# GENEID GENE PREDICTIONS (DONE - 2014-03-07 - Hiram) + ssh hgwdev + mkdir /hive/data/genomes/hg38/bed/geneid + cd /hive/data/genomes/hg38/bed/geneid + mkdir download + cd download + for C in `cut -f1 ../../../chrom.sizes` + do + echo $C + wget --timestamping \ +http://genome.crg.es/genepredictions/H.sapiens/hg38/geneid_v1.4/${C}.gtf3 + wget --timestamping \ +http://genome.crg.es/genepredictions/H.sapiens/hg38/geneid_v1.4/${C}.prot + done + + cd .. + cat download/*.gtf | ldHgGene -gtf -genePredExt hg38 geneid stdin + # Read 33428 transcripts in 277332 lines in 1 files + # 33428 groups 92 seqs 1 sources 3 feature types + # 33428 gene predictions + +############################################################################ +# GENEREVIEWS TRACK (DONE 2014-05-17 - Chin) +# This track depends on some tasks completed for hg19, specifically: +# +# $HOME/kent/src/hg/lib/geneReviewsGrshortNBKid.sql +# $HOME/kent/src/hg/lib/geneReviewsGrshortTitleNBKid.sql +# $HOME/kent/src/hg/lib/geneReviewsDetail.sql +# $HOME/kent/src/hg/makeDb/trackDb/human/geneReviews.html +# +# Unlike hg19, this hg38 tracks is generated by the automatic geneReviews +# scripts in +# /hive/data/outside/otto/geneReviews, specifically buildGeneReviews.sh. +# Current data are fetched weekly from NCBI +# ftp://ftp.ncbi.nlm.nih.gov/pub/GeneReviews/ +# to /hive/data/outside/otto/geneReviews/${DATE}. + +########################################################################### +# Chimp Lastz run (DONE - 2014-05-27 - Hiram) + screen -S hg38PanTro4 # use a screen to manage this longish running job + mkdir /hive/data/genomes/hg38/bed/lastzPanTro4.2014-05-27 + cd /hive/data/genomes/hg38/bed/lastzPanTro4.2014-05-27 + + # always set the BLASTZ program so we know what version was used + cat << '_EOF_' > DEF +# human vs chimp +BLASTZ=/cluster/bin/penn/lastz-distrib-1.03.52/bin/lastz +BLASTZ_O=600 +BLASTZ_E=150 +# maximum M allowed with lastz is only 254 +BLASTZ_M=254 + +BLASTZ_T=2 +BLASTZ_Y=15000 +BLASTZ_K=4500 +BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q +# A C G T +# 90 -330 -236 -356 +# -330 100 -318 -236 +# -236 -318 100 -330 +# -356 -236 -330 90 + +# TARGET: Human Hg38 +SEQ1_DIR=/scratch/data/hg38/hg38.2bit +SEQ1_LEN=/scratch/data/hg38/chrom.sizes +SEQ1_CHUNK=10000000 +SEQ1_LAP=10000 +SEQ1_IN_CONTIGS=0 + +# QUERY: Chimp PanTro4 +SEQ2_DIR=/hive/data/genomes/panTro4/panTro4.2bit +SEQ2_LEN=/hive/data/genomes/panTro4/chrom.sizes +SEQ2_CHUNK=10000000 +SEQ2_LAP=0 +SEQ2_LIMIT=200 +SEQ2_IN_CONTIGS=0 + +BASE=/hive/data/genomes/hg38/bed/lastzPanTro4.2014-05-27 +TMPDIR=/dev/shm +'_EOF_' + # << emacs + + time (doBlastzChainNet.pl `pwd`/DEF -verbose=2 \ + -chainMinScore=5000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -syntenicNet) > do.log 2>&1 + # real 154m12.215s + cat fb.hg38.chainPanTro4Link.txt + # 2839294579 bases of 3049335806 (93.112%) in intersection + + # filter with doRecipBest.pl + time (doRecipBest.pl -workhorse=hgwdev -buildDir=`pwd` \ + hg38 panTro4) > rbest.log 2>&1 + # real 57m55.320s + + # running the swap + mkdir /hive/data/genomes/panTro4/bed/blastz.hg38.swap + cd /hive/data/genomes/panTro4/bed/blastz.hg38.swap + time (doBlastzChainNet.pl -verbose=2 \ + -swap /hive/data/genomes/hg38/bed/lastzPanTro4.2014-05-27/DEF \ + -chainMinScore=5000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -syntenicNet) > swap.log 2>&1 + cat fb.panTro4.chainHg38Link.txt + # 2776497530 bases of 2902338967 (95.664%) in intersection + # real 98m23.729s + + time (doRecipBest.pl -workhorse=hgwdev -buildDir=`pwd` \ + panTro4 hg38) > rbest.log 2>&1 + # real 64m33.812s + +############################################################################# +# Opossum Lastz run (DONE - 2014-05-27 - Hiram) + screen -S hg38MonDom5 # use a screen to manage this longish running job + mkdir /hive/data/genomes/hg38/bed/lastzMonDom5.2014-05-27 + cd /hive/data/genomes/hg38/bed/lastzMonDom5.2014-05-27 + + # always set the BLASTZ program so we know what version was used + cat << '_EOF_' > DEF +# human vs chimp +BLASTZ=/cluster/bin/penn/lastz-distrib-1.03.52/bin/lastz +BLASTZ_M=50 + +BLASTZ_Y=3400 +BLASTZ_L=6000 +BLASTZ_K=2200 +BLASTZ_Q=/scratch/data/blastz/HoxD55.q +# A C G T +# 91 -90 -25 -100 +# -90 100 -100 -25 +# -25 -100 100 -90 +# -100 -25 -90 91 + +# TARGET: Human Hg38 +SEQ1_DIR=/scratch/data/hg38/hg38.2bit +SEQ1_LEN=/scratch/data/hg38/chrom.sizes +SEQ1_CHUNK=10000000 +SEQ1_LAP=10000 +SEQ1_LIMIT=5 + +# QUERY: Opossum MonDom5 +SEQ2_DIR=/scratch/data/monDom5/monDom5.2bit +SEQ2_LEN=/hive/data/genomes/monDom5/chrom.sizes +SEQ2_CHUNK=10000000 +SEQ2_LAP=0 + +BASE=/hive/data/genomes/hg38/bed/lastzMonDom5.2014-05-27 +TMPDIR=/dev/shm +'_EOF_' + # << emacs + + time (doBlastzChainNet.pl `pwd`/DEF -verbose=2 \ + -chainMinScore=5000 -chainLinearGap=loose \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -syntenicNet) > do.log 2>&1 + # real 670m13.280s + # one failed chain run for hg19, finished manually on hgwdev, then: + time (doBlastzChainNet.pl `pwd`/DEF -verbose=2 \ + -continue=chainMerge -chainMinScore=5000 -chainLinearGap=loose \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -syntenicNet) > chainMerge.log 2>&1 + # real 164m28.822s + + cat fb.hg38.chainMonDom5Link.txt + # 438195373 bases of 3049335806 (14.370%) in intersection + + # filter with doRecipBest.pl + time (/cluster/bin/scripts/doRecipBest.pl -buildDir=`pwd` \ + -dbHost=hgwdev -workhorse=hgwdev hg38 monDom5) > rbest.log 2>&1 + # real 130m22.825s + + # running the swap + mkdir /hive/data/genomes/monDom5/bed/blastz.hg38.swap + cd /hive/data/genomes/monDom5/bed/blastz.hg38.swap + time (doBlastzChainNet.pl -verbose=2 \ + /hive/data/genomes/hg38/bed/lastzMonDom5.2014-05-27/DEF \ + -swap -chainMinScore=5000 -chainLinearGap=loose \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -syntenicNet) > swap.log 2>&1 + # real 102m41.443s + + cat fb.monDom5.chainHg38Link.txt + # 420069915 bases of 3501660299 (11.996%) in intersection + time (/cluster/bin/scripts/doRecipBest.pl -buildDir=`pwd` \ + -dbHost=hgwdev -workhorse=hgwdev monDom5 hg38) > rbest.log 2>&1 + # real 90m56.189s + +_EOF_ +############################################################################# +# LOCUS REFERENCE GENOMIC (LRG) REGIONS AND TRANSCRIPTS (DONE 10/25/19 angie) +# Redmine #13359, #24285 -- otto-mate To Do #17877 +# previously done 7/7/14, 9/9/16, 5/30/18 +# THIS IS NOW AN OTTO JOB !! + set today = `date +%Y_%m_%d` + mkdir -p /hive/data/genomes/hg38/bed/lrg/$today + cd /hive/data/genomes/hg38/bed/lrg/$today + wget ftp://ftp.ebi.ac.uk/pub/databases/lrgex/LRG_public_xml_files.zip + unzip LRG_public_xml_files.zip + + # Run script to convert LRG*.xml files to BED+ for regions and genePredExt+fa for transcripts: + # parseLrgXml.pl updated 2020-09-16 to add four new fields to the gp output + # the four extra fields are identifiers for: + # NCBI transcript, Ensembl transcript, NCBI protein, Ensembl protein + + ~/kent/src/hg/utils/automation/parseLrgXml.pl GRCh38 + genePredCheck lrgTranscriptsUnmapped.gp +#Error: lrgTranscriptsUnmapped.gp:765: LRG_7t1 no exonFrame on CDS exon 46 +#checked: 1029 failed: 1 + # If there are complaints e.g. about exonFrame, look for inconsistencies in the + # affected transcript's coding_region/coordinates vs. exon/intron info in xml. + # Contact Variation team leader Fiona Cunningham @EBI to resolve in the background + # (missing exonFrame info doesn't affect our track representation because we end up using + # psl). We agreed to disagree about exon 46 of LRG_7t1 because that last coding exon + # portion is only the stop codon. + + # No longer necessary to filter out alt and fix patches since they have been added to hg38. + + # and we need the transcript plus gene name later: + cut -f1,12 lrgTranscriptsUnmapped.gp | sort > transcript.gene.name.txt + + # five extra columns have been added to the genePred (2020-10-05 - Hiram) + # extract them so they can be added to the psl: + awk -F$'\t' '{printf "%s\t%s\t%s\t%s\t%s\t%s %s %s %s\n", $1,$16,$17,$18,$19, $16,$18,$17,$19}' lrgTranscriptsUnmapped.gp | sort \ + | join -t$'\t' - transcript.gene.name.txt \ + | awk -F$'\t' '{printf "%s\t%s\t%s\t%s\t%s\t%s\t%s %s\n", $1,$2,$3,$4,$5,$7,$6,$7}' > lrgTransExtraFields.tsv + + # the five extra fields are identifiers for: + # NCBI transcript, Ensembl transcript, NCBI protein, Ensembl protein, + # Gene name + + # Load LRG regions: + #bedToBigBed lrg.bed /hive/data/genomes/hg38/chrom.sizes lrg.bb \ + #-tab -type=bed12+ -as=$HOME/kent/src/hg/lib/lrg.as -extraIndex=name + # after ML #29689, added ncbiAcc field, Max, July 1, 2022 + # changed to: + bedToBigBed lrg.bed /hive/data/genomes/hg38/chrom.sizes lrg.bb \ + -tab -type=bed12+ -as=$HOME/kent/src/hg/lib/lrg.as -extraIndex=name,ncbiAcc + ln -sf `pwd`/lrg.bb /gbdb/hg38/bbi/lrg.bb + hgBbiDbLink hg38 lrg /gbdb/hg38/bbi/lrg.bb + + # Map LRG fixed_annotation transcripts from LRG coords to hg38 coords (HT MarkD): + lrgToPsl lrg.bed /hive/data/genomes/hg38/chrom.sizes lrg.psl + pslCheck lrg.psl +#checked: 919 failed: 0 errors: 0 + awk '{print $10 "\t" $11;}' lrg.psl > lrg.sizes + genePredToFakePsl -chromSize=lrg.sizes placeholder \ + lrgTranscriptsUnmapped.gp lrgTranscriptsFakePsl.psl lrgTranscripts.cds + pslMap lrgTranscriptsFakePsl.psl lrg.psl lrgTranscriptsHg38.psl + mrnaToGene -genePredExt -cdsFile=lrgTranscripts.cds -keepInvalid \ + lrgTranscriptsHg38.psl lrgTranscriptsHg38NoName2.gp +#Warning: no CDS for LRG_163t1 +#Warning: no CDS for LRG_347t1 + # It's OK if mrnaToGene complains about "no CDS" for a non-coding tx (RefSeq accession NR_*). + grep -l NR_ LRG_163.xml LRG_347.xml +#LRG_163.xml +#LRG_347.xml + + cat lrgCdna.tab | sed -e 's/^/>/;' | tr '\t' '\n' > lrgCdna.fa + # construct bigPsl with five extra fields + pslToBigPsl -fa=lrgCdna.fa -cds=lrgTranscripts.cds \ + lrgTranscriptsHg38.psl bigPsl.txt + + # add the five extra identifiers to the bigPsl file: + join -t$'\t' -1 4 \ + -o 1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8,1.9,1.10,1.11,1.12,1.13,1.14,1.15\ +,1.16,1.17,1.18,1.19,1.20,1.21,1.22,1.23,1.24,1.25,2.2,2.3,2.4,2.5,2.6,2.7 \ + <(sort -k4 bigPsl.txt) lrgTransExtraFields.tsv \ + | sort -k1,1 -k2,2n > lrgExtraTranscriptsHg38.bigPsl.bed + + bedToBigBed -as=bigPsl+6.as -type=bed12+19 -tab \ + lrgExtraTranscriptsHg38.bigPsl.bed ../../../chrom.sizes lrgBigPsl.bb + bigBedInfo lrgBigPsl.bb + rm -f /gbdb/hg38/bbi/lrgBigPsl.bb + ln -sf `pwd`/lrgBigPsl.bb /gbdb/hg38/bbi + hgBbiDbLink hg38 lrgBigPsl /gbdb/hg38/bbi/lrgBigPsl.bb + + + # Load PSL, CDS and sequences. + hgLoadPsl hg38 -table=lrgTranscriptAli lrgTranscriptsHg38.psl + hgLoadSqlTab hg38 lrgCds ~/kent/src/hg/lib/cdsSpec.sql lrgTranscripts.cds + hgPepPred hg38 tab lrgCdna lrgCdna.tab + hgPepPred hg38 tab lrgPep lrgPep.tab + + +############################################################################# +## 7-Way Multiz (DONE - 2014-06-02 - Hiram) + ssh hgwdev + mkdir /hive/data/genomes/hg38/bed/multiz7way + cd /hive/data/genomes/hg38/bed/multiz7way + + # from the 63-way in the source tree, select out the 7 used here: + /cluster/bin/phast/tree_doctor \ + --prune-all-but hg19,panTro4,rheMac3,mm10,rn5,canFam3,monDom5 \ + /cluster/home/hiram/kent/src/hg/utils/phyloTrees/130way.nh \ + | sed -e 's/hg19/hg38/' > hg38.7way.nh + + # what that looks like: + ~/kent/src/hg/utils/phyloTrees/asciiTree.pl hg38.7way.nh +# (((((hg38:0.006550, +# panTro4:0.006840):0.027424, +# rheMac3:0.037601):0.109934, +# (mm10:0.084509, +# rn5:0.091589):0.271974):0.020593, +# canFam3:0.165928):0.258392, +# monDom5:0.340786); + + # extract species list from that .nh file + sed 's/[a-z][a-z]*_//g; s/:[0-9\.][0-9\.]*//g; s/;//; /^ *$/d' \ + hg38.7way.nh | xargs echo | sed 's/ //g; s/,/ /g' \ + | sed 's/[()]//g; s/,/ /g' | tr '[ ]' '[\n]' > species.list.txt + + # construct db to name translation list: + cat species.list.txt | while read DB +do +hgsql -N -e "select name,organism from dbDb where name=\"${DB}\";" hgcentraltest +done | sed -e "s/\t/->/; s/ /_/g;" | sed -e 's/$/;/' | sed -e 's/\./_/g' \ + > db.to.name.txt + + # construct a common name .nh file: + /cluster/bin/phast/tree_doctor --rename \ + "`cat db.to.name.txt`" hg38.7way.nh | sed -e 's/00*)/)/g; s/00*,/,/g' \ + | $HOME/kent/src/hg/utils/phyloTrees/asciiTree.pl /dev/stdin \ + > hg38.7way.commonNames.nh + + $HOME/kent/src/hg/utils/phyloTrees/asciiTree.pl hg38.7way.nh > t.nh + $HOME/kent/src/hg/utils/phyloTrees/scientificNames.sh t.nh \ + | $HOME/kent/src/hg/utils/phyloTrees/asciiTree.pl /dev/stdin \ + > hg38.7way.scientificNames.nh + rm -f t.nh + cat hg38.7way.scientificNames.nh +# (((((Homo_sapiens:0.00655, +# Pan_troglodytes:0.00684):0.027424, +# Macaca_mulatta:0.037601):0.109934, +# (Mus_musculus:0.084509, +# Rattus_norvegicus:0.091589):0.271974):0.020593, +# Canis_lupus_familiaris:0.165928):0.258392, +# Monodelphis_domestica:0.340786); + + ~/kent/src/hg/utils/phyloTrees/asciiTree.pl hg38.7way.commonNames.nh +# (((((Human:0.00655, +# Chimp:0.00684):0.027424, +# Rhesus:0.037601):0.109934, +# (Mouse:0.084509, +# Rat:0.091589):0.271974):0.020593, +# Dog:0.165928):0.258392, +# Opossum:0.340786); + + # Use this specification in the phyloGif tool: + # http://genome.ucsc.edu/cgi-bin/phyloGif + # to obtain a png image for src/hg/htdocs/images/phylo/hg38_7way.png + + /cluster/bin/phast/all_dists hg38.7way.nh | grep hg38 \ + | sed -e "s/hg38.//" | sort -k2n > 7way.distances.txt + # Use this output to create the table below + head 7way.distances.txt +# taeGut1 0.075718 +# melUnd1 0.220312 +# galGal4 0.507021 +# melGal1 0.509140 +# hg19 1.175433 +# mm10 1.383071 + + cat << '_EOF_' > sizeStats.pl +#!/usr/bin/env perl + +use strict; +use warnings; + +open (FH, "<7way.distances.txt") or + die "can not read 7way.distances.txt"; + +my $count = 0; +while (my $line = <FH>) { + chomp $line; + my ($D, $dist) = split('\s+', $line); + my $chain = "chain" . ucfirst($D); + my $B="/hive/data/genomes/hg38/bed/lastz.$D/fb.hg38." . + $chain . "Link.txt"; + my $chainLinkMeasure = + `awk '{print \$5}' ${B} 2> /dev/null | sed -e "s/(//; s/)//"`; + chomp $chainLinkMeasure; + $chainLinkMeasure = 0.0 if (length($chainLinkMeasure) < 1); + $chainLinkMeasure =~ s/\%//; + my $swapFile="/hive/data/genomes/${D}/bed/lastz.hg38/fb.${D}.chainHg38Link.txt"; + my $swapMeasure = "N/A"; + if ( -s $swapFile ) { + $swapMeasure = + `awk '{print \$5}' ${swapFile} 2> /dev/null | sed -e "s/(//; s/)//"`; + chomp $swapMeasure; + $swapMeasure = 0.0 if (length($swapMeasure) < 1); + $swapMeasure =~ s/\%//; + } + my $orgName= + `hgsql -N -e "select organism from dbDb where name='$D';" hgcentraltest`; + chomp $orgName; + if (length($orgName) < 1) { + $orgName="N/A"; + } + ++$count; + printf "# %02d %.4f (%% %06.3f) (%% %06.3f) - %s %s\n", $count, $dist, + $chainLinkMeasure, $swapMeasure, $orgName, $D; +} +close (FH); +'_EOF_' + # << happy emacs + chmod +x ./sizeStats.pl + ./sizeStats.pl +# + +# If you can fill in all the numbers in this table, you are ready for +# the multiple alignment procedure + +# featureBits chainLink measures +# chainLink +# N distance on hg38 on other other species +# 01 0.0134 (% 93.112) (% 95.664) - Chimp panTro4 +# 02 0.0716 (% 79.729) (% 86.715) - Rhesus rheMac3 +# 03 0.3304 (% 49.978) (% 60.083) - Dog canFam3 +# 04 0.5004 (% 31.629) (% 35.323) - Mouse mm10 +# 05 0.5075 (% 30.788) (% 36.312) - Rat rn5 +# 06 0.7637 (% 14.370) (% 11.996) - Opossum monDom5 + +# None of this concern for distances matters in building the first step, the +# maf files. + + # create species list and stripped down tree for autoMZ + sed 's/[a-z][a-z]*_//g; s/:[0-9\.][0-9\.]*//g; s/;//; /^ *$/d' \ + hg38.7way.nh | xargs echo | sed 's/ //g; s/,/ /g' > tree.nh + + sed 's/[()]//g; s/,/ /g' tree.nh > species.list + # hg38 panTro4 rheMac3 mm10 rn5 canFam3 monDom5 + + # bash shell syntax here ... + cd /hive/data/genomes/hg38/bed/multiz7way + export H=/hive/data/genomes/hg38/bed + mkdir mafLinks + # want syntenic net for: panTro4 rheMac3 mm10 rn5 canFam3 + # and unfiltered maf net for: monDom5 + for G in panTro4 rheMac3 mm10 rn5 canFam3 + do + mkdir mafLinks/$G + echo ln -s ${H}/lastz.$G/axtChain/hg38.${G}.synNet.maf.gz ./mafLinks/$G + ln -s ${H}/lastz.$G/axtChain/hg38.${G}.synNet.maf.gz ./mafLinks/$G + done + + mkdir mafLinks/monDom5 + echo ln -s ${H}/lastz.monDom5/mafNet/*.maf.gz ./mafLinks/monDom5 + ln -s ${H}/lastz.monDom5/mafNet/*.maf.gz ./mafLinks/monDom5 + # verify the symLinks are good: + ls -ogrtL mafLinks/*/* +#-rw-rw-r-- 1 709500062 Jan 25 12:15 mafLinks/mm10/hg38.mm10.synNet.maf.gz +#-rw-rw-r-- 1 1089643630 Jan 27 19:15 mafLinks/canFam3/hg38.canFam3.synNet.maf.gz +#-rw-rw-r-- 1 1277455681 Jan 28 21:52 mafLinks/rheMac3/hg38.rheMac3.synNet.maf.gz +#-rw-rw-r-- 1 687500679 Mar 1 12:27 mafLinks/rn5/hg38.rn5.synNet.maf.gz +#-rw-rw-r-- 1 1463969868 May 27 11:41 mafLinks/panTro4/hg38.panTro4.synNet.maf.gz +#-rw-rw-r-- 1 323347908 May 29 12:38 mafLinks/monDom5/hg38.monDom5.net.maf.gz + + # split the maf files into a set of hashed named files + # this hash named split keeps the same chr/contig names in the same + # named hash file. + mkdir /hive/data/genomes/hg38/bed/multiz7way/mafSplit + cd /hive/data/genomes/hg38/bed/multiz7way/mafSplit + for D in `sed -e "s/hg38 //" ../species.list` +do + echo "${D}" + mkdir $D + cd $D + echo "mafSplit -byTarget -useHashedName=8 /dev/null . ../../mafLinks/${D}/*.maf.gz" + mafSplit -byTarget -useHashedName=8 /dev/null . \ + ../../mafLinks/${D}/*.maf.gz + cd .. +done + + # construct a list of all possible maf file names. + # they do not all exist in each of the species directories + find . -type f | wc -l + # 641 + find . -type f | grep ".maf$" | xargs -L 1 basename | sort -u > maf.list + wc -l maf.list + # 118 maf.list + + mkdir /hive/data/genomes/hg38/bed/multiz7way/splitRun + cd /hive/data/genomes/hg38/bed/multiz7way/splitRun + mkdir maf run + cd run + mkdir penn + cp -p /cluster/bin/penn/multiz.2009-01-21_patched/multiz penn + cp -p /cluster/bin/penn/multiz.2009-01-21_patched/maf_project penn + cp -p /cluster/bin/penn/multiz.2009-01-21_patched/autoMZ penn + + # set the db and pairs directories here + cat > autoMultiz.csh << '_EOF_' +#!/bin/csh -ef +set db = hg38 +set c = $1 +set result = $2 +set run = `/bin/pwd` +set tmp = /dev/shm/$db/multiz.$c +set pairs = /hive/data/genomes/hg38/bed/multiz7way/mafSplit +/bin/rm -fr $tmp +/bin/mkdir -p $tmp +/bin/cp -p ../../tree.nh ../../species.list $tmp +pushd $tmp > /dev/null +foreach s (`/bin/sed -e "s/$db //" species.list`) + set in = $pairs/$s/$c + set out = $db.$s.sing.maf + if (-e $in.gz) then + /bin/zcat $in.gz > $out + if (! -s $out) then + echo "##maf version=1 scoring=autoMZ" > $out + endif + else if (-e $in) then + /bin/ln -s $in $out + else + echo "##maf version=1 scoring=autoMZ" > $out + endif +end +set path = ($run/penn $path); rehash +$run/penn/autoMZ + T=$tmp E=$db "`cat tree.nh`" $db.*.sing.maf $c \ + > /dev/null +popd > /dev/null +/bin/rm -f $result +/bin/cp -p $tmp/$c $result +/bin/rm -fr $tmp +'_EOF_' +# << happy emacs + chmod +x autoMultiz.csh + + cat << '_EOF_' > template +#LOOP +./autoMultiz.csh $(file1) {check out line+ /hive/data/genomes/hg38/bed/multiz7way/splitRun/maf/$(root1).maf} +#ENDLOOP +'_EOF_' +# << happy emacs + + ln -s ../../mafSplit/maf.list maf.list + ssh ku + cd /hive/data/genomes/hg38/bed/multiz7way/splitRun/run + gensub2 maf.list single template stdout > jobList + para -ram=8g create jobList +# Completed: 118 of 118 jobs +# CPU time in finished jobs: 118241s 1970.69m 32.84h 1.37d 0.004 y +# IO & Wait Time: 682s 11.36m 0.19h 0.01d 0.000 y +# Average job time: 1008s 16.80m 0.28h 0.01d +# Longest finished job: 10068s 167.80m 2.80h 0.12d +# Submission to last job: 10076s 167.93m 2.80h 0.12d + + # combine into one file (the 1>&2 redirect sends the echo to stderr) + cd /hive/data/genomes/hg38/bed/multiz7way + head -1 splitRun/maf/017.maf > multiz7way.maf + for F in splitRun/maf/*.maf +do + echo "${F}" 1>&2 + egrep -v "^#" ${F} +done >> multiz7way.maf + tail -1 splitRun/maf/017.maf >> multiz7way.maf +# -rw-rw-r-- 1 15635828403 Jun 3 11:49 multiz7way.maf + + # Load into database + ssh hgwdev + cd /hive/data/genomes/hg38/bed/multiz7way + mkdir /gbdb/hg38/multiz7way + ln -s `pwd`/multiz7way.maf /gbdb/hg38/multiz7way + cd /dev/shm + time nice -n +17 hgLoadMaf hg38 multiz7way + # Loaded 10270624 mafs in 1 files from /gbdb/hg38/multiz7way + # real 3m51.265s + + time nice -n +17 hgLoadMafSummary -verbose=2 -minSize=30000 \ + -mergeGap=1500 -maxSize=200000 hg38 multiz7waySummary \ + /gbdb/hg38/multiz7way/multiz7way.maf + # Created 1260918 summary blocks from 35384988 components + # and 10270624 mafs from /gbdb/hg38/multiz7way/multiz7way.maf + # real 5m39.388s + + + wc -l multiz7way*.tab + # 10270624 multiz7way.tab + # 1260918 multiz7waySummary.tab + # 11531542 total + + rm multiz7way*.tab + +############################################################################## +# GAP ANNOTATE MULTIZ7WAY MAF AND LOAD TABLES (DONE - 2014-06-03 - Hiram) + # mafAddIRows has to be run on single chromosome maf files, it does not + # function correctly when more than one reference sequence + # are in a single file. Need to split of the maf file into individual + # maf files + mkdir -p /hive/data/genomes/hg38/bed/multiz7way/anno/mafSplit + cd /hive/data/genomes/hg38/bed/multiz7way/anno/mafSplit + + time mafSplit -outDirDepth=1 -byTarget -useFullSequenceName \ + /dev/null . ../../multiz7way.maf + # real 4m8.617s + + find . -type f | wc -l + # 353 + + # check for N.bed files everywhere: + cd /hive/data/genomes/hg38/bed/multiz7way/anno + for DB in `cat ../species.list` +do + if [ ! -s /hive/data/genomes/${DB}/${DB}.N.bed ]; then + echo "MISS: ${DB}" +# cd /hive/data/genomes/${DB} +# twoBitInfo -nBed ${DB}.2bit ${DB}.N.bed + else + echo " OK: ${DB}" + fi +done + + cd /hive/data/genomes/hg38/bed/multiz7way/anno + for DB in `cat ../species.list` +do + echo "${DB} " + ln -s /hive/data/genomes/${DB}/${DB}.N.bed ${DB}.bed + echo ${DB}.bed >> nBeds + ln -s /hive/data/genomes/${DB}/chrom.sizes ${DB}.len + echo ${DB}.len >> sizes +done + # make sure they all are successful symLinks: + ls -ogrtL + + screen -S hg38 # use a screen to control this longish job + ssh ku + cd /hive/data/genomes/hg38/bed/multiz7way/anno + mkdir result + for D in `ls mafSplit` +do + echo mkdir result/${D} + mkdir result/${D} +done + cat << '_EOF_' > template +#LOOP +mafAddIRows -nBeds=nBeds mafSplit/$(path1) /hive/data/genomes/hg38/hg38.2bit {check out exists+ result/$(path1)} +#ENDLOOP +'_EOF_' + # << happy emacs + + find ./mafSplit -type f | sed -e 's#^./mafSplit/##' > maf.list + gensub2 maf.list single template jobList + # limit jobs on a node with the ram=32g requirement because they go fast + para -ram=32g create jobList + para try ... check ... push ... +# Completed: 353 of 353 jobs +# CPU time in finished jobs: 530s 8.83m 0.15h 0.01d 0.000 y +# IO & Wait Time: 1057s 17.62m 0.29h 0.01d 0.000 y +# Average job time: 4s 0.07m 0.00h 0.00d +# Longest finished job: 63s 1.05m 0.02h 0.00d +# Submission to last job: 220s 3.67m 0.06h 0.00d + + # verify all result files have some content, look for 0 size files: + find ./result -type f -size 0 + # should see none + # or in this manner: + find ./result -type f | xargs ls -og | sort -k3nr | tail + + # combine into one file (the 1>&2 redirect sends the echo to stderr) + head -q -n 1 result/0/chr8.maf > hg38.7way.maf + find ./result -type f | while read F +do + echo "${F}" 1>&2 + grep -h -v "^#" ${F} +done >> hg38.7way.maf + + # these maf files do not have the end marker, this does nothing: + # tail -q -n 1 result/0/chr8.maf >> hg38.7way.maf + # How about an official end marker: + echo "##eof maf" >> hg38.7way.maf + ls -og +# -rw-rw-r-- 1 17795297196 Jun 3 14:01 hg38.7way.maf + + du -hsc hg38.7way.maf + # 17G hg38.7way.maf + + # construct symlinks to get the individual maf files into gbdb: + rm /gbdb/hg38/multiz7way/multiz7way.maf # remove previous results + ln -s `pwd`/hg38.7way.maf /gbdb/hg38/multiz7way/multiz7way.maf + + # Load into database + cd /dev/shm + time nice -n +19 hgLoadMaf -pathPrefix=/gbdb/hg38/multiz7way \ + hg38 multiz7way + # Loaded 10359242 mafs in 1 files from /gbdb/hg38/multiz7way + # real 4m21.862s + + time hgLoadMafSummary -verbose=2 -minSize=30000 \ + -mergeGap=1500 -maxSize=200000 hg38 multiz7waySummary \ + /gbdb/hg38/multiz7way/multiz7way.maf +# Created 1260918 summary blocks from 35384988 components +# and 10359242 mafs from /gbdb/hg38/multiz7way/multiz7way.maf +# real 6m6.583s + +# -rw-rw-r-- 1 530538267 Jun 3 14:05 multiz7way.tab +# -rw-rw-r-- 1 60616616 Jun 3 14:15 multiz7waySummary.tab + + rm multiz7way*.tab + +###################################################################### +# MULTIZ7WAY MAF FRAMES (DONE - 2014-06-03 - Hiram) + ssh hgwdev + mkdir /hive/data/genomes/hg38/bed/multiz7way/frames + cd /hive/data/genomes/hg38/bed/multiz7way/frames +# survey all the genomes to find out what kinds of gene tracks they have + cat << '_EOF_' > showGenes.csh +#!/bin/csh -fe +foreach db (`cat ../species.list`) + echo -n "${db}: " + set tables = `hgsql $db -N -e "show tables like '%Gene%'"` + foreach table ($tables) + if ($table == "ensGene" || $table == "refGene" || \ + $table == "mgcGenes" || $table == "knownGene" || \ + $table == "xenoRefGene" ) then + set count = `hgsql $db -N -e "select count(*) from $table"` + echo -n "${table}: ${count}, " + endif + end + set orgName = `hgsql hgcentraltest -N -e \ + "select scientificName from dbDb where name='$db'"` + set orgId = `hgsql hg19 -N -e \ + "select id from organism where name='$orgName'"` + if ($orgId == "") then + echo "Mrnas: 0" + else + set count = `hgsql hg19 -N -e "select count(*) from gbCdnaInfo where organism=$orgId"` + echo "Mrnas: ${count}" + endif +end +'_EOF_' + # << happy emacs + chmod +x ./showGenes.csh + time ./showGenes.csh +# hg38: knownGene: 104178, mgcGenes: 34081, refGene: 54852, xenoRefGene: 172740, Mrnas: 10723716 +# panTro4: ensGene: 29160, refGene: 2622, xenoRefGene: 280516, Mrnas: 11163 +# rheMac3: refGene: 6369, xenoRefGene: 275096, Mrnas: 443642 +# mm10: ensGene: 94647, knownGene: 61642, mgcGenes: 26768, refGene: 33765, xenoRefGene: 161178, Mrnas: 5224613 +# rn5: ensGene: 29188, mgcGenes: 6924, refGene: 18567, xenoRefGene: 175416, Mrnas: 1247500 +# canFam3: ensGene: 29884, refGene: 1582, xenoRefGene: 253196, Mrnas: 387195 +# monDom5: ensGene: 24882, refGene: 492, xenoRefGene: 248251, Mrnas: 2461 + + # from that summary, use these gene sets: + # refGene - rheMac3 + # ensGene - panTro4 rn5 canFam3 monDom5 + # knownGene - hg38 mm10 + + mkdir genes + # 1. knownGene: hg38 mm10 + for DB in hg38 mm10 +do + hgsql -N -e "select name,chrom,strand,txStart,txEnd,cdsStart,cdsEnd,exonCount,exonStarts,exonEnds from knownGene" ${DB} \ + | genePredSingleCover stdin stdout | gzip -2c \ + > genes/${DB}.gp.gz +done + # 2. ensGene: + for DB in panTro4 rn5 canFam3 monDom5 +do +hgsql -N -e "select name,chrom,strand,txStart,txEnd,cdsStart,cdsEnd,exonCount,exonStarts,exonEnds from ensGene" ${DB} \ + | genePredSingleCover stdin stdout | gzip -2c \ + > /scratch/tmp/${DB}.tmp.gz + mv /scratch/tmp/${DB}.tmp.gz genes/$DB.gp.gz + echo "${DB} done" +done + # 3. refGene + for DB in rheMac3 +do +hgsql -N -e "select * from refGene" ${DB} | cut -f2- \ + | genePredSingleCover stdin stdout | gzip -2c \ + > /scratch/tmp/${DB}.tmp.gz + mv /scratch/tmp/${DB}.tmp.gz genes/$DB.gp.gz + echo "${DB} done" +done + + # verify counts for genes are reasonable: + for T in genes/*.gz +do + echo -n "# $T: " + zcat $T | cut -f1 | sort | uniq -c | wc -l +done +# genes/canFam3.gp.gz: 19507 +# genes/hg38.gp.gz: 21887 +# genes/mm10.gp.gz: 21013 +# genes/monDom5.gp.gz: 21033 +# genes/panTro4.gp.gz: 18657 +# genes/rheMac3.gp.gz: 5614 +# genes/rn5.gp.gz: 22863 + + time (cat ../anno/hg38.7way.maf \ + | nice -n +19 genePredToMafFrames hg38 stdin stdout \ + `sed -e "s#\([a-zA-Z0-9]*\)#\1 genes/\1.gp.gz#g" ../species.list` \ + | gzip > multiz7wayFrames.bed.gz) + # real 3m44.591s + + # verify there are frames on everything, should be 7 species: + zcat multiz7wayFrames.bed.gz | awk '{print $4}' | sort | uniq -c +# 265160 canFam3 +# 208941 hg38 +# 253323 mm10 +# 574521 monDom5 +# 200156 panTro4 +# 49802 rheMac3 +# 244731 rn5 + + # load the resulting file + ssh hgwdev + cd /hive/data/genomes/hg38/bed/multiz7way/frames + time hgLoadMafFrames hg38 multiz7wayFrames multiz7wayFrames.bed.gz + # real 0m19.959s + + time featureBits -countGaps hg38 multiz7wayFrames + # 52686177 bases of 3209286105 (1.642%) in intersection + # real 0m12.593s + + # enable the trackDb entries: +# frames multiz7wayFrames +# irows on + # appears to work OK + +######################################################################### +# Phylogenetic tree from 7-way (DONE - 2014-06-04 - Hiram) + mkdir /hive/data/genomes/hg38/bed/multiz7way/4d + cd /hive/data/genomes/hg38/bed/multiz7way/4d + + # the annotated maf is: + ../anno/hg38.7way.maf + + # using knownGene for hg38 + hgsql -N -e "select name,chrom,strand,txStart,txEnd,cdsStart,cdsEnd,exonCount,exonStarts,exonEnds from knownGene" hg38 > hg38.knownGene.gp + + genePredSingleCover hg38.knownGene.gp stdout | sort > hg38.knownGeneNR.gp + wc -l hg38.knownGeneNR.gp + # 21887 hg38.knownGeneNR.gp + + mkdir annoSplit + cd annoSplit + time mafSplit -verbose=2 -outDirDepth=1 -byTarget -useFullSequenceName \ + /dev/null . ../../anno/hg38.7way.maf + # real 5m14.770s + + find . -type f | wc -l + # 353 + ssh ku + mkdir /hive/data/genomes/hg38/bed/multiz7way/4d/run + cd /hive/data/genomes/hg38/bed/multiz7way/4d/run + mkdir ../mfa + + # newer versions of msa_view have a slightly different operation + # the sed of the gp file inserts the reference species in the chr name + cat << '_EOF_' > 4d.csh +#!/bin/csh -fe +set PHASTBIN = /cluster/bin/phast.build/cornellCVS/phast.2010-12-30/bin +set r = "/hive/data/genomes/hg38/bed/multiz7way" +set c = $1:r +set infile = $r/4d/annoSplit/$2 +set outDir = $r/4d/mfa/$3:h +set outfile = $r/4d/mfa/$3 +/bin/mkdir -p $outDir +cd /scratch/tmp +/bin/awk -v C=$c '$2 == C {print}' $r/4d/hg38.knownGeneNR.gp | sed -e "s/\t$c\t/\thg38.$c\t/" > $c.gp +set NL=`wc -l $c.gp| gawk '{print $1}'` +echo $NL +if ("$NL" != "0") then + $PHASTBIN/msa_view --4d --features $c.gp -i MAF $infile -o SS > $c.ss + $PHASTBIN/msa_view -i SS --tuple-size 1 $c.ss > $outfile +else + echo "" > $outfile +endif +/bin/rm -f $c.gp $c.ss +'_EOF_' + # << happy emacs + chmod +x 4d.csh + + find ../annoSplit -type f | sed -e "s#../annoSplit/##" > maf.list + + cat << '_EOF_' > template +#LOOP +4d.csh $(file1) $(path1) {check out line+ ../mfa/$(dir1)/$(root1).mfa} +#ENDLOOP +'_EOF_' + # << happy emacs + + gensub2 maf.list single template jobList + para create jobList + para try ... check + para time +# Completed: 353 of 353 jobs +# CPU time in finished jobs: 836s 13.93m 0.23h 0.01d 0.000 y +# IO & Wait Time: 1172s 19.54m 0.33h 0.01d 0.000 y +# Average job time: 6s 0.09m 0.00h 0.00d +# Longest finished job: 72s 1.20m 0.02h 0.00d +# Submission to last job: 89s 1.48m 0.02h 0.00d + + # Not all results have contents, that is OK + + # combine mfa files + ssh hgwdev + cd /hive/data/genomes/hg38/bed/multiz7way/4d + # remove the broken empty files, size 0 and size 1: + find ./mfa -type f -size 0 | xargs rm -f + # most interesting, this did not identify files of size 1: +# find ./mfa -type f -size 1 + find ./mfa -type f | xargs ls -og | awk '$3 == 1' | awk '{print $NF}' \ + > empty.list + cat empty.list | xargs rm -f + #want comma-less species.list + /cluster/bin/phast.build/cornellCVS/phast.2010-12-30/bin/msa_view \ + --aggregate "`cat ../species.list`" mfa/*/*.mfa | sed s/"> "/">"/ \ + > 4d.all.mfa + # check they are all in there: + grep "^>" 4d.all.mfa + # >hg38 + # >panTro4 + # >rheMac3 + # >mm10 + # >rn5 + # >canFam3 + # >monDom5 + + sed 's/[a-z][a-z]*_//g; s/:[0-9\.][0-9\.]*//g; s/;//; /^ *$/d' \ + ../hg38.7way.nh | xargs echo | sed -e 's/ //g' > tree_commas.nh + # tree_commas.nh looks like: + # (((((hg38,panTro4),rheMac3),(mm10,rn5)),canFam3),monDom5) + # use phyloFit to create tree model (output is phyloFit.mod) + time nice -n +19 \ + /cluster/bin/phast.build/cornellCVS/phast.2010-12-30/bin/phyloFit \ + --EM --precision MED --msa-format FASTA --subst-mod REV \ + --tree tree_commas.nh 4d.all.mfa + # real 0m6.583s + + + mv phyloFit.mod all.mod + + grep TREE all.mod +# TREE: (((((hg38:0.00673596,panTro4:0.00686169):0.0248146,rheMac3:0.0357598):0.0970072,(mm10:0.081661,rn5:0.0874126):0.246527):0.0264964,canFam3:0.156769):0.303241,monDom5:0.303241); + + # compare these calculated lengths to the tree extracted from 130way: + grep TREE all.mod | sed -e 's/TREE: //' \ + | /cluster/bin/phast/all_dists /dev/stdin | grep hg38 | sort -k3n \ + | sed -e "s/hg38.//; s/^/ # /" + # panTro4 0.013598 + # rheMac3 0.067310 + # canFam3 0.311823 + # mm10 0.456746 + # rn5 0.462497 + # monDom5 0.761536 + + # yes, somewhat similar + /cluster/bin/phast/all_dists ../hg38.7way.nh | grep hg38 \ + | sort -k3n | sed -e "s/hg38.//; s/^/ # /" + # panTro4 0.013390 + # rheMac3 0.071575 + # canFam3 0.330429 + # mm10 0.500391 + # rn5 0.507471 + # monDom5 0.763679 + +######################################################################### +# phastCons 7-way (DONE - 2014-06-04 - Hiram) + # split 7way mafs into 10M chunks and generate sufficient statistics + # files for # phastCons + ssh ku + mkdir -p /hive/data/genomes/hg38/bed/multiz7way/cons/SS + cd /hive/data/genomes/hg38/bed/multiz7way/cons/SS + mkdir result done + + cat << '_EOF_' > mkSS.csh +#!/bin/csh -ef +set d = $1 +set c = $2 +set doneDir = done/$d +set MAF = /hive/data/genomes/hg38/bed/multiz7way/anno/result/$d/$c.maf +set WINDOWS = /hive/data/genomes/hg38/bed/multiz7way/cons/SS/result/$d/$c +set WC = `cat $MAF | wc -l` +set NL = `grep "^#" $MAF | wc -l` +if ( -s $3 ) then + exit 0 +endif +if ( -s $3.running ) then + exit 0 +endif + +/bin/mkdir -p $doneDir +/bin/date >> $3.running + +/bin/rm -fr $WINDOWS +/bin/mkdir -p $WINDOWS +pushd $WINDOWS > /dev/null +if ( $WC != $NL ) then +/cluster/bin/phast.build/cornellCVS/phast.2010-12-30/bin/msa_split \ + $MAF -i MAF -o SS -r $WINDOWS/$c -w 10000000,0 -I 1000 -B 5000 +endif +popd > /dev/null +/bin/date >> $3 +/bin/rm -f $3.running +'_EOF_' + # << happy emacs + chmod +x mkSS.csh + + cat << '_EOF_' > template +#LOOP +mkSS.csh $(dir1) $(root1) {check out line+ done/$(dir1)/$(root1)} +#ENDLOOP +'_EOF_' + # << happy emacs + + # do the easy ones first to see some immediate results + find ../../anno/result -type f | sed -e "s#../../anno/result/##" > maf.list + + gensub2 maf.list single template jobList + para -ram=32g create jobList + para try ... check ... etc +# Completed: 353 of 353 jobs +# CPU time in finished jobs: 1216s 20.27m 0.34h 0.01d 0.000 y +# IO & Wait Time: 1385s 23.08m 0.38h 0.02d 0.000 y +# Average job time: 7s 0.12m 0.00h 0.00d +# Longest finished job: 111s 1.85m 0.03h 0.00d +# Submission to last job: 189s 3.15m 0.05h 0.00d + + find ./result -type f | wc -l + # 641 + + # Run phastCons + # This job is I/O intensive in its output files, beware where this + # takes place or do not run too many at once. + ssh ku + mkdir -p /hive/data/genomes/hg38/bed/multiz7way/cons/run.cons + cd /hive/data/genomes/hg38/bed/multiz7way/cons/run.cons + + # This is setup for multiple runs based on subsets, but only running + # the 'all' subset here. + # It triggers off of the current working directory + # $cwd:t which is the "grp" in this script. Running: + # all and vertebrates + + cat << '_EOF_' > doPhast.csh +#!/bin/csh -fe +set PHASTBIN = /cluster/bin/phast.build/cornellCVS/phast.2010-12-30/bin +set c = $1 +set d = $2 +set f = $3 +set len = $4 +set cov = $5 +set rho = $6 +set grp = $cwd:t +set cons = /hive/data/genomes/hg38/bed/multiz7way/cons +set tmp = $cons/tmp/${d}_${c} +mkdir -p $tmp +set ssSrc = $cons/SS/result +set useGrp = "$grp.mod" +if (-s $cons/$grp/$grp.non-inf) then + ln -s $cons/$grp/$grp.mod $tmp + ln -s $cons/$grp/$grp.non-inf $tmp + ln -s $ssSrc/$d/$f $tmp +else + ln -s $ssSrc/$d/$f $tmp + ln -s $cons/$grp/$grp.mod $tmp +endif +pushd $tmp > /dev/null +if (-s $grp.non-inf) then + $PHASTBIN/phastCons $f $useGrp \ + --rho $rho --expected-length $len --target-coverage $cov --quiet \ + --not-informative `cat $grp.non-inf` \ + --seqname $c --idpref $c --most-conserved $c.bed --score > $c.pp +else + $PHASTBIN/phastCons $f $useGrp \ + --rho $rho --expected-length $len --target-coverage $cov --quiet \ + --seqname $c --idpref $c --most-conserved $c.bed --score > $c.pp +endif +popd > /dev/null +mkdir -p pp/$d bed/$d +sleep 4 +touch pp/$d bed/$d +rm -f pp/$d/$c.pp +rm -f bed/$d/$c.bed +mv $tmp/$c.pp pp/$d +mv $tmp/$c.bed bed/$d +rm -fr $tmp +rmdir --ignore-fail-on-non-empty $cons/tmp/$d:h +'_EOF_' + # << happy emacs + chmod +x doPhast.csh + + # this template will serve for all runs + # root1 == chrom name, file1 == ss file name without .ss suffix + cat << '_EOF_' > template +#LOOP +../run.cons/doPhast.csh $(root1) $(dir1) $(file1) 45 0.3 0.3 {check out line+ pp/$(dir1)/$(root1).pp} +#ENDLOOP +'_EOF_' + # << happy emacs + + find ../SS/result -type f | sed -e "s#../SS/result/##" > ss.list + wc -l ss.list + # 641 ss.list + + # Create parasol batch and run it + # run for all species + cd /hive/data/genomes/hg38/bed/multiz7way/cons + mkdir -p all + cd all + # Using the .mod tree + cp -p ../../4d/all.mod ./all.mod + + gensub2 ../run.cons/ss.list single ../run.cons/template jobList + para -ram=32g create jobList + para try ... check ... + para push +# Completed: 641 of 641 jobs +# CPU time in finished jobs: 6557s 109.29m 1.82h 0.08d 0.000 y +# IO & Wait Time: 4497s 74.94m 1.25h 0.05d 0.000 y +# Average job time: 17s 0.29m 0.00h 0.00d +# Longest finished job: 33s 0.55m 0.01h 0.00d +# Submission to last job: 120s 2.00m 0.03h 0.00d + + # create Most Conserved track + cd /hive/data/genomes/hg38/bed/multiz7way/cons/all + cut -f1 ../../../../chrom.sizes | while read C +do + ls -d bed/?/${C} 2> /dev/null | while read D + do + echo ${D}/${C}*.bed 1>&2 + cat ${D}/${C}*.bed + done | sort -k1,1 -k2,2n \ + | awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", "'${C}'", $2, $3, $5, $5;}' +done > tmpMostConserved.bed + + /cluster/bin/scripts/lodToBedScore tmpMostConserved.bed > mostConserved.bed + # -rw-rw-r-- 1 42636652 Jun 4 10:45 tmpMostConserved.bed + # -rw-rw-r-- 1 43721828 Jun 4 10:45 mostConserved.bed + + # load into database + ssh hgwdev + cd /hive/data/genomes/hg38/bed/multiz7way/cons/all + time nice -n +19 hgLoadBed hg38 phastConsElements7way mostConserved.bed + # Read 1234990 elements of size 5 from mostConserved.bed + # real 0m11.390s + + # on human we often try for 5% overall cov, and 70% CDS cov + # most bets are off here for that goal, these alignments are too few + # and too far between + # --rho 0.3 --expected-length 45 --target-coverage 0.3 + featureBits hg38 -enrichment knownGene:cds phastConsElements7way + # knownGene:cds 1.266%, phastConsElements7way 4.551%, + # both 0.888%, cover 70.16%, enrich 15.42x + + # Create merged posterier probability file and wiggle track data files + cd /hive/data/genomes/hg38/bed/multiz7way/cons/all + mkdir downloads + + # the third sed fixes the chrom names, removing the partition extensions + time (find ./pp -type f | sed -e "s#^./##; s#\.# d #g; s#-# m #;" \ + | sort -k1,1 -k3,3n | sed -e "s# d #.#g; s# m #-#g;" | xargs cat \ + | sed -e 's/\.[0-9][0-9]*-[0-9][0-9]* start/ start/' \ + | gzip -c > downloads/phastCons7way.wigFix.gz) + # real 37m47.242s + + # check integrity of data with wigToBigWig + time (zcat downloads/phastCons7way.wigFix.gz \ + | wigToBigWig -verbose=2 stdin /hive/data/genomes/hg38/chrom.sizes \ + phastCons7way.bw) > bigWig.log 2>&1 & + tail bigWig.log + # pid=34733: VmPeak: 33106324 kB + # real 40m53.287s + + bigWigInfo phastCons7way.bw +# version: 4 +# isCompressed: yes +# isSwapped: 0 +# primaryDataSize: 5,675,802,079 +# primaryIndexSize: 92,579,900 +# zoomLevels: 10 +# chromCount: 353 +# basesCovered: 2,898,191,577 +# mean: 0.168088 +# min: 0.000000 +# max: 1.000000 +# std: 0.233827 + + # encode those files into wiggle data + time (zcat downloads/phastCons7way.wigFix.gz \ + | wigEncode stdin phastCons7way.wig phastCons7way.wib) + # Converted stdin, upper limit 1.00, lower limit 0.00 + # real 15m28.525s + + du -hsc *.wi? + # 2.7G phastCons7way.wib + # 282M phastCons7way.wig + # 3.0G total + + # Load gbdb and database with wiggle. + ln -s `pwd`/phastCons7way.wib /gbdb/hg38/multiz7way/phastCons7way.wib + time nice -n +19 hgLoadWiggle -pathPrefix=/gbdb/hg38/multiz7way \ + hg38 phastCons7way phastCons7way.wig + # real 0m33.502s + + # use to set trackDb.ra entries for wiggle min and max + # and verify table is loaded correctly + + wigTableStats.sh hg38 phastCons7way +# db.table min max mean count sumData stdDev viewLimits +hg38.phastCons7way 0 1 0.168088 2898191577 4.87152e+08 0.233827 viewLimits=0:1 + + # Create histogram to get an overview of all the data + time nice -n +19 hgWiggle -doHistogram -db=hg38 \ + -hBinSize=0.001 -hBinCount=1000 -hMinVal=0.0 -verbose=2 \ + phastCons7way > histogram.data 2>&1 + # real 2m40.179s + + # create plot of histogram: + + cat << '_EOF_' | gnuplot > histo.png +set terminal png small x000000 xffffff xc000ff x66ff66 xffff00 x00ffff +set size 1.4, 0.8 +set key left box +set grid noxtics +set grid ytics +set title " Human hg38 Histogram phastCons7way track" +set xlabel " phastCons7way score" +set ylabel " Relative Frequency" +set y2label " Cumulative Relative Frequency (CRF)" +set y2range [0:1] +set y2tics +set yrange [0:0.02] + +plot "histogram.data" using 2:5 title " RelFreq" with impulses, \ + "histogram.data" using 2:7 axes x1y2 title " CRF" with lines +'_EOF_' + # << happy emacs + + display histo.png & + +######################################################################### +# phyloP for 7-way (DONE - 2014-06-04 - Hiram) + # run phyloP with score=LRT + ssh ku + mkdir /cluster/data/hg38/bed/multiz7way/consPhyloP + cd /cluster/data/hg38/bed/multiz7way/consPhyloP + + mkdir run.phyloP + cd run.phyloP + # Adjust model file base composition background and rate matrix to be + # representative of the chromosomes in play + grep BACKGROUND ../../cons/all/all.mod | awk '{printf "%0.3f\n", $3 + $4}' + # 0.556 + /cluster/bin/phast.build/cornellCVS/phast.2010-12-30/bin/modFreqs \ + ../../cons/all/all.mod 0.556 > all.mod + # verify, the BACKGROUND should now be paired up: + grep BACK all.mod + # BACKGROUND: 0.222000 0.278000 0.278000 0.222000 + + cat << '_EOF_' > doPhyloP.csh +#!/bin/csh -fe +set PHASTBIN = /cluster/bin/phast.build/cornellCVS/phast.2010-12-30/bin +set f = $1 +set d = $f:h +set file1 = $f:t +set out = $2 +set cName = $f:t:r +set grp = $cwd:t +set cons = /hive/data/genomes/hg38/bed/multiz7way/consPhyloP +set tmp = $cons/tmp/$grp/$f +/bin/rm -fr $tmp +/bin/mkdir -p $tmp +set ssSrc = "/hive/data/genomes/hg38/bed/multiz7way/cons/SS/result/$f" +set useGrp = "$grp.mod" +/bin/ln -s $cons/run.phyloP/$grp.mod $tmp +pushd $tmp > /dev/null +$PHASTBIN/phyloP --method LRT --mode CONACC --wig-scores --chrom $cName \ + -i SS $useGrp $ssSrc.ss > $file1.wigFix +popd > /dev/null +/bin/mkdir -p $out:h +sleep 4 +/bin/touch $out:h +/bin/mv $tmp/$file1.wigFix $out +/bin/rm -fr $tmp +/bin/rmdir --ignore-fail-on-non-empty $cons/tmp/$grp/$d +/bin/rmdir --ignore-fail-on-non-empty $cons/tmp/$grp/$d:h +/bin/rmdir --ignore-fail-on-non-empty $cons/tmp/$grp +/bin/rmdir --ignore-fail-on-non-empty $cons/tmp +'_EOF_' + # << happy emacs + + # Create list of chunks + find ../../cons/SS/result -type f | grep ".ss$" \ + | sed -e "s/.ss$//; s#^../../cons/SS/result/##" > ss.list + # make sure the list looks good + wc -l ss.list + # 641 ss.list + + # Create template file + # file1 == $chr/$chunk/file name without .ss suffix + cat << '_EOF_' > template +#LOOP +../run.phyloP/doPhyloP.csh $(path1) {check out line+ wigFix/$(dir1)/$(file1).wigFix} +#ENDLOOP +'_EOF_' + # << happy emacs + + ###################### Running all species ####################### + # setup run for all species + mkdir /hive/data/genomes/hg38/bed/multiz7way/consPhyloP/all + cd /hive/data/genomes/hg38/bed/multiz7way/consPhyloP/all + rm -fr wigFix + mkdir wigFix + + gensub2 ../run.phyloP/ss.list single ../run.phyloP/template jobList + # the -ram=8g will allow only one job per node to slow this down since + # it would run too fast otherwise. Either run on one of the small + # klusters or use the -ram=8g on the para create + para -ram=32g create jobList + para try ... check ... push ... etc ... + para time > run.time +# Completed: 641 of 641 jobs +# CPU time in finished jobs: 4755s 79.24m 1.32h 0.06d 0.000 y +# IO & Wait Time: 4343s 72.39m 1.21h 0.05d 0.000 y +# Average job time: 14s 0.24m 0.00h 0.00d +# Longest finished job: 27s 0.45m 0.01h 0.00d +# Submission to last job: 1152s 19.20m 0.32h 0.01d + + # make downloads + mkdir downloads + + time (find ./wigFix -type f | sed -e "s#^./##; s#\.# d #g; s#-# m #;" \ + | sort -k1,1 -k3,3n | sed -e "s# d #.#g; s# m #-#g;" | xargs cat \ + | gzip -c > downloads/phyloP7way.wigFix.gz) & + # real 29m51.665s + + # check integrity of data with wigToBigWig + time (zcat downloads/phyloP7way.wigFix.gz \ + | wigToBigWig -verbose=2 stdin /hive/data/genomes/hg38/chrom.sizes \ + phyloP7way.bw) > bigWig.log 2>&1 & + egrep "real|VmPeak" bigWig.log + # pid=76577: VmPeak: 33106320 kB + # real 42m53.038s + + + bigWigInfo phyloP7way.bw +# version: 4 +# isCompressed: yes +# isSwapped: 0 +# primaryDataSize: 3,759,451,708 +# primaryIndexSize: 92,579,900 +# zoomLevels: 10 +# chromCount: 353 +# basesCovered: 2,898,191,577 +# mean: 0.074472 +# min: -5.220000 +# max: 1.062000 +# std: 0.545945 + + # encode those files into wiggle data + time (zcat downloads/phyloP7way.wigFix.gz \ + | wigEncode stdin phyloP7way.wig phyloP7way.wib) & + # Converted stdin, upper limit 1.06, lower limit -5.22 + # real 16m11.861s + + + du -hsc *.wi? + # 47M phyloP7way.wib + # 12M phyloP7way.wig + # 58M total + + # Load gbdb and database with wiggle. + ln -s `pwd`/phyloP7way.wib /gbdb/hg38/multiz7way/phyloP7way.wib + nice hgLoadWiggle -pathPrefix=/gbdb/hg38/multiz7way hg38 \ + phyloP7way phyloP7way.wig + + # use to set trackDb.ra entries for wiggle min and max + # and verify table is loaded correctly + + wigTableStats.sh hg38 phyloP7way +# db.table min max mean count sumData +# hg38.phyloP7way -5.22 1.062 0.0744721 2898191577 2.15834e+08 +# stdDev viewLimits +# 0.545945 viewLimits=-2.65525:1.062 + + # that range is: 5.22+1.062 = 6.282 for hBinSize=0.006282 + + # Create histogram to get an overview of all the data + time nice -n +19 hgWiggle -doHistogram \ + -hBinSize=0.006282 -hBinCount=1000 -hMinVal=-5.22 -verbose=2 \ + -db=hg38 phyloP7way > histogram.data 2>&1 + # real 2m55.843s + + + # find out the range for the 2:5 graph + grep -v chrom histogram.data | grep "^[0-9]" | ave -col=5 stdin +# Q1 0.000001 +# median 0.000060 +# Q3 0.000656 +# average 0.001022 +# min 0.000000 +# max 0.065461 +# count 978 +# total 0.999982 +# standard deviation 0.004157 + + # create plot of histogram: + cat << '_EOF_' | gnuplot > histo.png +set terminal png small x000000 xffffff xc000ff x66ff66 xffff00 x00ffff +set size 1.4, 0.8 +set key left box +set grid noxtics +set grid ytics +set title " Human hg38 Histogram phyloP7way track" +set xlabel " phyloP7way score" +set ylabel " Relative Frequency" +set y2label " Cumulative Relative Frequency (CRF)" +set y2range [0:1] +set y2tics +set yrange [0:0.02] + +plot "histogram.data" using 2:5 title " RelFreq" with impulses, \ + "histogram.data" using 2:7 axes x1y2 title " CRF" with lines +'_EOF_' + # << happy emacs + + display histo.png & + +############################################################################# +# construct download files for 7-way (DONE - 2014-06-05 - Hiram) + mkdir /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/multiz7way + mkdir /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/phastCons7way + mkdir /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/phyloP7way + mkdir /hive/data/genomes/hg38/bed/multiz7way/downloads + cd /hive/data/genomes/hg38/bed/multiz7way/downloads + mkdir multiz7way phastCons7way phyloP7way + cd multiz7way + time cp -p ../../anno/hg38.7way.maf . + # real 0m55.984s + time gzip *.maf + # real 46m53.149s + + ln -s ../../hg38.7way.nh . + ln -s ../../hg38.7way.commonNames.nh . + time md5sum *.nh *.maf.gz > md5sum.txt + # real 1m55.317s + ln -s `pwd`/* \ + /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/multiz7way + + du -hsc *.maf.gz ../../anno/hg38.7way.maf + # 3.5G hg38.7way.maf.gz + # 17G ../../anno/hg38.7way.maf + + ##################################################################### + cd /hive/data/genomes/hg38/bed/multiz7way/downloads/phastCons7way + + ln -s ../../cons/all/downloads/phastCons7way.wigFix.gz \ + ./hg38.phastCons7way.wigFix.gz + ln -s ../../cons/all/phastCons7way.bw ./hg38.phastCons7way.bw + ln -s ../../cons/all/all.mod ./hg38.phastCons7way.mod + time md5sum *.gz *.mod *.bw > md5sum.txt + # real 0m37.384s + # obtain the README.txt from petMar2/phastCons7way and update for this + # situation + ln -s `pwd`/*.gz `pwd`/*.mod `pwd`/*.bw `pwd`/*.txt \ + /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/phastCons7way + + ##################################################################### + cd /hive/data/genomes/hg38/bed/multiz7way/downloads/phyloP7way + + ln -s ../../consPhyloP/all/downloads/phyloP7way.wigFix.gz \ + ./hg38.phyloP7way.wigFix.gz + ln -s ../../consPhyloP/run.phyloP/all.mod hg38.phyloP7way.mod + ln -s ../../consPhyloP/all/phyloP7way.bw hg38.phyloP7way.bw + + time md5sum *.mod *.bw *.gz > md5sum.txt + # real 0m29.431s + + # obtain the README.txt from geoFor1/phyloP7way and update for this + # situation + ln -s `pwd`/* \ + /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/phyloP7way + + ########################################################################### + ## create upstream refGene maf files + cd /hive/data/genomes/hg38/bed/multiz7way/downloads/multiz7way + # bash script +#!/bin/sh +export geneTbl="knownGene" +for S in 1000 2000 5000 +do + echo "making upstream${S}.maf" + featureBits hg38 ${geneTbl}:upstream:${S} -fa=/dev/null -bed=stdout \ + | perl -wpe 's/_up[^\t]+/\t0/' | sort -k1,1 -k2,2n \ + | /cluster/bin/$MACHTYPE/mafFrags hg38 multiz7way \ + stdin stdout \ + -orgs=/hive/data/genomes/hg38/bed/multiz7way/species.list \ + | gzip -c > upstream${S}.${geneTbl}.maf.gz + echo "done upstream${S}.${geneTbl}.maf.gz" +done + # real 60m16.631s + + md5sum upstream*.gz >> md5sum.txt + + # some other symlinks were already made above + # obtain the README.txt from geoFor1/multiz7way and update for this + # situation + ln -s `pwd`/upstream*.gz README.txt \ + /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/multiz7way + +############################################################################# +# hgPal downloads (DONE - 2014-06-06 - Hiram) +# FASTA from 7-way for knownGene, refGene and knownCanonical + + ssh hgwdev + screen -S hg38HgPal + mkdir /hive/data/genomes/hg38/bed/multiz7way/pal + cd /hive/data/genomes/hg38/bed/multiz7way/pal + cat ../species.list | tr '[ ]' '[\n]' > order.list + + export mz=multiz7way + export gp=knownGene + export db=hg38 + export I=0 + mkdir exonAA exonNuc + for C in `sort -nk2 ../../../chrom.sizes | cut -f1` + do + I=`echo $I | awk '{print $1+1}'` + echo "mafGene -chrom=$C -exons -noTrans $db $mz $gp order.list stdout | gzip -c > exonNuc/$C.exonNuc.fa.gz &" + echo "mafGene -chrom=$C -exons $db $mz $gp order.list stdout | gzip -c > exonAA/$C.exonAA.fa.gz &" + if [ $I -gt 6 ]; then + echo "date" + echo "wait" + I=0 + fi + done > $gp.jobs + echo "date" >> $gp.jobs + echo "wait" >> $gp.jobs + + time ./$gp.jobs > $gp.jobs.log 2>&1 & + # real 28m46.919s + + time zcat exonAA/*.gz | gzip -c > $gp.$mz.exonAA.fa.gz + # real 0m23.798s + time zcat exonNuc/*.gz | gzip -c > $gp.$mz.exonNuc.fa.gz + # real 1m28.197s + + export mz=multiz7way + export gp=knownGene + export db=hg38 + export pd=/usr/local/apache/htdocs-hgdownload/goldenPath/$db/$mz/alignments + mkdir -p $pd + md5sum *.fa.gz > md5sum.txt + ln -s `pwd`/$gp.$mz.exonAA.fa.gz $pd/$gp.exonAA.fa.gz + ln -s `pwd`/$gp.$mz.exonNuc.fa.gz $pd/$gp.exonNuc.fa.gz + ln -s `pwd`/md5sum.txt $pd/ + + rm -rf exonAA exonNuc + + ### need other gene track alignments also + # running up refGene + cd /hive/data/genomes/hg38/bed/multiz7way/pal + export mz=multiz7way + export gp=refGene + export db=hg38 + export I=0 + mkdir exonAA exonNuc + for C in `sort -nk2 ../../../chrom.sizes | cut -f1` + do + I=`echo $I | awk '{print $1+1}'` + echo "mafGene -chrom=$C -exons -noTrans $db $mz $gp order.list stdout | gzip -c > exonNuc/$C.exonNuc.fa.gz &" + echo "mafGene -chrom=$C -exons $db $mz $gp order.list stdout | gzip -c > exonAA/$C.exonAA.fa.gz &" + if [ $I -gt 6 ]; then + echo "date" + echo "wait" + I=0 + fi + done > $gp.jobs + echo "date" >> $gp.jobs + echo "wait" >> $gp.jobs + + time sh -x $gp.jobs > $gp.jobs.log 2>&1 + # real 15m15.424s + + export mz=multiz7way + export gp=refGene + export db=hg38 + time zcat exonAA/*.gz | gzip -c > $gp.$mz.exonAA.fa.gz + # real 0m23.119s + time zcat exonNuc/*.gz | gzip -c > $gp.$mz.exonNuc.fa.gz + # real 1m15.547s + + du -hsc exonAA exonNuc refGene*.fa.gz + # 59M exonAA + # 101M exonNuc + # 59M refGene.multiz7way.exonAA.fa.gz + # 101M refGene.multiz7way.exonNuc.fa.gz + # 317M total + + rm -rf exonAA exonNuc + + # we're only distributing exons at the moment + export mz=multiz7way + export gp=refGene + export db=hg38 + export pd=/usr/local/apache/htdocs-hgdownload/goldenPath/$db/$mz/alignments + mkdir -p $pd + md5sum *.fa.gz > md5sum.txt + ln -s `pwd`/$gp.$mz.exonAA.fa.gz $pd/$gp.exonAA.fa.gz + ln -s `pwd`/$gp.$mz.exonNuc.fa.gz $pd/$gp.exonNuc.fa.gz + ln -s `pwd`/md5sum.txt $pd/ + + ### And knownCanonical + cd /hive/data/genomes/hg38/bed/multiz7way/pal + export mz=multiz7way + export gp=knownCanonical + export db=hg38 + mkdir exonAA exonNuc ppredAA ppredNuc knownCanonical + + cut -f1 ../../../chrom.sizes | while read C + do + echo $C + hgsql hg38 -N -e "select chrom, chromStart, chromEnd, transcript from knownCanonical where chrom='$C'" > knownCanonical/$C.known.bed + done + + ls knownCanonical/*.known.bed | while read F + do + if [ -s $F ]; then + echo $F | sed -e 's#knownCanonical/##; s/.known.bed//' + fi + done | while read C + do + echo "date" + echo "mafGene -geneBeds=knownCanonical/$C.known.bed $db $mz knownGene order.list stdout | \ + gzip -c > ppredAA/$C.ppredAA.fa.gz" + echo "mafGene -geneBeds=knownCanonical/$C.known.bed -noTrans $db $mz knownGene order.list stdout | \ + gzip -c > ppredNuc/$C.ppredNuc.fa.gz" + echo "mafGene -geneBeds=knownCanonical/$C.known.bed -exons -noTrans $db $mz knownGene order.list stdout | \ + gzip -c > exonNuc/$C.exonNuc.fa.gz" + echo "mafGene -geneBeds=knownCanonical/$C.known.bed -exons $db $mz knownGene order.list stdout | \ + gzip -c > exonAA/$C.exonAA.fa.gz" + done > $gp.$mz.jobs + + time sh -x $gp.$mz.jobs > $gp.$mz.job.log 2>&1 + # real 72m58.133s + + rm *.known.bed + mz=multiz7way + gp=knownCanonical + db=hg38 + zcat exonAA/c*.gz | gzip -c > $gp.$mz.exonAA.fa.gz & + zcat exonNuc/c*.gz | gzip -c > $gp.$mz.exonNuc.fa.gz & + zcat ppredAA/c*.gz | gzip -c > $gp.$mz.ppredAA.fa.gz & + zcat ppredNuc/c*.gz | gzip -c > $gp.$mz.ppredNuc.fa.gz + + rm -rf exonAA exonNuc ppredAA ppredNuc + + mz=multiz7way + gp=knownCanonical + db=hg38 + pd=/usr/local/apache/htdocs-hgdownload/goldenPath/$db/$mz/alignments + mkdir -p $pd + ln -s `pwd`/$gp.$mz.exonAA.fa.gz $pd/$gp.exonAA.fa.gz + ln -s `pwd`/$gp.$mz.exonNuc.fa.gz $pd/$gp.exonNuc.fa.gz + cd $pd + md5sum *.exon*.fa.gz > md5sum.txt + +############################################################################# +# wiki page for 7-way (DONE - 2014-06-04 - Hiram) + mkdir /hive/users/hiram/bigWays/hg38.7way + cd /hive/users/hiram/bigWays + echo "hg38" > hg38.7way/ordered.list + awk '{print $1}' /hive/data/genomes/hg38/bed/multiz7way/7way.distances.txt \ + >> hg38.7way/ordered.list + + # sizeStats.sh catches up the cached measurements required for data + # in the tables. They may already be done. + ./sizeStats.sh hg38.7way/ordered.list + # dbDb.sh constructs hg38.7way/Hg38_7-way_conservation_alignment.html + ./dbDb.sh hg38 7way + # sizeStats.pl constructs hg38.7way/Hg38_7-way_Genome_size_statistics.html + ./sizeStats.pl hg38 7way + + # defCheck.pl constructs Hg38_7-way_conservation_lastz_parameters.html + ./defCheck.pl hg38 7way + + # this constructs the html pages in hg38.7way/: +# -rw-rw-r-- 1 4153 Jun 5 11:03 Hg38_7-way_conservation_alignment.html +# -rw-rw-r-- 1 5833 Jun 5 11:04 Hg38_7-way_Genome_size_statistics.html +# -rw-rw-r-- 1 3854 Jun 5 11:04 Hg38_7-way_conservation_lastz_parameters.html + + # add those pages to the genomewiki. Their page names are the + # names of the .html files without the .html: +# Hg38_7-way_conservation_alignment +# Hg38_7-way_Genome_size_statistics +# Hg38_7-way_conservation_lastz_parameters + + # when you view the first one you enter, it will have links to the + # missing two. + +############################################################################# +# GRC Incident database (DONE - 2014-06-14 - Hiram) + # this procedure is run as a cron job in Hiram's account: + + # 33 09 * * * /hive/data/outside/grc/incidentDb/runUpdate.sh makeItSo + + # data comes from: ftp://ftp.ncbi.nlm.nih.gov/pub/grc/ + # processed by /hive/data/outside/grc/incidentDb/grcUpdate.sh + + # the table in the dataBase is: grcIncidentDb + # which is the URL to the bb file, a single row: + # http://genomewiki.ucsc.edu/images/7/7f/Hg38.grcIncidentDb.bb + +############################################################################# +# RepeatMasker Visualization track (DONE - 2014-07-25 - Hiram) + mkdir /hive/data/genomes/hg38/bed/rmskJoined + cd /hive/data/genomes/hg38/bed/rmskJoined + + ln -s ../repeatMasker/hg38.sorted.fa.out . + ln -s ../repeatMasker/hg38.fa.align.gz . + + # working on fixing this script for the next release of RM + /scratch/data/RepeatMasker140131/util/nextVerRmToUCSCTables.pl \ + -out hg38.sorted.fa.out -align hg38.fa.align.gz + + hgLoadBed -sqlTable=$HOME/kent/src/hg/lib/rmskJoined.sql \ + -renameSqlTable -verbose=4 -tab \ + -type=bed4+9 -as=$HOME/kent/src/hg/lib/rmskJoined.as hg38 \ + rmskJoinedBaseline hg38.sorted.fa.join.bed \ + > loadJoined.log 2>&1 + + hgLoadSqlTab hg38 rmskAlignBaseline \ + /cluster/home/hiram/kent/src/hg/lib/rmskAlign.sql \ + hg38.fa.align.tsv > loadAlign.log 2>&1 + + hgLoadOutJoined -verbose=2 hg38 hg38.sorted.fa.out > loadOut.log 2>&1 + + featureBits -countGaps hg38 rmskJoinedBaseline + # 2716777279 bases of 3209286105 (84.654%) in intersection + +############################################################################## +# LASTZ Macaca Mulatta RheMac2 (DONE - 2014-07-13 - braney) + mkdir /hive/data/genomes/hg38/bed/lastzRheMac2.2014-07-11 + cd /hive/data/genomes/hg38/bed/lastzRheMac2.2014-07-11 + + # best to always specify an exact path to lastz so we know which one is used + # lastz default parameters are human-mouse parameters + + cat << '_EOF_' > DEF +# human vs macaca mulatta +BLASTZ=/cluster/bin/penn/lastz-distrib-1.03.52/bin/lastz +# maximum M allowed with lastz is only 254 +BLASTZ_M=254 +BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q +BLASTZ_O=600 +BLASTZ_E=150 +# other parameters from panTro2 vs hg18 lastz on advice from Webb +BLASTZ_K=4500 +BLASTZ_Y=15000 +BLASTZ_T=2 + +# TARGET: Human Hg38 +SEQ1_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ1_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ1_CTGDIR=/hive/data/genomes/hg38/hg38.contigs.2bit +SEQ1_CTGLEN=/hive/data/genomes/hg38/hg38.contigs.chrom.sizes +SEQ1_LIFT=/hive/data/genomes/hg38/jkStuff/hg38.contigs.lift +SEQ1_CHUNK=20000000 +SEQ1_LAP=10000 + +# QUERY: Macaca Mulatta RheMac2 +SEQ2_DIR=/scratch/data/rheMac2/rheMac2.2bit +SEQ2_LEN=/scratch/data/rheMac2/chrom.sizes +SEQ2_CHUNK=20000000 +SEQ2_LAP=0 +SEQ2_IN_CONTIGS=0 + +BASE=/hive/data/genomes/hg38/bed/lastzRheMac2.2014-07-11 +TMPDIR=/dev/shm +'_EOF_' + # << happy emacs + time $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \ + `pwd`/DEF \ + -syntenicNet -fileServer=hgwdev \ + -chainMinScore=5000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku > do.log 2>&1 + # Elapsed time: 141m36s + cat fb.hg38.chainRheMac2Link.txt + # 2455106923 bases of 3049335806 (80.513%) in intersection + + # running the swap + mkdir /hive/data/genomes/rheMac2/bed/blastz.hg38.swap + cd /hive/data/genomes/rheMac2/bed/blastz.hg38.swap + time nice -n +19 doBlastzChainNet.pl -verbose=2 \ + /hive/data/genomes/hg38/bed/lastzRheMac2.2014-07-11/DEF \ + -swap -syntenicNet -chainMinScore=5000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku > swap.log 2>&1 + # 83m26.095s + cat fb.rheMac2.chainHg38Link.txt + # 2313950599 bases of 2646704109 (87.428%) in intersection +# + +######################################################################### +# LASTZ Chlorocebus sabaeus (DONE - 2014-07-13 - braney) + mkdir /hive/data/genomes/hg38/bed/lastzChlSab2.2014-07-11 + cd /hive/data/genomes/hg38/bed/lastzChlSab2.2014-07-11 + + # best to always specify an exact path to lastz so we know which one is used + # lastz default parameters are human-mouse parameters + + cat << '_EOF_' > DEF +# human vs Chlorocebus sabaeus +BLASTZ=/cluster/bin/penn/lastz-distrib-1.03.52/bin/lastz +# maximum M allowed with lastz is only 254 +BLASTZ_M=254 +BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q +BLASTZ_O=600 +BLASTZ_E=150 +# other parameters from panTro2 vs hg18 lastz on advice from Webb +BLASTZ_K=4500 +BLASTZ_Y=15000 +BLASTZ_T=2 + + +# TARGET: Human Hg38 +SEQ1_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ1_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ1_CTGDIR=/hive/data/genomes/hg38/hg38.contigs.2bit +SEQ1_CTGLEN=/hive/data/genomes/hg38/hg38.contigs.chrom.sizes +SEQ1_LIFT=/hive/data/genomes/hg38/jkStuff/hg38.contigs.lift +SEQ1_CHUNK=20000000 +SEQ1_LAP=10000 + +# QUERY Chlorocebus sabaeus chlSab2 +SEQ2_DIR=/scratch/data/chlSab2/chlSab2.2bit +SEQ2_LEN=/scratch/data/chlSab2/chrom.sizes +SEQ2_CHUNK=20000000 +SEQ2_LAP=0 +SEQ2_IN_CONTIGS=0 + +BASE=/hive/data/genomes/hg38/bed/lastzChlSab2.2014-07-11 +TMPDIR=/dev/shm +'_EOF_' + # << happy emacs + time $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \ + `pwd`/DEF \ + -syntenicNet -fileServer=hgwdev \ + -chainMinScore=5000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku > do.log 2>&1 + # Elapsed time: 142m4s + cat fb.hg38.chainChlSab2Link.txt + # 2573435303 bases of 3049335806 (84.393%) in intersection + + # running the swap + mkdir /hive/data/genomes/chlSab2/bed/blastz.hg38.swap + cd /hive/data/genomes/chlSab2/bed/blastz.hg38.swap + time nice -n +19 doBlastzChainNet.pl -verbose=2 \ + /hive/data/genomes/hg38/bed/lastzChlSab2.2014-07-11/DEF \ + -swap -syntenicNet -chainMinScore=5000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku > swap.log 2>&1 + # 88m48.411s + cat fb.chlSab2.chainHg38Link.txt + # 2429053010 bases of 2752019208 (88.264%) in intersection + +######################################################################### +# SEGMENTAL DUPLICATIONS (DONE - 2014-08-13 - Hiram) + # redmine issue: refs #13580 + + # file received in email from Archana Natarajan Raja (araja at uw.edu) + mkdir /hive/data/genomes/hg38/bed/genomicSuperDups + cd /hive/data/genomes/hg38/bed/genomicSuperDups +# -rw-r--r-- 1 16478617 Aug 11 16:18 GenomicSuperDup.tab + + # no longer filtering items smaller than 1,000 bases, see note + # in redmine issue refs #13580 +# While the size of the 24 alignments are less than 1000 bases , the size of +# their pairs to which they align are always >1000, you can confirm this by +# looking at the value in column 22 in your table (alignB -ucsc format), will +# always be >1000 bp . We are seeing this only now because there are lots of +# new and resolved duplications added to hg38. Hence , I would recommend not +# filtering these items and uploading the current set as is. + + # there is no chrEBV in the browser: + grep -v chrEBV GenomicSuperDup.tab | sed -e 's/\t_\t/\t-\t/;' \ + | hgLoadBed hg38 genomicSuperDups stdin \ + -sqlTable=$HOME/kent/src/hg/lib/genomicSuperDups.sql + # Read 69894 elements of size 29 from stdin + + checkTableCoords hg38 genomicSuperDups + # <silence> (the chrEBV was found with this check) + + featureBits -countGaps hg38 genomicSuperDups + # 175429664 bases of 3209286105 (5.466%) in intersection + + featureBits -countGaps hg19 genomicSuperDups + # 166092393 bases of 3137161264 (5.294%) in intersection + featureBits -countGaps hg18 genomicSuperDups + # 159204446 bases of 3107677273 (5.123%) in intersection + + featureBits -countGaps mm10 genomicSuperDups + # 214917441 bases of 2730871774 (7.870%) in intersection + featureBits -countGaps mm9 genomicSuperDups + # 208214567 bases of 2725765481 (7.639%) in intersection + +############################################################################## +# cloneEnds (DONE - 2014-08-14 - Hiram) + + mkdir /hive/data/genomes/hg38/bed/cloneEnds + cd /hive/data/genomes/hg38/bed/cloneEnds + + # fetch the NCBI INSDC name correspondence file: + rsync -a -P rsync://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/All/GCF_000001405.26.assembly.txt ./ + + # fetch the clone reports + mkdir reports + rsync -a -P \ +rsync://ftp.ncbi.nih.gov/repository/clone/reports/Homo_sapiens/*.GCF_000001405.26.106.*.gff \ + ./reports/ + + # script to establish refSeq to UCSC chrom names: + + cat << '_EOF_' > refSeqNames.pl +#!/usr/bin/env perl + +use strict; +use warnings; + +open (FH, "<GCF_000001405.26.assembly.txt") or die "can not read GCF_000001405.26.assembly.txt"; +while (my $line = <FH>) { + chomp $line; + next if ($line =~ m/^#/); + my @a = split('\t', $line); + my $chrN = $a[2]; + my $refSeq = $a[6]; + my $contig = $a[4]; + my $type = $a[1]; + next if (!defined $type); + next if (!defined $refSeq); + next if (!defined $contig); + my $suffix = ""; + if ($type eq "alt-scaffold") { + $suffix = "_alt"; + } elsif ($type eq "unlocalized-scaffold") { + $suffix = "_random"; + } elsif ($type eq "unplaced-scaffold") { + $chrN = "Un"; + } + $chrN = "M" if ($chrN eq "MT"); + if ($a[0] =~ m/_/) { + $contig =~ s/\./v/; + printf "%s\tchr%s_%s%s\n", $refSeq, $chrN, $contig, $suffix; + } else { + printf "%s\tchr%s\n", $refSeq, $chrN; + } +} +close (FH); +'_EOF_' + # << happy emacs + + chmod +x refSeqNames.pl + + ./refSeqNames.pl > refSeq.ucscName.tab + + # establish full library list: + ls reports/*.GCF_000001405.26.106.*.gff | sed -e 's#reports/##' \ + | cut -d"." -f1 | sort -u > library.list.txt + + # a script to scan the GFF files, with the refSeq.ucscName.tab + # name correspondence to construct bed files + + cat << '_EOF_' > hg38.pl +#!/usr/bin/env perl + +use strict; +use warnings; + +my $argc = scalar(@ARGV); + +if ($argc < 1) { + printf STDERR "usage: ./hg38.pl <report.gff> [moreReports.gff]\n"; + exit 255; +} + +my %refSeqToUcsc; # key is refSeq name, value is UCSC chrom name +open (FH, "<refSeq.ucscName.tab") or die "can not read refSeq.ucscName.tab"; +while (my $line = <FH>) { + chomp $line; + my ($refSeq, $ucsc) = split('\t', $line); + $refSeqToUcsc{$refSeq} = $ucsc; +} +close (FH); + +my %chromSizes; # key is UCSC chrom name, key is chrom size +open (FH, "</hive/data/genomes/hg38/chrom.sizes") or die "can not read hg38/chrom.sizes"; +while (my $line = <FH>) { + chomp $line; + my ($chr, $size) = split('\t', $line); + $chromSizes{$chr} = $size; +} +close (FH); + +while (my $file = shift) { +my %starts; # key is parent ID, value is start end coordinates start,end +my %ends; # key is parent ID, value is end end coordinates start,end +my %parents; # key is parent ID, value is 1 to signify exists +my %endNames; # key is parent ID, value is the Name of the parent clone_insert + +printf STDERR "# processing $file\n"; + +open (FH, "<$file") or die "can not read $file"; +while (my $line = <FH>) { + chomp $line; + next if ($line=~ m/^#/); + my @a = split('\t', $line); + next if (scalar(@a) < 1); + my $contig = $a[0]; + $contig =~ s/ref.//; + $contig =~ s/\|//; + my $ucscChr = $refSeqToUcsc{$contig}; + if (!defined($ucscChr)) { + printf STDERR "# ERR: contig not in refSeqToUcsc: '$contig'\n"; + next; + } + next if (! exists($chromSizes{$ucscChr})); + my $chromSize = $chromSizes{$ucscChr}; + my $chromStart = $a[3] - 1; + my $chromEnd = $a[4]; + if ($chromStart > $chromSize) { + printf STDERR "# warning chromStart over size $ucscChr $chromStart $chromEnd\n"; + $chromStart = $chromSize-1; + } + if ($chromEnd > $chromSize) { + my $overRun = $chromEnd - $chromSize; + printf STDERR "# warning chromEnd over size by $overRun -> $ucscChr $chromStart $chromEnd\n"; + $chromEnd = $chromSize; + } + my $id="notFound"; + my $name="notFound"; + my $parent="notFound"; + my @b = split(';', $a[8]); + for (my $i = 0; $i < scalar(@b); ++$i) { + my ($tag, $value) = split('=', $b[$i]); + if ($tag eq "ID") { + $id = $value; + if ($id !~ m/-/) { + if (exists($parents{$id})) { + printf STDERR "# WARN: duplicate parent: $id"; + } else { + $parents{$id} = $ucscChr; + } + } + } elsif ($tag eq "Parent") { + $parent = $value; + } elsif ($tag eq "Name") { + $name = $value; + } + } + my $type="notFound"; + my $insertType = $a[2]; + if ($insertType =~ m/clone_insert_start/) { + $type = "start"; + if ($parent eq "notFound") { + printf STDERR "# ERR: can not find parent for start $name Ttype $id\n"; + } else { + if (!exists($parents{$parent})) { + printf STDERR "# ERR: start found $name with no parent $parent declared\n"; + } elsif (exists($starts{$parent})) { + printf STDERR "# ERR: duplicate start for $parent\n"; + } elsif ($ucscChr eq $parents{$parent}) { + $starts{$parent} = sprintf("%s\t%s", $chromStart, $chromEnd); + } else { + printf STDERR "# ERR: start on different chrom $ucscChr than parent $parent $parents{$parent}\n"; + } + } + } elsif ($insertType =~ m/clone_insert_end/) { + $type = "end"; + if ($parent eq "notFound") { + printf STDERR "# ERR: can not find parent for end $name Ttype $id\n"; + } else { + if (!exists($parents{$parent})) { + printf STDERR "# ERR: end found $name with no parent $parent declared\n"; + } elsif (exists($ends{$parent})) { + printf STDERR "# ERR: duplicate end for $parent\n"; + } elsif ($ucscChr eq $parents{$parent}) { + $ends{$parent} = sprintf("%s\t%s", $chromStart, $chromEnd); + } else { + printf STDERR "# ERR: end on different chrom $ucscChr than parent $parent $parents{$parent}\n"; + } + } + } elsif ($insertType =~ m/clone_insert/) { + $type = "insert"; + $endNames{$id} = $name; + } + $name =~ s/gi\|//g; + $id =~ s/gi\|//g; + printf STDERR "%s\t%d\t%d\t%s_%s_%s\t0\t%s\n", $ucscChr, $chromStart, $chromEnd, $name, $type, $id, $a[6]; +} # while (my $line = <FH>) + +close (FH); + +foreach my $parent (keys %parents) { + if (! exists($starts{$parent}) ) { + printf STDERR "# ERR: no start for $parent\n"; + } elsif (! exists($ends{$parent}) ) { + printf STDERR "# ERR: no end for $parent\n"; + } else { + my $strand = "+"; + my $chrStart = 0; + my $chrEnd = 0; + my $blockStart = 0; + my ($sStart, $sEnd) = split('\t', $starts{$parent}); + my ($eStart, $eEnd) = split('\t', $ends{$parent}); + my $startSize = $sEnd - $sStart; + my $endSize = $eEnd - $eStart; + if ($eStart < $sStart) { + $chrStart = $eStart; + $chrEnd = $sEnd; + $blockStart = $sStart - $chrStart; + $strand = "-"; + $startSize = $eEnd - $eStart; + $endSize = $sEnd - $sStart; + } else { + $chrStart = $sStart; + $chrEnd = $eEnd; + $blockStart = $eStart - $chrStart; + } + if ($startSize > $blockStart) { + printf STDERR "# startSize > blockStart $endNames{$parent}\n"; + } else { + printf "%s\t%d\t%d\t%s\t0\t%s\t%d\t%d\t0\t2\t%d,%d\t0,%d\n", $parents{$parent}, $chrStart, $chrEnd, $endNames{$parent}, $strand, $chrStart, $chrEnd, $startSize, $endSize, $blockStart; + } + } +} +} +'_EOF_' + # << happy emacs + + chmod +x hg38.pl + + # process GFF files into bed files into separateLibs/ directory +for L in `cat library.list.txt` +do + export destDir="separateLibs/${L}" + echo "working: ${L}" 1>&1 + mkdir -p "${destDir}" + ./hg38.pl reports/${L}.GCF_000001405.26.106.*.gff \ + 2> ${destDir}/tmp.bed6 | sort -k1,1 -k2,2n > ${destDir}/hg38.${L}.bed + sort -k1,1 -k2,2n ${destDir}/tmp.bed6 > ${destDir}/hg38.${L}.items.bed6 +done + + # use only those libraries with more than 20,000 clone ends + wc -l separateLibs/*/*.bed | sort -n | grep -v total | awk '$1 > 20000' \ + | sed -e 's#.*separateLibs/##; s#/.*##' > libs.over20K.list + + # note those libraries with less than 20,000 clone ends + wc -l separateLibs/*/*.bed | grep -v total | awk '$1 < 20000' | sed -e 's#.*separateLibs/##; s#/.*##' > libs.under20K.list + + # filter out bad ends, length must be <= median size times three + cat libs.over20K.list | while read D +do + if [ ! -s separateLibs/${D}/lengths.txt ]; then + awk '{print $3-$2}' separateLibs/${D}/hg38.${D}.bed \ + > separateLibs/${D}/lengths.txt + fi + median3X=`ave separateLibs/${D}/lengths.txt | grep median | awk '{printf "%d", $2*3}'` + awk '($3-$2) < '$median3X'' separateLibs/${D}/hg38.${D}.bed > separateLibs/${D}/hg38.median3X.bed + awk '($3-$2) >= '$median3X'' separateLibs/${D}/hg38.${D}.bed > separateLibs/${D}/hg38.badMap.bed + before=`cat separateLibs/${D}/hg38.${D}.bed | wc -l` + after=`cat separateLibs/${D}/hg38.median3X.bed | wc -l` + dropped=`echo $before $after | awk '{print $1-$2}'` + perCent=`echo $dropped $before | awk '{printf "%.2f", 100*'$dropped/$before'}'` + echo "$D $before - $after = $dropped -> % $perCent dropped" +done + +# ABC20 24692 - 24474 = 218 -> % 0.88 dropped +# RP11 86660 - 85903 = 757 -> % 0.87 dropped +# CTD 95853 - 94941 = 912 -> % 0.95 dropped +# CH17 105618 - 105060 = 558 -> % 0.53 dropped +# ABC21 182154 - 180973 = 1181 -> % 0.65 dropped +# ABC22 189939 - 188743 = 1196 -> % 0.63 dropped +# COR02 208263 - 206782 = 1481 -> % 0.71 dropped +# ABC18 325080 - 322904 = 2176 -> % 0.67 dropped +# ABC27 334178 - 331822 = 2356 -> % 0.71 dropped +# ABC24 398944 - 395776 = 3168 -> % 0.79 dropped +# ABC23 436965 - 433896 = 3069 -> % 0.70 dropped +# ABC16 452220 - 449101 = 3119 -> % 0.69 dropped +# COR2A 583008 - 578578 = 4430 -> % 0.76 dropped +# WI2 587165 - 582843 = 4322 -> % 0.74 dropped +# ABC7 649297 - 644071 = 5226 -> % 0.80 dropped +# ABC11 729962 - 724864 = 5098 -> % 0.70 dropped +# ABC9 755994 - 750648 = 5346 -> % 0.71 dropped +# ABC12 777816 - 771827 = 5989 -> % 0.77 dropped +# ABC10 787969 - 781331 = 6638 -> % 0.84 dropped +# ABC13 810822 - 803589 = 7233 -> % 0.89 dropped +# ABC14 845573 - 839126 = 6447 -> % 0.76 dropped +# ABC8 1204275 - 1192784 = 11491 -> % 0.95 dropped + + # loading the median3X files +for L in `cat libs.over20K.list` +do + echo $L 1>&2 + hgLoadBed -type=bed12 hg38 cloneEnd${L} \ + separateLibs/${L}/hg38.median3X.bed \ + > separateLibs/loadBed.${L}.log 2>&1 +done + + # loading the dropped ends: + mkdir /hive/data/genomes/hg38/bed/cloneEnds/droppedTooBig + # link them to here + cat ../libs.over20K.list | while read L +do + ln -s ../separateLibs/${L}/hg38.badMap.bed ${L}.badMap.bed +done + # then load + hgLoadBed -type=bed12 hg38 cloneEndbadEnds *.badMap.bed + + # construct multiple mapped ends: +for L in `cat libs.over20K.list` +do + cat separateLibs/${L}/hg38.median3X.bed +done | sort -k4 > allEnds.bed + + cut -f4 allEnds.bed | sort | uniq -c | sort -rn > allEnds.names.count.txt + + awk '$1 > 1' allEnds.names.count.txt | awk '{print $2}' \ + | sort > multiples.names.txt + + join -t' ' -o "2.1,2.2,2.3,2.4,2.5,2.6,2.7,2.8,2.9,2.10,2.11,2.12" \ + -2 4 multiples.names.txt allEnds.bed | sort -k1,1 -k2,2n \ + > allEnds.multiple.locations.bed + + hgLoadBed -type=bed12 hg38 cloneEndmultipleMaps \ + allEnds.multiple.locations.bed > load.multipleMaps.log 2>&1 + + awk '$6 == "+"' allEnds.bed | sort -k1,1 -k2,2n \ + | bedItemOverlapCount hg38 stdin > allEnds.forward.bedGraph + + awk '$6 == "-"' allEnds.bed | sort -k1,1 -k2,2n \ + | bedItemOverlapCount hg38 stdin > allEnds.reverse.bedGraph + + bedGraphToBigWig allEnds.forward.bedGraph \ + /hive/data/genomes/hg38/chrom.sizes \ + cloneEndcoverageForward.bw + + bedGraphToBigWig allEnds.reverse.bedGraph \ + /hive/data/genomes/hg38/chrom.sizes \ + cloneEndcoverageReverse.bw + + mkdir /gbdb/hg38/bbi/cloneEnd + ln -s `pwd`/cloneEndcoverageForward.bw /gbdb/hg38/bbi/cloneEnd + ln -s `pwd`/cloneEndcoverageReverse.bw /gbdb/hg38/bbi/cloneEnd + + hgBbiDbLink hg38 cloneEndcoverageForward \ + /gbdb/hg38/bbi/cloneEnd/cloneEndcoverageForward.bw + hgBbiDbLink hg38 cloneEndcoverageReverse \ + /gbdb/hg38/bbi/cloneEnd/cloneEndcoverageReverse.bw + + ### Fixup the scores to indicate how many multiple mappings as mentioned + ### in the hg19 bacEnds description page: one mapping: score = 1000 + ### multiple mappings: score = 1500/count + ### the sort | uniq -c | awk does this score calculation with the name + ### in column 1 + ### The join puts the existing table together with those scores + ### DONE - 2015-06-18 - Hiram + + mkdir /hive/data/genomes/hg38/bed/cloneEnds/addCounts + cd /hive/data/genomes/hg38/bed/cloneEnds/addCounts + mkdir score withScore noScore withScore + for table in cloneEndABC10 cloneEndABC11 cloneEndABC12 cloneEndABC13 \ +cloneEndABC14 cloneEndABC16 cloneEndABC18 cloneEndABC20 cloneEndABC21 \ +cloneEndABC22 cloneEndABC23 cloneEndABC24 cloneEndABC27 cloneEndABC7 \ +cloneEndABC8 cloneEndABC9 cloneEndCH17 cloneEndCOR02 cloneEndCOR2A \ +cloneEndCTD cloneEndRP11 cloneEndWI2 cloneEndbadEnds cloneEndmultipleMaps +do + hgsql -N -e "select name from $table;" hg38 | sort | uniq -c | + awk '{ if (1 == $1) {printf "%s\t1000\n", $2} else {printf "%s\t%d\n", $2, 1500/$1} }' \ + | sort > score/hg38.$table.score.tab + hgsql -N -e "select * from $table order by name;" hg38 \ + | sort -k5 > noScore/hg38.$table.tab + join -t'^I' -1 5 noScore/hg38.$table.tab score/hg38.$table.score.tab \ + | awk '{printf "%d\t%s\t%d\t%d\t%s\t%d\t%s\t%d\t%d\t%d\t%d\t%s\t%s\n", $2,$3,$4,$5,$1,$14,$7,$8,$9,$10,$11,$12,$13}' \ + | sort -k2,2 -k3,3n > withScore/hg38.$table.withScore.tab + hgsql -e "delete from $table;" hg38 + hgsql -e "load data local infile \"withScore/hg38.$table.withScore.tab\" into table $table;" hg38 +done + +############################################################################## +# SIB Transcriptome (DONE 2014-08-27 Steve) + + # Create working directory and download data from where Christian + # Iseli (christian.iseli@unil.ch) put it, and unpack. + mkdir -p /hive/data/genomes/hg38/bed/sibTranscriptome + cd /hive/data/genomes/hg38/bed/sibTranscriptome + wget --timestamping http://ludwig-sun1.unil.ch/~chris/HTr.gtf.gz + wget --timestamping http://ludwig-sun1.unil.ch/~chris/txg.tar.gz + + tar -zxvf txg.tar.gz + + zcat HTr.gtf.gz | ldHgGene hg38 sibGene stdin + # Reading stdin + # Read 208508 transcripts in 2824960 lines in 1 files + # 208508 groups 25 seqs 1 sources 2 feature types + # 208508 gene predictions + + # Do a little data cleanup and transformation and load splice graphs + # into database. + sed 's/altGraphX/sibTxGraph/' ~/kent/src/hg/lib/altGraphX.sql > sibTxGraph.sql + cat txg/*.txg | txgToAgx stdin stdout | hgLoadBed -notItemRgb \ + -sqlTable=sibTxGraph.sql hg38 sibTxGraph stdin + # Reading stdin + # Read 47817 elements of size 18 from stdin + # Sorted + # Creating table definition for sibTxGraph from sql: sibTxGraph.sql + # Saving bed.tab + # Loading hg38 + + # Create sibAltEvents track for analyzed alt-splices. + # Not on RR for hg18 and hg19, so do not push it out + cat txg/*.txg | txgAnalyze stdin /cluster/data/hg38/hg38.2bit sibAltEvents.bed + awk '$2 >= 0' sibAltEvents.bed | sort | uniq > foo.bed + hgLoadBed hg38 sibAltEvents foo.bed + # Reading foo.bed + # Read 452436 elements of size 6 from foo.bed + # Sorted + # Creating table definition for sibAltEvents, bedSize: 6 + # Saving bed.tab + # Loading hg38 + + # push sibGene and sibTxGraph for hg38 + +############################################################################ +# Orangutan Lastz run (DONE - 2014-05-27 - Hiram) + screen -S hg38PonAbe2 # use a screen to manage this longish running job + mkdir /hive/data/genomes/hg38/bed/lastzPonAbe2.2014-09-02 + cd /hive/data/genomes/hg38/bed/lastzPonAbe2.2014-09-02 + + # always set the BLASTZ program so we know what version was used + cat << '_EOF_' > DEF +# human vs chimp +BLASTZ=/cluster/bin/penn/lastz-distrib-1.03.52/bin/lastz +BLASTZ_O=600 +BLASTZ_E=150 +# maximum M allowed with lastz is only 254 +BLASTZ_M=254 + +BLASTZ_T=2 +BLASTZ_Y=15000 +BLASTZ_K=4500 +BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q +# A C G T +# 90 -330 -236 -356 +# -330 100 -318 -236 +# -236 -318 100 -330 +# -356 -236 -330 90 + +# TARGET: Human Hg38 +SEQ1_DIR=/scratch/data/hg38/hg38.2bit +SEQ1_LEN=/scratch/data/hg38/chrom.sizes +SEQ1_CHUNK=20000000 +SEQ1_LAP=10000 +SEQ1_IN_CONTIGS=0 + +# QUERY: Orangutan PonAbe2 +SEQ2_DIR=/hive/data/genomes/ponAbe2/ponAbe2.2bit +SEQ2_LEN=/hive/data/genomes/ponAbe2/chrom.sizes +SEQ2_CHUNK=10000000 +SEQ2_LAP=0 +SEQ2_LIMIT=100 +SEQ2_IN_CONTIGS=0 + +BASE=/hive/data/genomes/hg38/bed/lastzPonAbe2.2014-09-02 +TMPDIR=/dev/shm +'_EOF_' + + time (doBlastzChainNet.pl `pwd`/DEF -verbose=2 \ + -chainMinScore=5000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -syntenicNet) > do.log 2>&1 + # real 144m46.575s + cat fb.hg38.chainPonAbe2Link.txt + # 2719618310 bases of 3049335806 (89.187%) in intersection + + # filter with doRecipBest.pl + time (doRecipBest.pl -workhorse=hgwdev -buildDir=`pwd` \ + hg38 ponAbe2) > rbest.log 2>&1 + # real 60m1.060s + time (doRecipBest.pl -load -continue=load -workhorse=hgwdev \ + -buildDir=`pwd` hg38 ponAbe2) > loadRBest.log 2>&1 & + # real 3m35.834s + + cat fb.hg38.chainRBestPonAbe2Link.txt + # 2538296592 bases of 3049335806 (83.241%) in intersection + + # running the swap + mkdir /hive/data/genomes/ponAbe2/bed/blastz.hg38.swap + cd /hive/data/genomes/ponAbe2/bed/blastz.hg38.swap + time (doBlastzChainNet.pl -verbose=2 \ + -swap /hive/data/genomes/hg38/bed/lastzPonAbe2.2014-09-02/DEF \ + -chainMinScore=5000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -syntenicNet) > swap.log 2>&1 + # real 102m27.866s + cat fb.ponAbe2.chainHg38Link.txt + # 2773568958 bases of 3093572278 (89.656%) in intersection + + time (doRecipBest.pl -workhorse=hgwdev -buildDir=`pwd` \ + ponAbe2 hg38) > rbest.log 2>&1 + # real 78m47.312s + + + + +############################################################################# +# Add chrX alts to par (DONE 2014-10-14 angie) +# Thanks to Hiram for pointing out that intersecting chrX positions in +# altLocations and par shows whether a chrX alt overlaps a PAR. + cd /hive/data/genomes/hg38/bed/par + hgsql hg38 -e 'select * from altLocations where chrom = "chrX"' +#+-----+-------+------------+----------+---------------------+ +#| bin | chrom | chromStart | chromEnd | name | +#+-----+-------+------------+----------+---------------------+ +#| 73 | chrX | 319337 | 601516 | chrX_KI270880v1_alt | +#| 73 | chrX | 326487 | 601516 | chrX_KI270913v1_alt | +#| 149 | chrX | 79965153 | 80097082 | chrX_KI270881v1_alt | +#+-----+-------+------------+----------+---------------------+ + hgsql hg38 -e 'select * from par where chrom = "chrX"' +#+-----+-------+------------+-----------+------+ +#| bin | chrom | chromStart | chromEnd | name | +#+-----+-------+------------+-----------+------+ +#| 9 | chrX | 10000 | 2781479 | PAR1 | +#| 221 | chrX | 155701382 | 156030895 | PAR2 | +#+-----+-------+------------+-----------+------+ + # chrX_KI270880v1_alt and chrX_KI270913v1_alt are entirely contained in PAR1; + # chrX_KI270881v1_alt is not in either PAR. + hgsql hg38 -e 'select chrom,size from chromInfo \ + where chrom in ("chrX_KI270880v1_alt", "chrX_KI270913v1_alt");' +#+---------------------+--------+ +#| chrom | size | +#+---------------------+--------+ +#| chrX_KI270880v1_alt | 284869 | +#| chrX_KI270913v1_alt | 274009 | +#+---------------------+--------+ + # Process that into bed4 with name=PAR1: + hgsql hg38 -NBe 'select chrom, 0, size, "PAR1" from chromInfo \ + where chrom in ("chrX_KI270880v1_alt", "chrX_KI270913v1_alt");' \ + >> hg38Par.bed4 + hgLoadBed hg38 par hg38Par.bed4 + checkTableCoords hg38 par + + +############################################################################# +# LASTZ Cow bosTau8 (DONE - 2014-10-15 - Steve) + mkdir /hive/data/genomes/hg38/bed/lastzBosTau8.2014-10-215 + cd /hive/data/genomes/hg38/bed/lastzBosTau8.2014-10-15 + + cat << '_EOF_' > DEF +# human vs cow +# maximum M allowed with lastz is only 254 +BLASTZ_M=254 + +# TARGET: Human hg38 +SEQ1_DIR=/scratch/data/hg38/hg38.2bit +SEQ1_LEN=/scratch/data/hg38/chrom.sizes +SEQ1_CHUNK=10000000 +SEQ1_LAP=10000 + +# QUERY: Cow bosTau8 +SEQ2_DIR=/hive/data/genomes/bosTau8/bosTau8.2bit +SEQ2_LEN=/hive/data/genomes/bosTau8/chrom.sizes +SEQ2_CHUNK=10000000 +SEQ2_LAP=0 + + +BASE=/hive/data/genomes/hg38/bed/lastzBosTau8.2014-10-15 +TMPDIR=/scratch/tmp +'_EOF_' + # << happy emacs + time nice -n +19 doBlastzChainNet.pl -verbose=2 \ + `pwd`/DEF \ + -syntenicNet \ + -noLoadChainSplit \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -chainMinScore=3000 -chainLinearGap=medium > do.log 2>&1 + # real 602m37.523s + cat fb.hg38.chainBosTau8Link.txt + # 1401921010 bases of 3049335806 (45.975%) in intersection + # Create link + cd /hive/data/genomes/hg38/bed + ln -s lastzBosTau8.2014-10-15 lastz.bosTau8 + + # running the swap + mkdir /hive/data/genomes/bosTau8/bed/blastz.hg38.swap + cd /hive/data/genomes/bosTau8/bed/blastz.hg38.swap + time nice -n +19 doBlastzChainNet.pl -verbose=2 \ + /hive/data/genomes/hg38/bed/lastzBosTau8.2014-10-15/DEF \ + -swap -syntenicNet \ + -noLoadChainSplit \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 + # real 116m32.121s + cat fb.bosTau8.chainHg38Link.txt + # 1336307377 bases of 2649307237 (50.440%) in intersection + cd /hive/data/genomes/bosTau8/bed + ln -s blastz.hg38.swap lastz.hg38 + +############################################################################ +# NCBI ClinVar (new version -DONE - 2014-11-08 - Max) +# see hg19.txt +######################################################################### + +######################################################################## +# CNV Developmental Delay track (2014-11-21 Steve) + + mkdir /hive/data/genomes/hg38/bed/cnvDevDelay + cd /hive/data/genomes/hg38/bed/cnvDevDelay + +wget --timestamping 'ftp://ftp.ncbi.nlm.nih.gov/pub/dbVar/data/Homo_sapiens/by_study/nstd100_Coe_et_al_2014/gvf/nstd100_Coe_et_al_2014.GRCh38.remap.all.germline.ucsc.gvf.gz' +wget --timestamping 'ftp://ftp.ncbi.nlm.nih.gov/pub/dbVar/data/Homo_sapiens/by_study/nstd54_Cooper_et_al_2011/gvf/nstd54_Cooper_et_al_2011.GRCh38.remap.all.germline.ucsc.gvf.gz' + +cp /kent/src/hg/utils/automation/gvfToBed8Attrs.pl . +mv gvfToBed8Attrs.pl gvfToBed8AttrsCase.pl +cp gvfToBed8AttrsCase.pl gvfToBed8AttrsControl100.pl +cp gvfToBed8AttrsCase.pl gvfToBed8AttrsControl54.pl + +# made three local copies of Angie's gvf conversion script - one to include +# only case individuals from nstd100, one to include only control individuals +# from nstd100 and one to include only control individuals from nstd54 + +# had to add an additional elsif statement to the nstd100 scripts to filter +# based on sample_name field: + +# } elsif ($tag eq "sample_name") { +# $sample_name = $val; +# } + +# added line 33/35 to each file: + +# next if ($sample_name eq "Unknown"); # keep only "case" individuals from nstd100 +# next if ($sample_name ne "Unknown"); # keep only "control" individuals from nstd100 +# next if ($phenotype ne "not_reported"); # keep only "control" individuals from nstd54 + +zcat nstd100_Coe_et_al_2014.GRCh38.remap.all.germline.ucsc.gvf.gz | gvfToBed8AttrsCase.pl > cnvDevDelayAllCase.bed +zcat nstd100_Coe_et_al_2014.GRCh38.remap.all.germline.ucsc.gvf.gz | gvfToBed8AttrsControl100.pl > cnvDevDelayAllControl.bed +zcat nstd54_Cooper_et_al_2011.GRCh38.remap.all.germline.ucsc.gvf.gz | gvfToBed8AttrsControl54.pl >> cnvDevDelayAllControl.bed + +# GRCh38 data from dbVar had different naming scheme for alternate chromosomes +# (e.g., chr1|NT_187515.1 instead of chr1_KI270762v1_alt), so needed to write +# a script to substitute the correct UCSC names + + cat << '_EOF_' > chromXref.pl +#!/usr/bin/env perl + +use strict; +use warnings; + +sub usage() { + printf STDERR "usage: ./chromXref.pl <infile> <outfile>\n" +} + +my $argc = scalar(@ARGV); + +if ($argc != 2) { + usage; + exit 255; +} + +open (file1, "<hg38.xref") or die "cannot read hg38.xref"; + +my @accArray = (); +my $i = 0; +while (my $line = <file1>) { + chomp($line); + my ($type, $chr, $acc1, $acc2) = split('\t', $line); + ($type, undef) = split('-', $type); + ($acc1, my $version) = split('\.', $acc1); + if ($type eq "unlocalized") { + $type = "random"; + } + my $ucscAcc = "_" . $acc1 . "v" . $version . "_" . $type; + $accArray[$i][0] = $ucscAcc; + $accArray[$i][1] = $acc2; + $i++; +} + +close (file1); + +open (file2, "<$ARGV[0]") or die "cannot read $ARGV[0]"; +open (file3, ">$ARGV[1]") or die "cannot read $ARGV[1]"; +local $/; +my $fileContents = <file2>; +for ($i = 0; $i < scalar(@accArray); $i++) { + my $temp1 = $accArray[$i][1]; + my $temp2 = $accArray[$i][0]; + if ($fileContents =~ m/\|$temp1/) { + $fileContents =~ s/\|$temp1/$temp2/g; + } +} + +print file3 $fileContents; +close (file2); +close (file3); +'_EOF_' + # << happy emacs + +cp /hive/data/genomes/hg38/genbank/GCF_000001405.26.assembly.txt . + +cat GCF_000001405.26.assembly.txt | grep -v '^#\|assembled\|unplaced' | awk '{print $2 "\t" $3 "\t" $5 "\t" $7}' > hg38.xref + +chromXref.pl cnvDevDelayAllCase.bed cnvDevDelayAllCaseUcsc.bed +chromXref.pl cnvDevDelayAllControl.bed cnvDevDelayAllControlUcsc.bed + +hgLoadBed -tab -renameSqlTable -sqlTable=$HOME/kent/src/hg/lib/bed8Attrs.sql \ + -allowStartEqualEnd hg38 cnvDevDelayCase cnvDevDelayAllCaseUcsc.bed + +hgLoadBed -tab -renameSqlTable -sqlTable=$HOME/kent/src/hg/lib/bed8Attrs.sql \ + -allowStartEqualEnd hg38 cnvDevDelayControl cnvDevDelayAllControlUcsc.bed + + checkTableCoords hg38 cnvDevDelayCase + checkTableCoords hg38 cnvDevDelayControl + + +######################################################################### +# RETROFINDER RETROPOSED GENES ucscRetro track VERSION 9 +# (2015-01-12 - 2015-01-20, hartera, DONE) +ssh hgwdev +mkdir -p /hive/groups/gencode/pseudogenes/retroFinder/hg38.20150112 +cd /hive/groups/gencode/pseudogenes/retroFinder/hg38.20150112 + +cat << '_EOF_' > DEF + +RETRO_OPTIONS="-verbose=4 -minAli=0.98 -nearTop=0.005 " +VERSION=9 +RUNDATE="2015-01-12" +DB=hg38 +SCORETHRESH=550 +GENOMENAME='Homo sapiens' +GBDB=hg +DATE=20150112 +RUNDIR=/hive/groups/gencode/pseudogenes/retroFinder/$DB.$DATE +BINDIR=/hive/users/hartera/GencodeWG/retroFinder/branches/version2/bin +KENTDIR=/cluster/home/hartera/kent +KENTBINDIR=/cluster/home/hartera/bin/x86_64 +MRNABASE=/hive/data/genomes/$DB/bed/mrnaBlastz.$VERSION +TMPMRNA=$RUNDIR/mrnaBlastz/$DB +TMPEST=$RUNDIR/est/$DB +USEALTSEQS=0 +EST=all_est +SPLICED_EST=intronEst +SPLIT_EST=0 +SPLIT_SPLICED_EST=0 +LASTZPROG=/cluster/bin/penn/x86_64/lastz +SCRIPT=/hive/users/hartera/GencodeWG/retroFinder/branches/version2/src/pipeline +GENOME=/hive/data/genomes +RETRODIR=$GENOME/$DB/bed/retro +BASE=$RUNDIR/retro +OUTDIR=${BASE}/version${VERSION}/${DB} +RESULT=$OUTDIR/result +RESULTSPLIT=$OUTDIR/resultSplit +LOG=$OUTDIR/log +OUT=$OUTDIR/out +OVERLAPDIR=$OUTDIR/run.o +TABLE=ucscRetroInfo$VERSION +ORTHOTABLE=ucscRetroOrtho$VERSION +ALIGN=ucscRetroAli$VERSION +LOCAL=/scratch/data/$DB +TWOBIT=$GENOME/$DB/$DB.2bit +RMSK=rmsk +NET1=netMm10 +NET2=netCanFam3 +NET3=netRheMac3 +# these two nets determine which retros are classified as ancient, +# use two farthest nets +ANCIENT1=netMm10 +ANCIENT2=netCanFam3 +GENE1=knownGene +GENE2=refGene +GENE3=wgEncodeGencodeCompV19 +CLUSTER=ku +SPECIES="hg38 mm10" +ROOTDIR="/cluster/home/hartera/public_html/retro/hg38Jun14" +WEBROOT=$ROOTDIR/retro.$VERSION +WEBSERVER=http://hgwdev-hartera.soe.ucsc.edu +SHUFFLEDIR=shuffle +SHUFFLEROOT=$WEBROOT/$SHUFFLEDIR +DUPDIR=dups +DUPROOT=$WEBROOT/$DUPDIR +AGEDIR=age +AGEROOT=$WEBROOT/$AGEDIR +EXPDIR=exp +GENEPFAM=knownGene +PFAM=knownToPfam +PFAMIDFIELD=name +PFAMDOMAIN=value +ALTSPICE= +#ALTSPLICE=sibTxGraph +SPLITBYAGE=$SCRIPT/splitRetrosByAge +PDB=proteins140122 +#ARRAY=gnfAtlas2 +#AFFYPROBE="affyU133A,affyGnf1h" +#ARRAYMEDIAN=hgFixed.gnfHumanAtlas2Median +#ARRAYRATIO=hgFixed.gnfHumanAtlas2AllRatio +#ARRAYABS=hgFixed.gnfHumanAtlas2All +#ARRAYEXP=hgFixed.gnfHumanAtlas2MedianExps +#ARRAYEXPALL=hgFixed.gnfHumanAtlas2AllExps +#ARRAYLOOKUP=knownToGnfAtlas2 +#ARRAYPSLS="/hive/data/genomes/hg19/bed/geneAtlas2/affyU133A.psl /hive/data/genomes/hg19/bed/geneAtlas2/affyGnf1h.psl" +'_EOF_' + # << happy emacs +chmod +x DEF + +mkdir -p /hive/data/genomes/hg38/bed/retro +mkdir -p /hive/data/genomes/hg38/bed/mrnaBlastz.9 +mkdir -p /hive/groups/gencode/pseudogenes/retroFinder/hg38.20150112/mrnaBlastz +cd /hive/groups/gencode/pseudogenes/retroFinder/hg38.20150112/mrnaBlastz +cp ../DEF . + +# Create S1.len file +rom.sizes without random chroms or chrM, there are many alt loci also +# in hg38 that were not in hg19 so 285 chroms total. +cat /hive/data/genomes/hg38/chrom.sizes | grep -v random \ + | grep -v chrUn | grep -v chrM > S1.len +cp S1.len /hive/data/genomes/hg38/bed/mrnaBlastz.9 + +screen +# Run steps 1 to 5 of RetroFinder pipeline from scripts in CCDS SVN source tree: +retroFinder/branches/version2/src/pipeline/ucscStep1.sh DEF +# check cluster jobs on ku +retroFinder/branches/version2/src/pipeline/ucscStep2.sh DEF +retroFinder/branches/version2/src/pipeline/ucscStep3.sh DEF +#check cluster jobs on ku +retroFinder/branches/version2/src/pipeline/ucscStep4.sh DEF +#check cluster jobs on ku + # Load the track +retroFinder/branches/version2/src/pipeline/ucscStep5.sh DEF +cd /hive/groups/gencode/pseudogenes/retroFinder/hg38.20150112/retro/version9/hg38 +retroFinder/branches/version2/src/pipeline/filterMrna.sh +retroFinder/branches/version2/src/pipeline/filterEst.sh +# Check cluster jobs on ku +retroFinder/branches/version2/src/pipeline/analyseExpress.sh +# Check cluster jobs on ku +#added ucscRetroAli9 to kent/src/hg/makeDb/human/hg38/trackDb.ra +# copied +# /hive/groups/gencode/pseudogenes/retroFinder/hg38.20150112/retro/version9/hg38/trackDb.retro +# entry to kent/src/hg/makeDb/trackDb/human/hg38/trackDb.ra and edited it to +# remove the full date and add: +# dataVersion Jan. 2015 +# Scripts copied ucscRetroAli9.psl, ucscRetroInfo9.bed and ucscRetroCds9.tab +# to /hive/data/genomes/hg38/bed/retro/ + +########## +# Make dbVar chrom to UCSC chrom lift file +# DONE braney 2/12/15 +cd /cluster/data/hg38/jkStuff +sort /cluster/data/hg38/chrom.sizes > tmpChrom +grep -v '^#\|assembled' /hive/data/genomes/hg38/genbank/GCF_000001405.26.assembly.txt | awk 'BEGIN {OFS="\t"} {print "chr" $3 "_" $5 "_" $2, "chr" $3 "|"$7}' | sed 's/-scaffold//' | sed 's/unlocalized/random/' | sed 's/_unplaced//' | sed 's/chrna/chrUn/g' | sed 's/\./v/' | sort | join /dev/stdin tmpChrom | awk 'BEGIN {OFS="\t"} {print 0, $2, $3, $1, $3}' > dbVar.lift +awk 'BEGIN {OFS="\t"} {print 0, $1, $2, $1, $2}' /cluster/data/hg38/chrom.sizes >> dbVar.lift +rm tmpChrom + +######################################################################### +# UCSC to RefSeq name correspondence (DONE - 2015-04-13 - Hiram) + + mkdir /hive/data/genomes/hg38/bed/ucscToRefSeq + cd /hive/data/genomes/hg38/bed/ucscToRefSeq + + # columns 5 and 7 are the INSDC and RefSeq names + + grep -v "^#" ../../genbank/GCF_000001405.26.assembly.txt \ + | awk -F'\t' '{printf "%s\t%s\n", $5,$7}' | sort > insdc.refSeq.tab + + hgsql -N -e 'select name,chrom,chromStart,chromEnd from ucscToINSDC;' hg38 \ + | sort > insdc.ucsc.tab + + join insdc.ucsc.tab insdc.refSeq.tab | tr '[ ]' '[\t]' \ + | cut -f2- > ucsc.refSeq.tab + + + export chrSize=`cut -f1 ucsc.refSeq.tab | awk '{print length($0)}' | sort -n | tail -1` + sed -e "s/21/$chrSize/" $HOME/kent/src/hg/lib/ucscToINSDC.sql \ + | sed -e 's/INSDC/RefSeq/g;' > ucscToRefSeq.sql + hgLoadSqlTab hg38 ucscToRefSeq ./ucscToRefSeq.sql ucsc.refSeq.tab + + checkTableCoords hg38 -table=ucscToRefSeq + +######################################################################### +#CREATE MICROSAT TRACK (DONE - 2015-05-22 - Hiram) + ssh hgwdev + mkdir /cluster/data/hg38/bed/microsat + cd /cluster/data/hg38/bed/microsat + + awk '($5==2 || $5==3) && $6 >= 15 && $8 == 100 && $9 == 0 {printf("%s\t%s\t%s\t%dx%s\n", $1, $2, $3, $6, $16);}' \ + ../simpleRepeat/simpleRepeat.bed > microsat.bed + + hgLoadBed hg38 microsat microsat.bed + +############################################################################# +# ENCODE Regulatory tracks (Kate & Chris) + +# see reg.txt +######################################################################### +# GWIPS-viz Ribo-seq - (DONE - 2016-02-05 - Steve) +# contact Audrey Michel (audreymannion@gmail.com) +# redmine #16765 + +obtained bigWig file from shared Google drive +https://drive.google.com/a/soe.ucsc.edu/folderview?id=0B_xvV_5tXzOGQ1h5NEh4bnhNTDg&usp=sharing_eid + +mkdir /hive/data/genomes/hg38/bed/gwipsvizRiboseq +cp Global_RiboProElong.10_02_2016.bw /hive/data/genomes/hg38/bed/gwipsvizRiboseq/gwipsvizRiboseq.bw + +mkdir /gbdb/hg38/bbi/gwipsvizRiboseq +cd /gbdb/hg38/bbi/gwipsvizRiboseq +ln -s /hive/data/genomes/hg38/bed/gwipsvizRiboseq/gwipsvizRiboseq.bw gwipsvizRiboseq.bw + +hgsql hg38 +create table gwipsvizRiboseq select * from gc5BaseBw; +update gwipsvizRiboseq set fileName="/gbdb/hg38/bbi/gwipsvizRiboseq/gwipsvizRiboseq.bw" where fileName="/gbdb/hg38/bbi/gc5BaseBw/gc5Base.bw"; + +######################################################################### +# COSMIC v81 DONE Chris Eisenhart 2017-05-11 +# Make a new COSCMIC track for hg19 +mkdir /hive/data/outside/cosmic/hg38/v81 +cd /hive/data/outside/cosmic/hg38/v81 + +# Get the new data +sftp ceisenha@ucsc.edu@sftp-cancer.sanger.ac.uk +# Login to SFTP server then run these commands +get /files/grch38/cosmic/v81/CosmicMutantExport.tsv.gz + +# Remove the 'NS' fields, search for the \t after to exclude the E'NS'ST transcripts. +zcat CosmicMutantExport.tsv.gz | sed 's/NS\t/\t/g' > cosMut.tsv + +# Use a script to convert to bed format. +cosmicToBed cosMut.tsv cosMut.bed +# This many lines were skipped, 131597 for not having genomic coordinate + +# Sort and convert to big bed using the .as file. +sort -k1,1 -k2,2n cosMut.bed > sCosMut.bed +bedToBigBed -type=bed4+31 -as=cosmicNew.as sCosMut.bed /hive/data/genomes/hg38/chrom.sizes cosMutHg38V81.bb -tab -extraIndex=name,cosmLabel + +# Link it up so the outside world can see it. +cd /gbdb/hg38/cosmic/ +ln -s /hive/data/outside/cosmic/hg38/v81/cosMutHg38V81.bb . +######################################################################### +# hoffmanMappability hub import (2 super tracks) DONE Chris Eisenhart 2017-05-16 +mkdir /hive/data/outside/hoffmanMappability/hg38 +cd /hive/data/outside/hoffmanMappability/hg38 +wget https://www.pmgenomics.ca/hoffmanlab/proj/bismap/trackhub/hg38/trackDb.txt +# Get the trackDb file +importTrackHub trackDb.txt hofMap.ra /gbdb/hg38/hoffmanMappability/ --trackDbPath=$HOME/kent/src/hg/makeDb/trackDb/human/hg38/ --test +# Check that the commands are what we want, then run for real +importTrackHub trackDb.txt hofMap.ra /gbdb/hg38/hoffmanMappability/ --trackDbPath=$HOME/kent/src/hg/makeDb/trackDb/human/hg38/ +# View the .ra file to make sure things are ok, here changed the groups to map, +# added the alpha tags, and removed the 'show' from 'superTrack on show' +cp hofMap.ra ~/kent/src/hg/makeDb/trackDb/human/hg38 +# Include hofMap.ra in the trackDb.ra file + +# the importTrackHub failed on redirection, fetch all the files manually: +# 2017-09-15 - Hiram + +cd /hive/data/outside/hoffmanMappability/hg38 + +grep bigDataUrl trackDb.txt | awk '{print $NF}' | sed -e 's#https://www.pmgenomics.ca/hoffmanlab/proj/bismap/trackhub/hg38/##;' | while read F +do + echo $F + rm -f $F + wget --timestamping "https://www.pmgenomics.ca/hoffmanlab/proj/bismap/trackhub/hg38/${F}" +done + # real 29m40.429s + +######################################################################### +# tcgaExpr super track Chris Eisenhart, DONE, 2017-05-17 +# tcgaTranscExpr +# TCGA transcript level expression barChart track, from TOIL pipeline recompute (John Vivian) +# biorxiv.org/content/biorxiv/early/2016/07/07/062497.full.pdf +mkdir /hive/data/outside/tcgaBarcharts/ +mkdir /hive/data/outside/tcgaBarcharts/transcripts +cd /hive/data/outside/tcgaBarcharts/transcripts + +# Get all the meta data +cp ~max/projects/cirm/datasetPages/tcgaGtex/tcgaMeta.tab . +# Cut out the meta data the script wants, sample name and group. +cut -f 1,5 tcgaMeta.tab | sed 's/ /_/g' > tcgaLargeSamples.tsv + +# Get and clean the matrix +cp ~max/projects/cirm/datasetPages/tcgaGtex/tcga.tpm.tab . +# Clean up the transcript names (remove the .#) +cut -f 1 tcga.tpm.tab | cut -f 1 -d "." > tcgaTranscripts.txt +cut -f 2- tcga.tpm.tab > tcgaTpmValues.tsv +paste tcgaTranscripts.txt tcgaTpmValues.tsv > tcgaMatrix.tsv + +# Build a coordinate map +hgsql hg38 -e "select * from ensGene" | cut -f 2- | sort > ensGene +hgsql hg38 -e "select * from ensemblToGeneName" | sort > ensemblToGeneName +join ensGene ensemblToGeneName | awk '{print $2"\t"$4"\t"$5"\t"$1"\t0\t"$3"\t"$16}' > coord.bed + +# Use the meta data, matrix, and coordinate map to generate a barchart bed +time expMatrixToBarchartBed tcgaLargeSamples.tsv tcgaMatrix.tsv coord.bed tcgaTransExp.bed --groupOrder tcgaGroupOrder.txt + +# NOTE: Use the header line of the bed file to populate the barChartBars field in the trackDb. +# The order of the labels in the barChartBars field should match the order of the labels in the +# expScores column in the bed file header. + +# Sort and convert into a bigBed file. +sort -k1,1 -k2,2n tcgaTransExp.bed > sortedTcgaTransExp.bed +bedToBigBed -type=bed6+5 -as=$HOME/kent/src/hg/lib/barChartTranscExp.as sortedTcgaTransExp.bed /hive/data/genomes/hg38/chrom.sizes tcgaTransExp.bb + +# Link the files into gbdb +cd /gbdb/hgFixed/human/expMatrix +ln -s /hive/data/outside/tcgaBarcharts/transcripts/tcgaLargeSamples.tsv tcgaLargeSamples.tab +ln -s /hive/data/outside/tcgaBarcharts/transcripts/tcgaMatrix.tsv tcgaMatrix.tab +ln -s /hive/data/outside/tcgaBarcharts/transcripts/tcgaTransExp.bb . + +###########3 +# Reload bigBed with a schema that will be shared with genes track, to support +# configuration as subtracks in a composite +# (2007-08-30 kate) +cd /hive/data/outside/tcgaBarcharts/transcripts +bedToBigBed -type=bed6+5 -as=$HOME/kent/src/hg/lib/barChartGeneExpr.as sortedTcgaTransExp.bed /hive/data/genomes/hg38/chrom.sizes tcgaTranscExpr.hg38.bb +mkdir /gbdb/hg38/tcga +ln -s `pwd`/tcgaTranscExpr.hg38.bb /gbdb/hg38/tcga/tcgaTranscExpr.bb + +# TCGA gene level expression barChart track, from TOIL pipeline recompute (John Vivian) +# tcgaGeneExpr +mkdir ../genes +cd ../genes + +# Get the gene matrix. +cp ~max/projects/cirm/datasetPages/tcgaGtex/tcga.geneTpm.tab . + +# Make a coordinate file, the genes in gtexGeneModelV6 have .# versions which are +# removed with the temp fils. +hgsql hg38 -e "select * from hg38.gtexGeneModelV6" | awk '{print $3"\t"$5"\t"$6"\t"$2"\t0\t"$4"\t"$2}' > coord6+1.bed.temp +cut -f 4 coord6+1.bed.temp | cut -f 1 -d "." > foo +cut -f 1-3 coord6+1.bed.temp > foo2 +paste foo2 foo > foo3 +cut -f 5- coord6+1.bed.temp > foo4 +paste foo3 foo4 > coord6+1.bed +# This bed file didn't have the right gene names (ENS rather than Hugo), fix it. +hgsql hg38 -e "select * From knownCanonical" > foo +wc foo +cut -f 6 foo | cut -f 1 -d "." +cut -f 6 foo | cut -f 1 -d "." > foo2 +head foo +cut -f 1-3 foo > foo3 +paste foo2 foo3 > foo4 +cut -f 4- coord6+1.bed > foo5 +join <(sort foo5) <(sort foo4) | awk '{print $5"\t"$6"\t"$7"\t"$1"\t0\t"$3"\t"$4}' > coord6+1.3.bed + +# Generate the bed file, can use the same transcript file +time expMatrixToBarchartBed ../transcripts/tcgaLargeSamples.tsv tcga.geneTpm.tab coord6+1.3.bed tcgaGeneExp.bed --groupOrder=../transcripts/tcgaGroupOrder.txt + +# Convert to big bed +sort -k1,1 -k2,2n tcgaGeneExp.bed > sortedTcgaGeneExp.bed +bedToBigBed -type=bed6+5 -as=$HOME/kent/src/hg/lib/barChartGeneExp.as sortedTcgaGeneExp.bed /hive/data/genomes/hg38/chrom.sizes tcgaGeneExp.bb + +# Link to gbdb +cd /gbdb/hgFixed/human/expMatrix +ln -s /hive/data/outside/tcgaBarcharts/genes/tcgaGeneExp.bb . +ln -s /hive/data/outside/tcgaBarcharts/genes/tcga.geneTpm.tab tcgaGeneMatrix.tab + +###########3 +# Reload bigBed with a schema that will be shared with transcript track, to support +# configuration as subtracks in a composite +# Apparently Chris actually loaded the #3 file (added gene names, adjusted end coord apparently) +# (2007-08-30 kate) +cd /hive/data/outside/tcgaBarcharts/genes +bedToBigBed -type=bed6+5 -as=$HOME/kent/src/hg/lib/barChartGeneExpr.as sortedTcgaGeneExp3.bed /hive/data/genomes/hg38/chrom.sizes tcgaGeneExpr.hg38.bb +mkdir /gbdb/hg38/tcga +ln -s `pwd`/tcgaGeneExpr.hg38.bb /gbdb/hg38/tcga/tcgaGeneExpr.bb + +######################################################################### +# gtexTransExp Chris Eisenhart, done, 2017-05-23 +# TCGA transcript level RNA-seq, from TOIL pipeline recompute (John Vivian) +# biorxiv.org/content/biorxiv/early/2016/07/07/062497.full.pdf +mkdir /hive/data/outside/gtex/barChartTrack +cd /hive/data/outside/gtex/barChartTrack + +# Seems John included some TCGA data (CML) in the GTEx matrix and samples, the cleaning steps remove this. +# Make a clean sample file +cat ../johnVivianRecompute/sraToSample.txt | sed 's/ male /\tmale\t/g' | sed 's/ female /\tfemale\t/g' | cut -f 3 | sed 's/ - /-/g' | sed 's/ /_/g' > gtexSampleGroups.txt +cat ../johnVivianRecompute/sraToSample.txt | cut -f 1 -d " " > gtexSampleNames.txt +paste gtexSampleNames.txt gtexSampleGroups.txt > gtexSamples.txt +grep -v '(CML)' gtexSamples.tsv > cleanGtexSamples.tsv + +# Make a clean matrix +cut -f 1 ../johnVivianRecompute/gtex.tpm.tab | cut -f 1 -d "." > gtexTranscripts.txt +cut -f 2- ../johnVivianRecompute/gtex.tpm.tab > gtexTpmValues.tsv +paste gtexTranscripts.txt gtexTpmValues.tsv > gtexMatrix.tsv +rowsToCols gtexMatrix.tsv tspsdGtexMatrix.tsv +sort tspsdGtexMatrix.tsv > sortedTspsdGtexMatrix.tsv +grep -v '(CML)' gtexSamples.tsv | cut -f 1 | sed 's/Run_s/#transcript/g' | sort > sortedCleanGtexSamples.tsv +join sortedCleanGtexSamples.tsv sortedTspsdGtexMatrix.tsv > cleanTspsdGtexMatrix.tsv +rowsToCols cleanTspsdMatrix.tsv cleanGtexMatrix.tsv + +# Build a coordinate map +hgsql hg38 -e "select * from ensGene" | cut -f 2- | sort > ensGene +hgsql hg38 -e "select * from ensemblToGeneName" | sort > ensemblToGeneName +join ensGene ensemblToGeneName | awk '{print $2"\t"$4"\t"$5"\t"$1"\t0\t"$3"\t"$16}' > coord.bed +# NOTE: CHRISL10-05-2021 - the above ensGene steps weren't actually done or the files were removed, +# there was a coord.tsv which I used instead so the below re-run could work +tawk '{print $1,$2,$3,$4,0,$5,$6}' coord.tsv > coord.bed +# END CHRISL10-05-2021 NOTE) + +# Get the gtex ordering +hgsql hgFixed -e "select * from gtexTissue" | cut -f 3 | sed 's/ - /-/g' | sed 's/ /_/g' | sed '1D' > gtexGroupOrder.txt + +# Use the meta data, matrix, and coordinate map to generate a barchart bed +# NOTE: CHRISL10-05-2021 - re-ran this step to fix float parsing bug: +time expMatrixToBarchartBed cleanGtexSamples.tsv cleanGtexMatrix.tsv coord.bed gtexTransExp.bed --groupOrderFile gtexGroupOrder.txt + +# NOTE: Use the header line of the bed file to populate the barChartBars field in the trackDb. +# The order of the labels in the barChartBars field should match the order of the labels in the +# expScores column in the bed file header. + +# Sort and convert into a bigBed file. +sort -k1,1 -k2,2n gtexTransExp.bed > sortedGtexTransExp.bed +# NOTE: CHRISL10-05-2021 - re-ran bedToBigBed step with correct file names +bedToBigBed -as=$HOME/kent/src/hg/lib/barChartBed.as -type=bed6+5 sortedGtexTransExp.bed /hive/data/genomes/hg38/chrom.sizes gtexTranscExpr.bb + +# Link the files into gbdb +cd /gbdb/hgFixed/human/expMatrix +ln -s /hive/data/outside/gtex/barChartTrack/cleanGtexSamples.tsv cleanGtexSamples.tab +ln -s /hive/data/outside/gtex/barChartTrack/cleanGtexMatrix.tsv cleanGtexMatris.tab + +# <2007-08-30 kate) +cd /gbdb/hg38/gtex +ln -s /hive/data/outside/gtex/barChartTrack/gtexTranscExpr.bb . + +######################################################################### +# LASTZ human/hg38 vs. Zebrafish /danRer11 +# (DONE - 2017-06-12 - Chris) + + mkdir /hive/data/genomes/hg38/bed/lastzDanRer11.2017-06-12 + cd /hive/data/genomes/hg38/bed/lastzDanRer11.2017-06-12 + + printf '# human vs zebrafish danRer11 +BLASTZ=/cluster/bin/penn/lastz-distrib-1.03.66/bin/lastz +BLASTZ_M=254 + +# TARGET: human hg38 +SEQ1_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ1_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ1_CTGDIR=/hive/data/genomes/hg38/hg38.contigs.2bit +SEQ1_CTGLEN=/hive/data/genomes/hg38/hg38.contigs.chrom.sizes +SEQ1_LIFT=/hive/data/genomes/hg38/jkStuff/hg38.contigs.lift +SEQ1_CHUNK=40000000 +SEQ1_LIMIT=20 +SEQ1_LAP=10000 + +# QUERY: zebrafish danRer11 +SEQ2_DIR=/hive/data/genomes/danRer11/danRer11.2bit +SEQ2_LEN=/hive/data/genomes/danRer11/chrom.sizes +SEQ2_CHUNK=20000000 +SEQ2_LIMIT=200 +SEQ2_LAP=0 + +BASE=/hive/data/genomes/hg38/bed/lastzDanRer11.2017-06-12 +TMPDIR=/dev/shm +' > DEF + + time (doBlastzChainNet.pl `pwd`/DEF -verbose=2 \ + -chainMinScore=3000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -noDbNameCheck -syntenicNet) > do.log 2>&1 + # real 3327m39.074s + + cat fb.hg38.chainDanRer11Link.txt + # 41036733 bases of 3049335806 (1.346%) in intersection + + 973293331 bases of 3049335806 (31.918%) in intersection + + time (doRecipBest.pl -workhorse=hgwdev -buildDir=`pwd` hg38 danRer11) \ + > rbest.log 2>&1 & + + # and for the swap: + mkdir /hive/data/genomes/danRer11/bed/blastz.hg38.swap + cd /hive/data/genomes/danRer11/bed/blastz.hg38.swap + + time (doBlastzChainNet.pl -verbose=2 \ + /hive/data/genomes/hg38/bed/lastzDanRer11.2017-06-12/DEF \ + -swap -chainMinScore=3000 -chainLinearGap=medium \ + -workhorse=hgwdev -smallClusterHub=ku -bigClusterHub=ku \ + -noDbNameCheck -syntenicNet) > swap.log 2>&1 + # real 39m24.916s + + cat fb.danRer11.chainHg38Link.txt + # 47869194 bases of 1674677181 (2.858%) in intersection + + time (doRecipBest.pl -workhorse=hgwdev -buildDir=`pwd` danRer11 hg38) \ + > rbest.log 2>&1 & + # real 638m45.337s +_EOF_ +######################################################################### +# refSeqFuncElems NCBI refSeq functional elements, REDONE 2017-11-29 Angie +# previously done 2017-08-01 by Chris E + +mkdir /hive/data/genomes/hg38/bed/refSeqFuncElems.2017-11-29 +cd /hive/data/genomes/hg38/bed/refSeqFuncElems.2017-11-29 + +# NOTE FOR NEXT TIME: instead of using interim GFF, in the future these annotations might be +# folded into the same main release GFF3 from which the ncbiRefSeq* tables are extracted by +# doNcbiRefSeq.pl. +wget ftp://ftp.ncbi.nlm.nih.gov/genomes/H_sapiens/GFF_interim/interim_GRCh38.p11_top_level_2017-06-27.gff3.gz + +# Get mapping of RefSeq NC_* chromosome accs (and NT_*, NW_*) to hg38 chrom names +hgsql hg38 -NBe 'select alias, chrom from chromAlias where source = "refseq" order by alias' \ +> refSeqToChrom.tab +cut -f 2 refSeqToChrom.tab | sed -e 's/^/^/' > chrom.tab + +# Use Terence Murphy's list of feature types (and the multi-type attribute regulatory_class) +# to identify Functional Elements and swap in hg38 chrom names. +# Use subColumn -miss so it doesn't quit when it sees a patch contig that doesn't map to an +# hg38 chrom. Use grep -f chrom.tab to filter out patch contig annotations. +zcat interim_GRCh38.p11_top_level_2017-06-27.gff3.gz \ +| grep -P "(\t(CAAT_signal|GC_rich_promoter_region|TATA_box|enhancer|insulator|locus_control_region|mobile_genetic_element|origin_of_replication|promoter|protein_binding_site|recombination_feature|regulatory_region|repeat_region|sequence_feature|sequence_secondary_structure|silencer|stem_loop)\t|regulatory_class=)" \ +| subColumn -miss=/dev/null 1 stdin refSeqToChrom.tab stdout \ +| grep -f chrom.tab > funcElems.gff +wc -l funcElems.gff +#5756 funcElems.gff + +# Transform GFF to BED+ +~/kent/src/hg/utils/automation/parseRefSeqFuncElems funcElems.gff /dev/stdout \ +| sort -k1,1 -k2n,2n > refSeqFuncElems.bed +wc -l refSeqFuncElems.bed +#5756 refSeqFuncElems.bed + +# Make bigBed and link from /gbdb +bedToBigBed -tab -type=bed9+7 -as=$HOME/kent/src/hg/lib/refSeqFuncElems.as \ + refSeqFuncElems.bed /hive/data/genomes/hg38/chrom.sizes refSeqFuncElems.bb +rm -f /gbdb/hg38/ncbiRefSeq/refSeqFuncElems.bb +ln -s `pwd`/refSeqFuncElems.bb /gbdb/hg38/ncbiRefSeq/ + +################################################################### +# cosmicRegions (DONE 2017-08-03 Chris) +# Make a new COSCMIC track for hg38 v82 +mkdir /hive/data/outside/cosmic/hg38/v82 +cd /hive/data/outside/cosmic/hg38/v82 + +# Get the new data +sftp ceisenha@ucsc.edu@sftp-cancer.sanger.ac.uk +# Login to SFTP server then run these commands +get /files/grch38/cosmic/v82/CosmicMutantExport.tsv.gz + +# Remove the 'NS' fields, search for the \t after to exclude the E'NS'ST transcripts. +zcat CosmicMutantExport.tsv.gz | sed 's/NS\t/\t/g' > cosMut.tsv + +# Use a script to convert to bed format. +cosmicToBed cosMut.tsv cosMut.bed +# This many lines were skipped, 134601 for not having genomic coordinate + +# Sort and convert to big bed using the .as file. +sort -k1,1 -k2,2n cosMut.bed > sCosMut.bed +bedToBigBed -type=bed8+31 -as=cosmicNew.as sCosMut.bed /hive/data/genomes/hg38/chrom.sizes cosMutHg38V82.bb -tab -extraIndex=name,cosmLabel + + +# Link it up so the outside world can see it. +cd /gbdb/hg38/cosmic/ +ln -s /hive/data/outside/cosmic/hg38/v82/cosMutHg38V82.bb . + +######################################################################### +# RepeatMasker Visualization track update (DONE - 2018-05-04 - ChrisL) + screen -S rmskJoined.2018-05-04 + mkdir /hive/data/genomes/hg38/bed/rmskJoined.2018-05-04 + cd /hive/data/genomes/hg38/bed/rmskJoined.2018-05-04 + + ln -s ../repeatMasker/hg38.sorted.fa.out . + ln -s ../repeatMasker/hg38.fa.align.gz . + + # this script points to the most recent RepeatMasker version: + time (/scratch/data/RepeatMasker/util/rmToUCSCTables.pl \ + -out hg38.sorted.fa.out -align hg38.fa.align.gz) > do.log 2>&1 & + + # no differences, forgot to remake rmsk files + # so instead remake the rmsk track and try again + mkdir /hive/data/genomes/hg38/bed/repeatMasker.2018-05-04 + cd /hive/data/genomes/hg38/bed/repeatMasker.2018-05-04 + + # remake the sorted.fa.out and fa.align.gz, stop after masking + # so rmsk table isn't overwritten + time (doRepeatMasker.pl -stop=mask -useHMMER -bigClusterHub=ku \ + -workhorse=hgwdev -dbHost=hgwdev -buildDir=`pwd` hg38) > mask.log 2>&1 & + # RepeatMasker bug?: Undefined id, line 1440295 of input: + # 10 26.1 0.0 0.0 chr13 114292339 114292382 (71946) C L1P4 LINE/L1 (17) 6149 6106 + # RepeatMasker bug?: Undefined id, line 3529762 of input: + # 992 2.3 0.5 0.0 chr3 180461254 180462048 (17833511) C L1PA3 LINE/L1 (3) 6152 5354 + # RepeatMasker bug?: Undefined id, line 3529763 of input: + # 1153 3.2 0.2 0.0 chr3 180462043 180463006 (17832553) + L1PA3 LINE/L1 4392 5357 (789) + # RepeatMasker bug?: Undefined id, line 5303571 of input: + # 220 22.5 0.0 17.7 chr9 105798076 105799127 (32595590) C SATR2 Satellite (4) 866 1 + # real 643m17.617s + + # get rid of the missing id items: + grep -v "114292339 114292382\|180461254 180462048\|180462043 180463006\|105798076 105799127" \ + hg38.fa.out > clean.hg38.fa.out + mv clean.hg38.fa.out hg38.fa.out + + # finish the last step of doCat.csh: + /cluster/bin/scripts/extractNestedRepeats.pl hg38.fa.out | sort -k1,1 -k2,2n > hg38.nestedRepeats.bed + + cd /hive/data/genomes/hg38/bed/rmskJoined.2018-05-04 + + rm hg38.sorted.fa.out + rm hg38.fa.align.gz + rm *.tsv + ln -s ../repeatMasker.2018-05-04/hg38.sorted.fa.out . + ln -s ../repeatMasker.2018-05-04/hg38.fa.align . + + # and then re-run + time (/scratch/data/RepeatMasker/util/rmToUCSCTables.pl \ + -out hg38.sorted.fa.out -align hg38.fa.align.gz) > rerun.log 2>&1 & + # real 141m7.268s + + # confirm the counts are different from the previous version: + # wc -l ../rmskJoined/hg38.fa.align.tsv ../rmskJoined/hg38.sorted.fa.join.bed ../rmskJoined/hg38.sorted.fa.out.tsv + 7203858 ../rmskJoined/hg38.fa.align.tsv + 4607727 ../rmskJoined/hg38.sorted.fa.join.bed + 5520118 ../rmskJoined/hg38.sorted.fa.out.tsv + 17331703 total + # wc -l *.tsv + 7227245 hg38.fa.align.tsv + 4828114 hg38.sorted.fa.join.tsv + 5916189 hg38.sorted.fa.out.tsv + 17971548 total + + hgLoadBed -sqlTable=$HOME/kent/src/hg/lib/rmskJoined.sql \ + -renameSqlTable -verbose=4 -tab \ + -type=bed4+9 -as=$HOME/kent/src/hg/lib/rmskJoined.as hg38 \ + rmskJoinedCurrent hg38.sorted.fa.join.tsv \ + > loadJoined.log 2>&1 + + hgLoadSqlTab hg38 rmskAlignCurrent \ + /cluster/home/chmalee/kent/src/hg/lib/rmskAlign.sql \ + hg38.fa.align.tsv > loadAlign.log 2>&1 + + hgLoadOutJoined -verbose=2 -table=rmskOutCurrent hg38 hg38.sorted.fa.out > loadOut.log 2>&1 + + featureBits -countGaps hg38 rmskJoinedCurrent + # 2796899855 bases of 3209286105 (87.150%) in intersection +######################################################################### +# Hi-C Visualization based on Krietenstein 2019 (DONE - 2019-10-07 - Jonathan) +mkdir -p /hive/data/genomes/hg38/bed/hic +cd /hive/data/genomes/hg38/bed/hic + +# Files are located on 4D Nucleome (data.4dnucleome.org). The URL for the paper on that +# site is https://data.4dnucleome.org/publications/b13590b2-a341-4e5e-ad5e-72e233b32e9d/. +# The four file IDs downloaded below are for contact matrix .hic files created for +# different cell-line/protocol combinations +wget 'https://data.4dnucleome.org/files-processed/4DNFI2TK7L2F/@@download/4DNFI2TK7L2F.hic' # H1-hESC Micro-C XL +wget 'https://data.4dnucleome.org/files-processed/4DNFIQYQWPF5/@@download/4DNFIQYQWPF5.hic' # H1-hESC in situ +wget 'https://data.4dnucleome.org/files-processed/4DNFI18Q799K/@@download/4DNFI18Q799K.hic' # HFFc6 Micro-C XL +wget 'https://data.4dnucleome.org/files-processed/4DNFIFLJLIS5/@@download/4DNFIFLJLIS5.hic' # HFFc6 in situ + +printf "All files were downloaded from the 4D Nucleome Data Portal at data.4dnucleome.org. +These are processed contact matrices from Krietenstein et al. (2019) Ultrastructural details +of mammalian chromosme architecture. (https://www.biorxiv.org/content/10.1101/639922v1). + +4DNFI2TK7L2F.hic - Micro-C XL data set on H1-hESC +4DNFIQYQWPF5.hic - in situ Hi-C data set on H1-hESC +4DNFI18Q799K.hic - Micro-C XL data set on HFFc6 +4DNFIFLJLIS5.hic - in situ Hi-C data set on HFFc6" > README.txt + +mkdir -p /gbdb/hg38/bbi/hic +cd /gbdb/hg38/bbi/hic +ln -s /hive/data/genomes/hg38/bed/hic/* . + + +######################################################################### +# LASTZ Self/hg38 (DONE 2020-02-11 - Angie) + # RM #24695 + # Re-run with updated process to include pslDropOverlap . + # Use "contigs" from previous run lastzSelf.2014-01-25/hg38.self.2bit + + screen -S hg38Self -t hg38Self + mkdir /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27 + cd /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27 + cat << _EOF_ > DEF +# human vs human with mouse defaults +BLASTZ=/cluster/bin/penn/lastz-distrib-1.04.00/bin/lastz + +# TARGET: Human hg38 +SEQ1_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ1_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ1_CTGDIR=/hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/hg38.self.2bit +SEQ1_CTGLEN=/hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/hg38.self.chrom.sizes +SEQ1_LIFT=/hive/data/genomes/hg38/jkStuff/hg38.contigs.lift +SEQ1_CHUNK=20000000 +SEQ1_LAP=10000 + +# QUERY: Human hg38 +SEQ2_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ2_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ2_CTGDIR=/hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/hg38.self.2bit +SEQ2_CTGLEN=/hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/hg38.self.chrom.sizes +SEQ2_LIFT=/hive/data/genomes/hg38/jkStuff/hg38.contigs.lift +SEQ2_CHUNK=20000000 +SEQ2_LAP=0 + +BASE=/hive/data/genomes/hg38/bed/lastzSelf.2020-01-27 +TMPDIR=/dev/shm +_EOF_ + + # NOTE FOR NEXT TIME: use -chainMinScore=10000 (at least), not 3000 + + ~/kent/src/hg/utils/automation/doBlastzChainNet.pl `pwd`/DEF -verbose=2 \ + -chainMinScore=3000 -chainLinearGap=medium -syntenicNet \ + -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ + -stop=net >& do.log & + tail -f do.log + + + # After two days, 4 jobs are running, one of which (part014.lst vs itself) crashed with + # out-of-mem error. After 3 days, 3 jobs completed but part014.lst runs lastz out of mem. + # Split part014.lst up into components, run on hgwdev (more mem). + mkdir /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/run.blastz/part014 + cd /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/run.blastz/part014 + mkdir psl + cp /dev/null jobList + for t in $(cat ../tParts/part014.lst); do + tBase=$(basename $t) + for q in $(cat ../tParts/part014.lst); do + qBase=$(basename $q) + echo "$HOME/kent/src/hg/utils/automation/blastz-run-ucsc -outFormat psl -dropSelf $t $q ../../DEF {check out exists psl/${tBase}_${qBase}.psl }" >> jobList + done + done + para create jobList + para try, check, push, etc, + # 94 of the jobs ran for 12s or less. The other 6 are chr{X_Y}_00 vs. self & each other, + # chr13_16 vs self and chr16_03 vs self. All but chr16_03 vs self completed in < 6 minutes. +#Completed: 99 of 100 jobs +#Crashed: 1 jobs +#CPU time in finished jobs: 1559s 25.98m 0.43h 0.02d 0.000 y +#IO & Wait Time: 248s 4.14m 0.07h 0.00d 0.000 y +#Average job time: 18s 0.30m 0.01h 0.00d +#Longest finished job: 321s 5.35m 0.09h 0.00d +#Submission to last job: 94681s 1578.02m 26.30h 1.10d + + # Dang, chr16_03 vs. self still runs out of mem even on hgwdev. + mkdir /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/run.blastz/part014/chr16_03 + cd /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/run.blastz/part014/chr16_03 + twoBitToFa /hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/hg38.self.2bit:chr16_03:0-1689648 \ + chr16_03.fa + faSplit -lift=chr16_03.lift size chr16_03.fa 169000 chr16_03_split_ + faToTwoBit chr16_03_split_*.fa chr16_03_split.2bit + twoBitInfo chr16_03_split.2bit stdout | sort -k2nr > chr16_03_split.sizes + sed -re 's@CTGDIR.*@CTGDIR=/hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/run.blastz/part014/chr16_03/chr16_03_split.2bit@; + s@CTGLEN.*@CTGLEN=/hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/run.blastz/part014/chr16_03/chr16_03_split.sizes@;' \ + ../../../DEF > DEF.split + mkdir psl + cwd=$(pwd) + while read tBase tSize; do + while read qBase qSize; do + echo "$HOME/kent/src/hg/utils/automation/blastz-run-ucsc -outFormat psl -dropSelf $cwd/chr16_03_split.2bit:$tBase:0-$tSize $cwd/chr16_03_split.2bit:$qBase:0-$qSize DEF.split {check out exists psl/${tBase}_${qBase}.psl}" + done < chr16_03_split.sizes + done < chr16_03_split.sizes > jobList + para create jobList + para try, check, push, etc, +#Completed: 100 of 100 jobs +#CPU time in finished jobs: 142614s 2376.89m 39.61h 1.65d 0.005 y +#IO & Wait Time: 167s 2.79m 0.05h 0.00d 0.000 y +#Average job time: 1428s 23.80m 0.40h 0.02d +#Longest finished job: 22861s 381.02m 6.35h 0.26d +#Submission to last job: 22874s 381.23m 6.35h 0.26d + # 6 hours for chr16_03_split_00 vs. itself. ~4.5h for _09 vs _00. + cat psl/*.psl \ + | liftUp -nohead -type=.psl stdout \ + chr16_03.lift error stdin \ + | liftUp -nohead -type=.psl -pslQ \ + ../psl/hg38.self.2bit:chr16_03:0-1689648_hg38.self.2bit:chr16_03:0-1689648.psl \ + chr16_03.lift error stdin + + cd .. + cat psl/* > ../../psl/part014.lst/part014.lst_part014.lst.psl + + # Make run.time file or doBlastzChainNet.pl won't continue: + cd /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/run.blastz + para time >& run.time + + # Resume doBlastzChainNet.pl: + cd /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27 + ~/kent/src/hg/utils/automation/doBlastzChainNet.pl `pwd`/DEF -verbose=2 \ + -chainMinScore=3000 -chainLinearGap=medium -syntenicNet \ + -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ + -continue=cat -stop=net >& do2.log & + tail -f do2.log +#Batch failed after 4 tries on chain.csh part016.lst chain/part016.lst.chain +#Command failed: +#ssh -x -o 'StrictHostKeyChecking = no' -o 'BatchMode = yes' hgwdev nice /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/axtChain/run/doChainRun.csh + + cd /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/axtChain/run + para problems + # mostly these: +#errAbort re-entered due to out-of-memory condition. Exiting. + # one job made it through errAbort: +#needLargeMem: Out of memory - request size 564838920 bytes, errno: 12 + para time +#Completed: 59 of 68 jobs +#Crashed: 9 jobs +#CPU time in finished jobs: 24727s 412.12m 6.87h 0.29d 0.001 y +#IO & Wait Time: 0s 0.00m 0.00h 0.00d 0.000 y +#Average job time: 409s 6.82m 0.11h 0.00d +#Longest finished job: 2350s 39.17m 0.65h 0.03d +#Submission to last job: 2462s 41.03m 0.68h 0.03d + para crashed +#chain.csh part012.lst {check out line+ chain/part012.lst.chain} +#chain.csh part017.lst {check out line+ chain/part017.lst.chain} +#chain.csh part016.lst {check out line+ chain/part016.lst.chain} +#chain.csh part015.lst {check out line+ chain/part015.lst.chain} +#chain.csh part014.lst {check out line+ chain/part014.lst.chain} +#chain.csh hg38.self.2bit:chr1_10: {check out line+ chain/hg38.self.2bit:chr1_10:.chain} +#chain.csh hg38.self.2bit:chr10_05: {check out line+ chain/hg38.self.2bit:chr10_05:.chain} +#chain.csh hg38.self.2bit:chr7_00: {check out line+ chain/hg38.self.2bit:chr7_00:.chain} + + # Run the jobs outside of parasol (~11h): + csh -efx chain.csh part012.lst chain/part012.lst.chain & + csh -efx chain.csh part017.lst chain/part017.lst.chain & + csh -efx chain.csh part016.lst chain/part016.lst.chain & + csh -efx chain.csh part015.lst chain/part015.lst.chain & + csh -efx chain.csh part014.lst chain/part014.lst.chain & + csh -efx chain.csh hg38.self.2bit:chr1_10: chain/hg38.self.2bit:chr1_10:.chain & + csh -efx chain.csh hg38.self.2bit:chr10_05: chain/hg38.self.2bit:chr10_05:.chain & + csh -efx chain.csh hg38.self.2bit:chr7_00: chain/hg38.self.2bit:chr7_00:.chain & + csh -efx chain.csh hg38.self.2bit:chr16_08: chain/hg38.self.2bit:chr16_08:.chain & + + # Resume doBlastzChainNet.pl again: + cd /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27 + ~/kent/src/hg/utils/automation/doBlastzChainNet.pl `pwd`/DEF -verbose=2 \ + -chainMinScore=3000 -chainLinearGap=medium -syntenicNet \ + -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ + -continue=chainMerge -stop=net >& do3.log & + tail -f do3.log +# *** All done ! Elapsed time: 19m11s + + # Load track w/new name chainSelfRedo to compare to existing chainSelf: + hgLoadChain -normScore -tIndex hg38 chainSelfRedo axtChain/hg38.hg38.all.chain.gz + + # No idea why but somehow the liftUp seems not to have worked for part012 and part017, + # so the all.chain had chr22_31, chr8_01 etc. :b run again again. + cd /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/axtChain/run + mv chain/part012.lst.chain{,.bak} + mv chain/part017.lst.chain{,.bak} + csh -efx chain.csh part012.lst chain/part012.lst.chain >& part012.log & + csh -efx chain.csh part017.lst chain/part017.lst.chain >& part017.log & + # Those completed successfully. Dunno why the earlier ones didn't get lifted. + cd .. + mv hg38.hg38.all{,.oopsPartUnlifted}.chain.gz + # Reconstruct hg38.hg38.all.chain.gz (the chainMerge step is just this command): + find /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/axtChain/run/chain -name "*.chain" \ + | chainMergeSort -inputList=stdin \ + | nice gzip -c \ + > /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/axtChain/hg38.hg38.all.chain.gz + + # NOTE FOR NEXT TIME: this filtering step will be unnecessary when -minScore=10000 is used + # from the beginning. + # Filter to minScore of 10000 (too much fluff with -minScore=3000) per Jim (see #24695) + cd /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/axtChain + mv hg38.hg38.all.chain.gz hg38.hg38.all.unfiltered.chain.gz + chainFilter hg38.hg38.all.unfiltered chain.gz -minScore=10000 \ + | gzip -c > hg38.hg38.all.chain.gz + hgLoadChain -normScore -tIndex hg38 chainSelfRedo hg38.hg38.all.chain.gz + checkTableCoords hg38 chainSelfRedo + + # Rename to chainSelf and update lastz symlinks and downloads + hgsql hg38 -e 'drop table chainSelf; drop table chainSelfLink; + rename table chainSelfRedo to chainSelf; + rename table chainSelfRedoLink to chainSelfLink;' + cd /hive/data/genomes/hg38/bed + rm lastz.self lastz.hg38 + ln -s lastzSelf.2020-01-27 lastz.self + ln -s lastzSelf.2020-01-27 lastz.hg38 + cd /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/axtChain + cp /hive/data/genomes/hg38/bed/lastzSelf.2014-01-25/axtChain/README.txt . + $EDITOR README.txt + md5sum hg38.hg38.all.chain.gz > md5sum.txt + # Make sure that the old download dir has only symlinks, no real files, then remove and rebuild. + ls -lR /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/vsSelf/ + rm -r /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/vsSelf/ + mkdir /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/vsSelf/ + cd /usr/local/apache/htdocs-hgdownload/goldenPath/hg38/vsSelf/ + ln -s /hive/data/genomes/hg38/bed/lastzSelf.2020-01-27/axtChain/{README.txt,hg38.hg38.all.chain.gz,md5sum.txt} . + + +######################################################################### +# NCBI ReMap alignments (DONE 2020-02-11 Angie) +# RM 24449 + mkdir /hive/data/genomes/hg38/bed/chainHg19ReMap + cd /hive/data/genomes/hg38/bed/chainHg19ReMap + wget ftp://ftp.ncbi.nlm.nih.gov/pub/remap/Homo_sapiens/current/GCF_000001405.39_GRCh38.p13/GCF_000001405.25_GRCh37.p13/GCF_000001405.39-GCF_000001405.25.gff + # We will need to substitute all the RefSeq chrom and contig IDs with our own names. + # The same alt contig can appear in both assemblies with the same name, so replace + # hg19 names at the beginning of the line and hg38 names after "Target=". + hgsql hg19 -NBe 'select alias, chrom from chromAlias where find_in_set("refseq", source)' \ + | sed -re 's/\./\\./;' \ + | awk '{print "s/^" $1 "\\b/" $2 "/;";}' \ + > hg38.hg19.chromAlias.sed + hgsql hg38 -NBe 'select alias, chrom from chromAlias where find_in_set("refseq", source)' \ + | sed -re 's/\./\\./;' \ + | awk '{print "s/Target=" $1 "\\b/Target=" $2 "/;";}' \ + >> hg38.hg19.chromAlias.sed + + # There are some GRCh38.p13 sequences that we have not yet imported into hg38 -- use -dropT. + sed -f hg38.hg19.chromAlias.sed GCF_000001405.39-GCF_000001405.25.gff \ + | gff3ToPsl -dropT /hive/data/genomes/{hg19,hg38}/chrom.sizes stdin stdout \ + | pslPosTarget stdin stdout \ + | sort -k14,14 -k16n,16n > remap.hg38.hg19.psl + + # Convert to chain for browser display. Some of the remap chains have minScore < 1000 and + # by default would be dropped by chainScore... use -minScore=0 to prevent that. + time pslToChain remap.hg38.hg19.psl stdout \ + | chainScore -minScore=0 stdin /hive/data/genomes/{hg38/hg38.2bit,hg19/hg19.2bit} \ + remap.hg38.hg19.chain +#real 9m31.900s +#user 9m1.624s +#sys 0m20.863s + hgLoadChain hg38 -tIndex chainHg19ReMap remap.hg38.hg19.chain +#Loading 5315 chains into hg38.chainHg19ReMap + time axtChain -psl -linearGap=medium -verbose=0 remap.hg38.hg19.psl \ + /hive/data/genomes/hg38/hg38.2bit /hive/data/genomes/hg19/hg19.2bit \ + remap.axtChain.hg38.hg19.chain +#real 2m26.333s +#user 2m4.237s +#sys 0m22.071s + hgLoadChain hg38 -tIndex chainHg19ReMapAxtChain remap.axtChain.hg38.hg19.chain +#Loading 2115 chains into hg38.chainHg19ReMapAxtChain + +################################################### +#Agilent SNP/CNV arrays 3/11/21 +#Downloaded by web browser +cd /hive/data/genomes/hg38/bed/agilentProbes/genetiSureCyto +fetchChromSizes hg38 > hg38.chrom.sizes +bedSort hg38.GenetiSure_Cyto_CGH+SNP_Microarray_4x180K_085591_D_BED_20200302.bed hg38.GenetiSure_Cyto_CGH+SNP_Microarray_4x180K_085591_D_BED_20200302.bed +uniq hg38.GenetiSure_Cyto_CGH+SNP_Microarray_4x180K_085591_D_BED_20200302.bed >hg38.GenetiSure_Cyto_CGH+SNP_Microarray_4x180K_085591_D_BED_20200302.uniq.bed +bedToBigBed hg38.GenetiSure_Cyto_CGH+SNP_Microarray_4x180K_085591_D_BED_20200302.uniq.bed hg38.chrom.sizes hg38.GenetiSure_Cyto_CGH+SNP_Microarray_4x180K_085591_D_BED_20200302.bb +bedSort hg38.GenetiSure_Cyto_CGH_Microarray_8x60K_085590_D_BED_20200302.bed hg38.GenetiSure_Cyto_CGH_Microarray_8x60K_085590_D_BED_20200302.bed +uniq hg38.GenetiSure_Cyto_CGH_Microarray_8x60K_085590_D_BED_20200302.bed > hg38.GenetiSure_Cyto_CGH_Microarray_8x60K_085590_D_BED_20200302.uniq.bed +bedToBigBed hg38.GenetiSure_Cyto_CGH_Microarray_8x60K_085590_D_BED_20200302.uniq.bed hg38.chrom.sizes hg38.GenetiSure_Cyto_CGH_Microarray_8x60K_085590_D_BED_20200302.bb +bedSort hg38.GenetiSure_Cyto_CGH_Microarray_4x180K_085589_D_BED_20200302.bed hg38.GenetiSure_Cyto_CGH_Microarray_4x180K_085589_D_BED_20200302.bed +uniq hg38.GenetiSure_Cyto_CGH_Microarray_4x180K_085589_D_BED_20200302.bed > hg38.GenetiSure_Cyto_CGH_Microarray_4x180K_085589_D_BED_20200302.uniq.bed +bedToBigBed hg38.GenetiSure_Cyto_CGH_Microarray_4x180K_085589_D_BED_20200302.uniq.bed hg38.chrom.sizes hg38.GenetiSure_Cyto_CGH_Microarray_4x180K_085589_D_BED_20200302.bb +mkdir -p /gbdb/hg38/snpCnvArrays/agilent +cd /gbdb/hg38/snpCnvArrays/agilent +ln -s /hive/data/genomes/hg38/bed/agilentProbes/genetiSureCyto/hg38.GenetiSure_Cyto_CGH+SNP_Microarray_4x180K_085591_D_BED_20200302.bb +ln -s /hive/data/genomes/hg38/bed/agilentProbes/genetiSureCyto/hg38.GenetiSure_Cyto_CGH_Microarray_8x60K_085590_D_BED_20200302.bb +ln -s /hive/data/genomes/hg38/bed/agilentProbes/genetiSureCyto/hg38.GenetiSure_Cyto_CGH_Microarray_4x180K_085589_D_BED_20200302.bb +vi ~/kent/src/hg/makeDb/trackDb/human/hg38/trackDb.ra + +######################################################################### +# DECIPHER CNV & SNV - initial build (DONE 2022-04-08 Jonathan) +# RM 29130 + +cd /hive/data/genomes/outside/otto/decipher +mkdir 2022-04-05 +cd 2022-04-05 + +# manually fetch decipher-variants-grch38-2022-04-03.bed from DECIPHER +../buildDecipher decipher-variants-grch38-2022-04-03.bed + +for i in `cat ../decipher.tables` + do + n=$i"New" + o=$i"Old" + hgsqlSwapTables hg38 $n $i $o -dropTable3 + done + +mkdir -p /gbdb/hg38/decipher +cd /gbdb/hg38/decipher +ln -s /hive/data/outside/otto/decipher/2022-04-05/decipherCnv.bb . + +######################################################################### +# COSMIC (DONE 07-11-2023) +# RM 29625 + +#Fetch file +cd /hive/data/outside/cosmic/hg38/v98/ +wget 'https://cog.sanger.ac.uk/cosmic/GRCh38/ucsc/v98/ucsc_export.bed.gz?AWSAccessKeyId=KRV7P7QR9DL41J9EWGA2&Expires=1686847188&Signature=4YV3CuFKudxIhqVdWAaCe0CMAiY%3D' -O ucsc_export.bed.gz +wget 'https://cog.sanger.ac.uk/cosmic/GRCh38/ucsc/v98/ucsc_export.bed.gz?AWSAccessKeyId=KRV7P7QR9DL41J9EWGA2&Expires=1687525456&Signature=jBdJOlOOaqmMWNnOtJUyNRptVj4%3D' +mv ucsc_export.bed.gz\?AWSAccessKeyId\=KRV7P7QR9DL41J9EWGA2\&Expires\=1687525456\&Signature\=jBdJOlOOaqmMWNnOtJUyNRptVj4\= ucsc_export.bed.gz + +#Reorder to columns to conform to bed 6+3 +zcat ucsc_export.bed.gz | awk -F'\t' -v OFS="\t" '{ print $1, $2, $3, $7, 0, $6, $4, $5, $8 }' | sort -k1,1 -k2,2n > cosmic.bed + +#Tiny bit of python to identify the broken lines in the file where chromStart > chromEnd + +#for line in myFile: +# newLine = line.split("\t") +# if int(newLine[1]) > int(newLine[2]): +# print(line) +# n+=1 +#print(n) + +#remove those broken records from the file +cat cosmic.bed | grep -vf badRecords.bed > cosmic.fixed.bed + +#subtract to conform to bed format for all the items that have same star and endPos + +cat cosmic.fixed.bed | awk 'BEGIN {OFS="\t"} { +if ($2 == $3) + print $1,$2-1,$3,$4,$5,$6,$7,$8,$9; +else + print $0; +}' > cosmic.fixedPos.bed + +bedToBigBed -type=bed6+3 -as=/hive/data/outside/cosmic/hg38/v98/cosmic.as /hive/data/outside/cosmic/hg38/v98/cosmic.fixedPos.bed /hive/data/genomes/hg38/chrom.sizes /hive/data/outside/cosmic/hg38/v98/cosmic.bb -tab + +#make symlink +ln -s /hive/data/outside/cosmic/hg38/v98/cosmic.bb /gbdb/hg38/cosmic/cosmic.bb + +#This data has since been updated, see new makedoc doc/hg38/cosmicV98.txt and rm #32430 + +############################################################################## +# LIFTOVER TO GCA_018873775.2_hg01243.v3.0 (DONE - 2023-08-13 - Hiram) + ssh hgwdev + # going to need an ooc for hg38.p14.2bit + cd /hive/data/genomes/hg38 + time blat hg38.p14.2bit /dev/null /dev/null -tileSize=11 \ + -makeOoc=hg38.p14.ooc -repMatch=1024 + # Wrote 36808 overused 11-mers to hg38.p14.ooc + # real 0m50.753s + + # and ooc for this GenArk hub + cd /hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0 + time blat GCA_018873775.2_hg01243.v3.0.2bit /dev/null /dev/null -tileSize=11 \ + -makeOoc=GCA_018873775.2_hg01243.v3.0.ooc -repMatch=1024 +# Wrote 39087 overused 11-mers to GCA_018873775.2_hg01243.v3.0.ooc +# real 0m49.426s + + mkdir /hive/data/genomes/hg38/bed/blat.GCA_018873775.2_hg01243.v3.0.2023-08-13 + cd /hive/data/genomes/hg38/bed/blat.GCA_018873775.2_hg01243.v3.0.2023-08-13 + + doSameSpeciesLiftOver.pl -verbose=2 -buildDir=`pwd` \ + -debug -bigClusterHub=hgwdev -dbHost=hgwdev -workhorse=hgwdev \ + -target2Bit=/hive/data/genomes/hg38/hg38.2bit \ + -targetSizes=/hive/data/genomes/hg38/chrom.sizes \ + -query2Bit=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0/GCA_018873775.2_hg01243.v3.0.2bit \ + -querySizes=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0/GCA_018873775.2_hg01243.v3.0.chrom.sizes \ + -ooc=/hive/data/genomes/hg38/hg38.p14.ooc \ + hg38 GCA_018873775.2 + + # trying -ram=6g to get full use of hgwdev kluster nodes + time (~/kent/src/hg/utils/automation/doSameSpeciesLiftOver.pl \ + -verbose=2 -buildDir=`pwd` \ + -bigClusterHub=hgwdev -dbHost=hgwdev -workhorse=hgwdev \ + -target2Bit=/hive/data/genomes/hg38/hg38.2bit \ + -targetSizes=/hive/data/genomes/hg38/chrom.sizes \ + -query2Bit=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0/GCA_018873775.2_hg01243.v3.0.2bit \ + -querySizes=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0/GCA_018873775.2_hg01243.v3.0.chrom.sizes \ + -ooc=/hive/data/genomes/hg38/hg38.p14.ooc \ + hg38 GCA_018873775.2) > doLiftOverToGCA_018873775.2.log 2>&1 + # real 12654m58.134s + + # broken after the alignment was done, with the parasol endless loop + # error message in the log file: + # select failure in rudp: Invalid argument + # killed that, cleaned the 4Tb log file, and gave up on this alignment + # since the lastz/chain/net is much better + + # see if the liftOver menus function in the browser from hg38 + # to GCA_018873775.2 + +############################################################################## +# LIFTOVER GCA_018873775.2_hg01243.v3.0 to hg38 (DONE - 2023-08-13 - Hiram) + ssh hgwdev + + mkdir /hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0/trackData/blat.hg38.2023-08-13 + cd /hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0/trackData/blat.hg38.2023-08-13 + + doSameSpeciesLiftOver.pl -verbose=2 -buildDir=`pwd` \ + -debug -bigClusterHub=hgwdev -dbHost=hgwdev -workhorse=hgwdev \ + -target2Bit=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0/GCA_018873775.2_hg01243.v3.0.2bit \ + -targetSizes=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0/GCA_018873775.2_hg01243.v3.0.chrom.sizes \ + -query2Bit=/hive/data/genomes/hg38/hg38.2bit \ + -querySizes=/hive/data/genomes/hg38/chrom.sizes \ + -ooc=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0/GCA_018873775.2_hg01243.v3.0.ooc \ + GCA_018873775.2 hg38 + + # trying -ram=6g to get full use of hgwdev kluster nodes + time (~/kent/src/hg/utils/automation/doSameSpeciesLiftOver.pl \ + -verbose=2 -buildDir=`pwd` \ + -bigClusterHub=hgwdev -dbHost=hgwdev -workhorse=hgwdev \ + -target2Bit=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0/GCA_018873775.2_hg01243.v3.0.2bit \ + -targetSizes=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0/GCA_018873775.2_hg01243.v3.0.chrom.sizes \ + -query2Bit=/hive/data/genomes/hg38/hg38.2bit \ + -querySizes=/hive/data/genomes/hg38/chrom.sizes \ + -ooc=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/873/775/GCA_018873775.2_hg01243.v3.0/GCA_018873775.2_hg01243.v3.0.ooc \ + GCA_018873775.2 hg38) > doLiftOverToHg38.log 2>&1 + + # broken after the alignment was done, with the parasol endless loop + # error message in the log file: + # select failure in rudp: Invalid argument + # killed that, cleaned the 4Tb log file, and gave up on this alignment + # since the lastz/chain/net is much better + # real 193m24.137s + + # see if the liftOver menus function in the browser from GCA_018873775.2 + # to hg38 + +############################################################################## +# LIFTOVER TO GCA_018503275.1_NA19240.pri.mat.f1_v2 (TBD - 2023-08-14 - Hiram) + ssh hgwdev + + # ooc for this GenArk hub + cd /hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2 + time blat GCA_018503275.1_NA19240.pri.mat.f1_v2.2bit /dev/null /dev/null \ + -tileSize=11 -repMatch=1024 \ + -makeOoc=GCA_018503275.1_NA19240.pri.mat.f1_v2.11.ooc + # Wrote 35866 overused 11-mers to GCA_018503275.1_NA19240.pri.mat.f1_v2.11.ooc + # real 0m32.298s + + mkdir /hive/data/genomes/hg38/bed/blat.GCA_018503275.1_NA19240.pri.mat.f1_v2.2023-08-14 + cd /hive/data/genomes/hg38/bed/blat.GCA_018503275.1_NA19240.pri.mat.f1_v2.2023-08-14 + + ~/kent/src/hg/utils/automation/doSameSpeciesLiftOver.pl -verbose=2 \ + -buildDir=`pwd` -ram=4g -chainRam=16g \ + -debug -bigClusterHub=hgwdev -dbHost=hgwdev -workhorse=hgwdev \ + -target2Bit=/hive/data/genomes/hg38/hg38.2bit \ + -targetSizes=/hive/data/genomes/hg38/chrom.sizes \ + -query2Bit=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2/GCA_018503275.1_NA19240.pri.mat.f1_v2.2bit \ + -querySizes=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2/GCA_018503275.1_NA19240.pri.mat.f1_v2.chrom.sizes \ + -ooc=/hive/data/genomes/hg38/hg38.p14.ooc \ + hg38 GCA_018503275.1 + + # trying -ram=4g to get full use of hgwdev kluster nodes + time (~/kent/src/hg/utils/automation/doSameSpeciesLiftOver.pl \ + -verbose=2 -buildDir=`pwd` -ram=4g -chainRam=16g \ + -bigClusterHub=hgwdev -dbHost=hgwdev -workhorse=hgwdev \ + -target2Bit=/hive/data/genomes/hg38/hg38.2bit \ + -targetSizes=/hive/data/genomes/hg38/chrom.sizes \ + -query2Bit=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2/GCA_018503275.1_NA19240.pri.mat.f1_v2.2bit \ + -querySizes=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2/GCA_018503275.1_NA19240.pri.mat.f1_v2.chrom.sizes \ + -ooc=/hive/data/genomes/hg38/hg38.p14.ooc \ + hg38 GCA_018503275.1) > doLiftOverToGCA_018503275.1.log 2>&1 + # real 11370m18.026s + + # broken after the alignment was done, with the parasol endless loop + # error message in the log file: + # select failure in rudp: Invalid argument + # killed that, cleaned the 4Tb log file, and gave up on this alignment + # since the lastz/chain/net is much better + # -rw-rw-r-- 1 4363949695640 Aug 22 09:16 doLiftOverToGCA_018503275.1.log + + # see if the liftOver menus function in the browser from hg38 + # to GCA_018503275.1 + +############################################################################## +# LIFTOVER GCA_018503275.1_NA19240.pri.mat.f1_v2 to hg38 (DONE - 2023-08-14 - Hiram) + ssh hgwdev + + mkdir /hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2/trackData/blat.hg38.2023-08-14 + cd /hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2/trackData/blat.hg38.2023-08-14 + + ~/kent/src/hg/utils/automation/doSameSpeciesLiftOver.pl -verbose=2 \ + -buildDir=`pwd` -ram=4g -chainRam=16g \ + -debug -bigClusterHub=hgwdev -dbHost=hgwdev -workhorse=hgwdev \ + -target2Bit=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2/GCA_018503275.1_NA19240.pri.mat.f1_v2.2bit \ + -targetSizes=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2/GCA_018503275.1_NA19240.pri.mat.f1_v2.chrom.sizes \ + -query2Bit=/hive/data/genomes/hg38/hg38.2bit \ + -querySizes=/hive/data/genomes/hg38/chrom.sizes \ + -ooc=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2/GCA_018503275.1_NA19240.pri.mat.f1_v2.11.ooc \ + GCA_018503275.1 hg38 + + time (~/kent/src/hg/utils/automation/doSameSpeciesLiftOver.pl -verbose=2 \ + -buildDir=`pwd` -ram=4g -chainRam=16g \ + -bigClusterHub=hgwdev -dbHost=hgwdev -workhorse=hgwdev \ + -target2Bit=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2/GCA_018503275.1_NA19240.pri.mat.f1_v2.2bit \ + -targetSizes=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2/GCA_018503275.1_NA19240.pri.mat.f1_v2.chrom.sizes \ + -query2Bit=/hive/data/genomes/hg38/hg38.2bit \ + -querySizes=/hive/data/genomes/hg38/chrom.sizes \ + -ooc=/hive/data/genomes/asmHubs/genbankBuild/GCA/018/503/275/GCA_018503275.1_NA19240.pri.mat.f1_v2/GCA_018503275.1_NA19240.pri.mat.f1_v2.11.ooc \ + GCA_018503275.1 hg38) > liftOverToHg38.log 2>&1 + # real 5082m17.500s + + # this is interesting, this alignment completed and actually has good + # coverage: + cat fb.GCA_018503275.1.chain.Hg38Link.txt + # 2928654519 bases of 3032066086 (96.589%) in intersection + + # see if the liftOver menus function in the browser from GCA_018503275.1 + # to hg38 + +############################################################################## +## update grp table add new row for HPRC (DONE - 2023-08-29 - Hiram) +## existing structure: + + hgsql -e 'desc grp;' hg38 + ++-----------------+-----------+------+-----+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++-----------------+-----------+------+-----+---------+-------+ +| name | char(255) | NO | PRI | | | +| label | char(255) | NO | | | | +| priority | float | NO | | 0 | | +| defaultIsClosed | int(11) | YES | | NULL | | ++-----------------+-----------+------+-----+---------+-------+ + + # add one new row: + hgsql hg38 \ + -e "INSERT INTO grp VALUES ('hprc', 'Human Pangenome - HPRC', 3.6, 0);" + + # resulting table: + + hgsql -e 'select * from grp order by priority;' hg38 ++------------+------------------------------------+----------+-----------------+ +| name | label | priority | defaultIsClosed | ++------------+------------------------------------+----------+-----------------+ +| user | Custom Tracks | 1 | 0 | +| remc | Reference Epigenome Mapping Center | 1.2 | 1 | +| map | Mapping and Sequencing | 2 | 0 | +| genes | Genes and Gene Predictions | 3 | 0 | +| phenDis | Phenotype and Literature | 3.4 | 0 | +| pub | Literature | 3.5 | 0 | +| hprc | Human Pangenome - HPRC | 3.6 | 0 | +| covid | COVID-19 | 3.6 | 0 | +| singleCell | Single Cell RNA-seq | 3.7 | 0 | +| rna | mRNA and EST | 4 | 0 | +| expression | Expression | 4.5 | 0 | +| regulation | Regulation | 5 | 0 | +| compGeno | Comparative Genomics | 6 | 0 | +| varRep | Variation | 7 | 0 | +| rep | Repeats | 8 | 0 | +| x | Experimental | 10 | 1 | ++------------+------------------------------------+----------+-----------------+ + +############################################################################## +# Affy CytoScan HD track, refs #32856 (2024-01-23 Gerardo) +cd /hive/data/genomes/hg38/bed/ +mkdir genotypeArrays; cd genotypeArrays +#The user sent Gerardo a direct email with a shared folder link. Gerardo downloaded the bed files and made them available on dev. +#The user provided two bed files (https://hgwdev-gperez2.gi.ucsc.edu/~gperez2/mlq/mlq_32791/). Gerardo used the version 2 bed file for the track. +wget https://hgwdev-gperez2.gi.ucsc.edu/~gperez2/mlq/mlq_32791/CytoScanHD_Accel_Array.na36.bed.zip +unzip CytoScanHD_Accel_Array.na36.bed.zip +# Removed header and sorted the file +grep -v 'track' CytoScanHD_Accel_Array.na36.bed | bedSort stdin stdout > affyCytoScanHD.bed +bedToBigBed -tab -type=bed12 affyCytoScanHD.bed https://hgdownload.soe.ucsc.edu/goldenPath/hg38/bigZips/hg38.chrom.sizes affyCytoScanHD.bb +cd /gbdb/hg38 +mkdir genotypeArrays; cd genotypeArrays +# Making symlink for big file and raw bed file +ln -s /hive/data/genomes/hg38/bed/genotypeArrays/affyCytoScanHD.bb +ln -s /hive/data/genomes/hg38/bed/genotypeArrays/CytoScanHD_Accel_Array.na36.bed.zip +cd ~/kent/src/hg/makeDb/trackDb/human/hg38 +vi trackDb.ra + +############################################################################## +# LASTZ Human Hg38 vs. California sea lion GCF_009762305.2 +# (DONE - 2024-03-06 - jairo) + + mkdir /hive/data/genomes/hg38/bed/lastzGCF_009762305.2.2024-03-06 + cd /hive/data/genomes/hg38/bed/lastzGCF_009762305.2.2024-03-06 + + printf '# California sea lion GCF_009762305.2 vs. Human Hg38 +BLASTZ=/cluster/bin/penn/lastz-distrib-1.04.03/bin/lastz + +# TARGET: Human hg38 +SEQ1_DIR=/hive/data/genomes/hg38/hg38.2bit +SEQ1_LEN=/hive/data/genomes/hg38/chrom.sizes +SEQ1_CHUNK=20000000 +SEQ1_LAP=10000 +SEQ1_LIMIT=40 + +# QUERY: California sea lion 2020-07-14 GCF_009762305.2_mZalCal1.pri.v2 +SEQ2_DIR=/hive/data/genomes/asmHubs/GCF/009/762/305/GCF_009762305.2/GCF_009762305.2.2bit +SEQ2_LEN=/hive/data/genomes/asmHubs/GCF/009/762/305/GCF_009762305.2/GCF_009762305.2.chrom.sizes.txt +SEQ2_CHUNK=20000000 +SEQ2_LAP=0 +SEQ2_LIMIT=100 + +BASE=/hive/data/genomes/hg38/bed/lastzGCF_009762305.2.2024-03-06 +TMPDIR=/dev/shm + +' > DEF + + time (~/kent/src/hg/utils/automation/doBlastzChainNet.pl -trackHub -noDbNameCheck -verbose=2 `pwd`/DEF -syntenicNet \ + -qAsmId GCF_009762305.2_mZalCal1.pri.v2 -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ + -chainMinScore=3000 -chainLinearGap=medium) > do.log 2>&1 + grep -w real do.log | sed -e 's/^/ # /;' + # real 1018m28.119s + + sed -e 's/^/ # /;' fb.hg38.chainGCF_009762305.2Link.txt + # 1633315994 bases of 3299210039 (49.506%) in intersection + sed -e 's/^/ # /;' fb.hg38.chainSynGCF_009762305.2Link.txt + # 1564193911 bases of 3299210039 (47.411%) in intersection + + time (~/kent/src/hg/utils/automation/doRecipBest.pl -trackHub -load -workhorse=hgwdev -buildDir=`pwd` \ + \ + -query2Bit="/hive/data/genomes/asmHubs/GCF/009/762/305/GCF_009762305.2/GCF_009762305.2.2bit" \ +-querySizes="/hive/data/genomes/asmHubs/GCF/009/762/305/GCF_009762305.2/GCF_009762305.2.chrom.sizes.txt" \ + hg38 GCF_009762305.2) > rbest.log 2>&1 + + grep -w real rbest.log | sed -e 's/^/ # /;' + # real 303m36.739s + + sed -e 's/^/ # /;' fb.hg38.chainRBest.GCF_009762305.2.txt + # 1461974620 bases of 3299210039 (44.313%) in intersection + + ### and for the swap + + cd /hive/data/genomes/asmHubs/allBuild/GCF/009/762/305/GCF_009762305.2_mZalCal1.pri.v2/trackData/blastz.hg38.swap + + time (~/kent/src/hg/utils/automation/doBlastzChainNet.pl -trackHub -noDbNameCheck -swap -verbose=2 \ + -qAsmId GCF_009762305.2_mZalCal1.pri.v2 /hive/data/genomes/hg38/bed/lastzGCF_009762305.2.2024-03-06/DEF -swapDir=`pwd` \ + -syntenicNet -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \ + -chainMinScore=3000 -chainLinearGap=medium) > swap.log 2>&1 + + grep -w real swap.log | sed -e 's/^/ # /;' + # real 103m25.220s + + sed -e 's/^/ # /;' fb.GCF_009762305.2.chainHg38Link.txt + # 1493183463 bases of 2409685272 (61.966%) in intersection + sed -e 's/^/ # /;' fb.GCF_009762305.2.chainSynHg38Link.txt + # 1457122207 bases of 2409685272 (60.469%) in intersection +\ time (~/kent/src/hg/utils/automation/doRecipBest.pl -trackHub -load -workhorse=hgwdev -buildDir=`pwd` \ + \ + -target2bit="/hive/data/genomes/asmHubs/GCF/009/762/305/GCF_009762305.2/GCF_009762305.2.2bit" \ +-targetSizes="/hive/data/genomes/asmHubs/GCF/009/762/305/GCF_009762305.2/GCF_009762305.2.chrom.sizes.txt" \ + GCF_009762305.2 hg38) > rbest.log 2>&1 + + grep -w real rbest.log | sed -e 's/^/ # /;' + # real 286m31.189s + + sed -e 's/^/ # /;' fb.GCF_009762305.2.chainRBest.Hg38.txt + # 1461710350 bases of 2409685272 (60.660%) in intersection + +############################################################################## +# hg38.chromAlias.bb was incorrectly built without indexes so it will not +# work with bedToBigBed 2024-04-08 markd + +cd /hive/data/genomes/hg38/goldenPath/bigZips/initial +mv hg38.chromAlias.bb hg38.chromAlias.noindexes.bb +bigBedInfo -asOut hg38.chromAlias.noindexes.bb >hg38.chromAlias.as +bigBedToBed hg38.chromAlias.noindexes.bb hg38.chromAlias.bed +bedToBigBed -tab -type=bed3+ -as=hg38.chromAlias.as hg38.chromAlias.bed -sizesIs2Bit -extraIndex=ucsc,assembly,ensembl,genbank,refseq hg38.2bit hg38.chromAlias.bb + +############################################################################## + +# ENCODE 4 TF rPeak Clusters - RM #34930 - Lou 12/19/24 + +mkdir /hive/data/genomes/hg38/bed/ENCODEv4TFrPeaks +cd /hive/data/genomes/hg38/bed/ENCODEv4TFrPeaks +hubClone -download https://users.wenglab.org/gaomingshi/TF.rpeak.test.txt +ln -s /hive/data/genomes/hg38/bed/ENCODEv4TFrPeaks/no_trim.TF_name.rPeaks.bb /gbdb/hg38/bbi/ENCODE4/TFrPeakClusters.bb +ln -s /hive/data/genomes/hg38/bed/ENCODEv4TFrPeaks/no_trim.TF_name.decorator.bb /gbdb/hg38/bbi/ENCODE4/TFrPeakClustersDecorator.bb +# Then just moved the files to the ENCODEv4TFrPeaks dir, moved/tweaked HTML and trackDb -# alphaMissense ticket #32269 (Jeltje, Jan 2025) -mkdir -p /hive/data/genomes/hg38/bed/alphaMissense/ -cd /hive/data/genomes/hg38/bed/alphaMissense -wget https://storage.googleapis.com/dm_alphamissense/AlphaMissense_hg38.tsv.gz -time python ~/kent/src/hg/makeDb/outside/alphaMissense/alphaMissenseToWig.py AlphaMissense_hg38.tsv.gz -wigToBigWig a.wig ../../chrom.sizes a.bw & -wigToBigWig c.wig ../../chrom.sizes c.bw & -wigToBigWig g.wig ../../chrom.sizes g.bw & -wigToBigWig t.wig ../../chrom.sizes t.bw & -wait - -##Colors were added using the script -#kent/src/hg/makeDb/scripts/wigColorByColors/makeWigColorByRevelCadd.py