59ea63821d41438b12fb9170d6d3c5dee3e7bb2f markd Mon Jun 17 02:50:46 2019 -0700 split gencode hg38 doc into a separate file diff --git src/hg/makeDb/doc/hg38/hg38.txt src/hg/makeDb/doc/hg38/hg38.txt index de24bc1..636fe8b 100644 --- src/hg/makeDb/doc/hg38/hg38.txt +++ src/hg/makeDb/doc/hg38/hg38.txt @@ -5300,85 +5300,30 @@ # <silence> (the chrEBV was found with this check) featureBits -countGaps hg38 genomicSuperDups # 175429664 bases of 3209286105 (5.466%) in intersection featureBits -countGaps hg19 genomicSuperDups # 166092393 bases of 3137161264 (5.294%) in intersection featureBits -countGaps hg18 genomicSuperDups # 159204446 bases of 3107677273 (5.123%) in intersection featureBits -countGaps mm10 genomicSuperDups # 214917441 bases of 2730871774 (7.870%) in intersection featureBits -countGaps mm9 genomicSuperDups # 208214567 bases of 2725765481 (7.639%) in intersection -######################################################################### -2014-08-11: import of UCSC GENCODE group processing of GENCODE V20 (markd) - # download files - mkdir -p /hive/data/genomes/hg38/bed/gencodeV20/data - cd /hive/data/genomes/hg38/bed/gencodeV20 - - # download gencode release - cd data - wget -nv -r -np ftp://ftp.sanger.ac.uk/pub/gencode/Gencode_human/release_20 - mv ftp.sanger.ac.uk/pub/gencode/Gencode_human/release_20 . - rm -rf ftp.sanger.ac.uk - - cd .. - # create Makefile from previous one. - cp /hive/data/genomes/hg19/bed/gencodeV19/Makefile . - - # build and load tables - (time nice make -j 10) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense. - make cmpRelease >gencode-cmp.tsv - - ## Copy and update trackDb files from previous release. - ## Change version and use lower priority so it sorts to top of - ## super track page. - ## Important to make sure filter attrs.transcriptType matches current set - ## figured out with - select distinct transcriptType from wgEncodeGencodeAttrsV20 order by transcriptType; - cd kent/src/hg/makeDb/trackDb - cp human/hg38/wgEncodeGencodeV18.ra human/hg38/wgEncodeGencodeV20.ra - cp human/hg38/wgEncodeGencodeV18.html human/hg38/wgEncodeGencodeV20.html - - # edit these plus human/hg38/trackDb.wgEncode.ra - # - set priorities in wgEncodeGencodeV20.ra in reverse order with previous - # tracks so newest shows up first - # priority - set to previous version priority minus 0.001 - # searchPriority - set each to previous minus 0.001 - # - make current track default to pack and hide previous [ONLY if it's going to be pushed] - # superTrack wgEncodeGencodeSuper pack - # - Update wgEncodeGencodeSuper.html to describe new release and to - # pick up other updates. - - ### IMPORTANT: make sure that hgTracks/gencodeTracks.c registers - ### track handler for this version of gencode: - registerTrackHandler("wgEncodeGencodeV20", gencodeGeneMethods); - - # update all.joiner and validate - # look for the last section `begin Gencode V??' in all.joiner - # and copy and update version - # repeat this until happy, editing minCheck as needed - cd /hive/data/genomes/hg38/bed/gencodeV20 - make joinerCheck # output in check/joiner.out - - ############################################################################## # cloneEnds (DONE - 2014-08-14 - Hiram) mkdir /hive/data/genomes/hg38/bed/cloneEnds cd /hive/data/genomes/hg38/bed/cloneEnds # fetch the NCBI INSDC name correspondence file: rsync -a -P rsync://ftp.ncbi.nlm.nih.gov/genomes/ASSEMBLY_REPORTS/All/GCF_000001405.26.assembly.txt ./ # fetch the clone reports mkdir reports rsync -a -P \ rsync://ftp.ncbi.nih.gov/repository/clone/reports/Homo_sapiens/*.GCF_000001405.26.106.*.gff \ ./reports/ @@ -5901,79 +5846,30 @@ where chrom in ("chrX_KI270880v1_alt", "chrX_KI270913v1_alt");' #+---------------------+--------+ #| chrom | size | #+---------------------+--------+ #| chrX_KI270880v1_alt | 284869 | #| chrX_KI270913v1_alt | 274009 | #+---------------------+--------+ # Process that into bed4 with name=PAR1: hgsql hg38 -NBe 'select chrom, 0, size, "PAR1" from chromInfo \ where chrom in ("chrX_KI270880v1_alt", "chrX_KI270913v1_alt");' \ >> hg38Par.bed4 hgLoadBed hg38 par hg38Par.bed4 checkTableCoords hg38 par -######################################################################### -2014-10-16: import of UCSC GENCODE group processing of GENCODE V21 (markd) - # download files - mkdir -p /hive/data/genomes/hg38/bed/gencodeV21/data - cd /hive/data/genomes/hg38/bed/gencodeV21 - - # create Makefile from previous one. - # WARNING: next build start with hg/makeDb/outside/gencode/gencodeLoad.mk - cp /hive/data/genomes/hg38/bed/gencodeV20/Makefile . - - # download, build and load tables - (time nice make -j 10) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense. - make cmpRelease >gencode-cmp.tsv - - ## Copy and update trackDb files from previous release. - ## Change version and use lower priority so it sorts to top of - ## super track page. - ## Important to make sure filter attrs.transcriptType matches current set - ## figured out with - select distinct transcriptType from wgEncodeGencodeAttrsV21 order by transcriptType; - cd kent/src/hg/makeDb/trackDb - cp human/hg38/wgEncodeGencodeV18.ra human/hg38/wgEncodeGencodeV21.ra - cp human/hg38/wgEncodeGencodeV18.html human/hg38/wgEncodeGencodeV21.html - - # edit these plus human/hg38/trackDb.wgEncode.ra - # - set priorities in wgEncodeGencodeV21.ra in reverse order with previous - # tracks so newest shows up first - # priority - set to previous version priority minus 0.001 - # searchPriority - set each to previous minus 0.001 - # - make current track default to pack and hide previous [ONLY if it's going to be pushed] - # superTrack wgEncodeGencodeSuper pack - # - Update wgEncodeGencodeSuper.html to describe new release and to - # pick up other updates. - - ### IMPORTANT: make sure that hgTracks/gencodeTracks.c registers - ### track handler for this version of gencode: - registerTrackHandler("wgEncodeGencodeV21", gencodeGeneMethods); - - # update all.joiner and validate - # look for the last section `begin Gencode V??' in all.joiner - # and copy and update version - # repeat this until happy, editing minCheck as needed - cd /hive/data/genomes/hg38/bed/gencodeV21 - make joinerCheck # output in check/joiner.out - - ############################################################################# # LASTZ Cow bosTau8 (DONE - 2014-10-15 - Steve) mkdir /hive/data/genomes/hg38/bed/lastzBosTau8.2014-10-215 cd /hive/data/genomes/hg38/bed/lastzBosTau8.2014-10-15 cat << '_EOF_' > DEF # human vs cow # maximum M allowed with lastz is only 254 BLASTZ_M=254 # TARGET: Human hg38 SEQ1_DIR=/scratch/data/hg38/hg38.2bit SEQ1_LEN=/scratch/data/hg38/chrom.sizes SEQ1_CHUNK=10000000 SEQ1_LAP=10000 @@ -6259,75 +6155,30 @@ # remove the full date and add: # dataVersion Jan. 2015 # Scripts copied ucscRetroAli9.psl, ucscRetroInfo9.bed and ucscRetroCds9.tab # to /hive/data/genomes/hg38/bed/retro/ ########## # Make dbVar chrom to UCSC chrom lift file # DONE braney 2/12/15 cd /cluster/data/hg38/jkStuff sort /cluster/data/hg38/chrom.sizes > tmpChrom grep -v '^#\|assembled' /hive/data/genomes/hg38/genbank/GCF_000001405.26.assembly.txt | awk 'BEGIN {OFS="\t"} {print "chr" $3 "_" $5 "_" $2, "chr" $3 "|"$7}' | sed 's/-scaffold//' | sed 's/unlocalized/random/' | sed 's/_unplaced//' | sed 's/chrna/chrUn/g' | sed 's/\./v/' | sort | join /dev/stdin tmpChrom | awk 'BEGIN {OFS="\t"} {print 0, $2, $3, $1, $3}' > dbVar.lift awk 'BEGIN {OFS="\t"} {print 0, $1, $2, $1, $2}' /cluster/data/hg38/chrom.sizes >> dbVar.lift rm tmpChrom ######################################################################### -2015-03-13: import of UCSC GENCODE group processing of GENCODE V22 (markd) - # download files - mkdir -p /hive/data/genomes/hg38/bed/gencodeV22 - cd /hive/data/genomes/hg38/bed/gencodeV22 - - # edit hg/makeDb/outside/gencode/gencodeLoad.mk to set - # release and transcript support versions - - - # download, build and load tables - (time nice make -j 10 -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense. - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk cmpRelease >gencode-cmp.tsv - - ## Copy and update trackDb files from previous release. - ## Change version and use lower priority so it sorts to top of - ## super track page. - ## Important to make sure filter attrs.transcriptType matches current set - ## figured out with - select distinct transcriptType from wgEncodeGencodeAttrsV22 order by transcriptType; - cd kent/src/hg/makeDb/trackDb - cp human/hg38/wgEncodeGencodeV18.ra human/hg38/wgEncodeGencodeV22.ra - cp human/hg38/wgEncodeGencodeV18.html human/hg38/wgEncodeGencodeV22.html - - # edit these plus human/hg38/trackDb.wgEncode.ra - # - set priorities in wgEncodeGencodeV22.ra in reverse order with previous - # tracks so newest shows up first - # priority - set to previous version priority minus 0.001 - # searchPriority - set each to previous minus 0.001 - # - make current track default to pack and hide previous [ONLY if it's going to be pushed] - # superTrack wgEncodeGencodeSuper pack - # - Update wgEncodeGencodeSuper.html to describe new release and to - # pick up other updates. - - # update all.joiner and validate - # look for the last section `begin Gencode V??' in all.joiner - # and copy and update version - # repeat this until happy, editing minCheck as needed - # output in check/joiner.out - cd /hive/data/genomes/hg38/bed/gencodeV22 - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk joinerCheck - -######################################################################### # UCSC to RefSeq name correspondence (DONE - 2015-04-13 - Hiram) mkdir /hive/data/genomes/hg38/bed/ucscToRefSeq cd /hive/data/genomes/hg38/bed/ucscToRefSeq # columns 5 and 7 are the INSDC and RefSeq names grep -v "^#" ../../genbank/GCF_000001405.26.assembly.txt \ | awk -F'\t' '{printf "%s\t%s\n", $5,$7}' | sort > insdc.refSeq.tab hgsql -N -e 'select name,chrom,chromStart,chromEnd from ucscToINSDC;' hg38 \ | sort > insdc.ucsc.tab join insdc.ucsc.tab insdc.refSeq.tab | tr '[ ]' '[\t]' \ | cut -f2- > ucsc.refSeq.tab @@ -6344,202 +6195,49 @@ #CREATE MICROSAT TRACK (DONE - 2015-05-22 - Hiram) ssh hgwdev mkdir /cluster/data/hg38/bed/microsat cd /cluster/data/hg38/bed/microsat awk '($5==2 || $5==3) && $6 >= 15 && $8 == 100 && $9 == 0 {printf("%s\t%s\t%s\t%dx%s\n", $1, $2, $3, $6, $16);}' \ ../simpleRepeat/simpleRepeat.bed > microsat.bed hgLoadBed hg38 microsat microsat.bed ############################################################################# # ENCODE Regulatory tracks (Kate & Chris) # see reg.txt ######################################################################### -2015-10-02: import of UCSC GENCODE group processing of GENCODE V23 (markd) - # download files - mkdir -p /hive/data/genomes/hg38/bed/gencodeV23 - cd /hive/data/genomes/hg38/bed/gencodeV23 - - # edit hg/makeDb/outside/gencode/gencodeLoad.mk to set - # release and transcript support versions - - - # download, build and load tables - (time nice make -j 10 -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense. - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk cmpRelease >gencode-cmp.tsv - - ## Copy and update trackDb files from previous release. - ## Change version and use lower priority so it sorts to top of - ## super track page. - ## Important to make sure filter attrs.transcriptType matches current set - ## figured out with - select distinct transcriptType from wgEncodeGencodeAttrsV23 order by transcriptType; - cd kent/src/hg/makeDb/trackDb - cp human/hg38/wgEncodeGencodeV22.ra human/hg38/wgEncodeGencodeV23.ra - cp human/hg38/wgEncodeGencodeV22.html human/hg38/wgEncodeGencodeV23.html - - # edit these plus human/hg38/trackDb.wgEncode.ra - # - set priorities in wgEncodeGencodeV23.ra tracks so newest shows up first - # priority - set to previous version priority minus 0.001 - # searchPriority - set each to previous minus 0.001 - # - make current track default to pack and hide previous [ONLY if it's going to be pushed] - # superTrack wgEncodeGencodeSuper pack - # - Update wgEncodeGencodeSuper.html to describe new release and to - # pick up other updates. - - # update all.joiner and validate - # look for the last section `begin Gencode V??' in all.joiner - # and copy and update version - # repeat this until happy, editing minCheck as needed - # output in check/joiner.out - cd /hive/data/genomes/hg38/bed/gencodeV23 - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk joinerCheck - -######################################################################### -2015-12-08: import of UCSC GENCODE group processing of GENCODE V24 (markd) - # download files - mkdir -p /hive/data/genomes/hg38/bed/gencodeV24 - cd /hive/data/genomes/hg38/bed/gencodeV24 - - # edit hg/makeDb/outside/gencode/gencodeLoad.mk to set - # release and transcript support versions - - - # download, build and load tables - (time nice make -j 10 -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense (results in gencode-cmp.tsv) - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk cmpRelease - - ## Copy and update trackDb files from previous release. - ## Change version and use lower priority so it sorts to top of - ## super track page. Follow instructiuons in ra file to ensure - ## filters are correct. - cd kent/src/hg/makeDb/trackDb - cp human/hg38/wgEncodeGencodeV23.ra human/hg38/wgEncodeGencodeV24.ra - cp human/hg38/wgEncodeGencodeV23.html human/hg38/wgEncodeGencodeV24.html - - - # edit these plus human/hg38/trackDb.wgEncode.ra - # - set priorities in wgEncodeGencodeV24.ra tracks so newest shows up first - # priority - set to previous version priority minus 0.001 - # searchPriority - set each to previous minus 0.001 - # - make current track default to pack and hide previous [ONLY if it's going to be pushed] - # superTrack wgEncodeGencodeSuper pack - # - Update wgEncodeGencodeSuper.html to describe new release and to - # pick up other updates. - - # update all.joiner and validate - # look for the last section `begin Gencode V??' in all.joiner - # and copy and update version - # repeat this until happy, editing minCheck as needed - # output in check/joiner.out - ### NOT DONE SINCE TRACK NOT BEING PUSHED - cd /hive/data/genomes/hg38/bed/gencodeV24 - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk joinerCheck - -######################################################################### # GWIPS-viz Ribo-seq - (DONE - 2016-02-05 - Steve) # contact Audrey Michel (audreymannion@gmail.com) # redmine #16765 obtained bigWig file from shared Google drive https://drive.google.com/a/soe.ucsc.edu/folderview?id=0B_xvV_5tXzOGQ1h5NEh4bnhNTDg&usp=sharing_eid mkdir /hive/data/genomes/hg38/bed/gwipsvizRiboseq cp Global_RiboProElong.10_02_2016.bw /hive/data/genomes/hg38/bed/gwipsvizRiboseq/gwipsvizRiboseq.bw mkdir /gbdb/hg38/bbi/gwipsvizRiboseq cd /gbdb/hg38/bbi/gwipsvizRiboseq ln -s /hive/data/genomes/hg38/bed/gwipsvizRiboseq/gwipsvizRiboseq.bw gwipsvizRiboseq.bw hgsql hg38 create table gwipsvizRiboseq select * from gc5BaseBw; update gwipsvizRiboseq set fileName="/gbdb/hg38/bbi/gwipsvizRiboseq/gwipsvizRiboseq.bw" where fileName="/gbdb/hg38/bbi/gc5BaseBw/gc5Base.bw"; ######################################################################### -2016-07-23: import of UCSC GENCODE group processing of GENCODE V25 (markd) - # edit hg/makeDb/outside/gencode/gencodeLoad.mk to set release and ensembl versions - - # download, build and load tables - mkdir -p /hive/data/genomes/hg38/bed/gencodeV25 - pushd /hive/data/genomes/hg38/bed/gencodeV25 - (time nice make -j 10 -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense. Results are in gencode-cmp.tsv - - ## Copy and update trackDb files from previous release. - ## Change version and use lower priority so it sorts to top of - ## super track page. Follow instructions in ra file to ensure - ## filters are correct. - cd kent/src/hg/makeDb/trackDb - cp mouse/hg38/wgEncodeGencodeVM9.ra mouse/hg38/wgEncodeGencodeV25.ra - cp mouse/hg38/wgEncodeGencodeVM9.html mouse/hg38/wgEncodeGencodeV25.html - - # edit these plus mouse/hg38/trackDb.wgEncode.ra - # - set priorities in wgEncodeGencodeV25.ra tracks so newest shows up first - # priority - set to previous version priority minus 0.001 - # searchPriority - set each to previous minus 0.001 - # - make current track default to pack and hide previous [ONLY if it's going to be pushed] - # superTrack wgEncodeGencodeSuper pack - # - Update wgEncodeGencodeSuper.html to describe new release and to - # pick up other updates. [ONLY if it's going to be pushed] - - # update all.joiner and validate - # look for the last section `begin Gencode V??' in all.joiner - # and copy and update version - # repeat this until happy, editing minCheck as needed - # output in check/joiner.out - cd /hive/data/genomes/hg38/bed/gencodeV25 - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk joinerCheck - -######################################################################### -2017-04-16: import of UCSC GENCODE group processing of GENCODE V26 (markd) - # edit hg/makeDb/outside/gencode/gencodeLoad.mk to set release and ensembl versions - - # download, build and load tables - mkdir -p /hive/data/genomes/hg38/bed/gencodeV26 - pushd /hive/data/genomes/hg38/bed/gencodeV26 - (time nice make -j 10 -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense. Results are in gencode-cmp.tsv - - # generate trackDb and joiner blurb - pushd kent/src/hg/makeDb/trackDb - ../../makeDb/outside/gencode/gencodeGenerateTrackDbs hg38 26 88 'March 2017' - - # edit human/hg38/trackDb.wgEncode.ra to add new .ra file include - make DBS=hg38 - - # Update human/hg38/wgEncodeGencodeSuper.html and update 'Release Notes' - # to describe new release. [ONLY if it's going to be pushed] - - # edit all.joiner to add ~/tmp/gencodeV26.joiner - # verify with: - pushd /hive/data/genomes/hg38/bed/gencodeV26 - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk joinerCheck - - # commit all -######################################################################### # COSMIC v81 DONE Chris Eisenhart 2017-05-11 # Make a new COSCMIC track for hg19 mkdir /hive/data/outside/cosmic/hg38/v81 cd /hive/data/outside/cosmic/hg38/v81 # Get the new data sftp ceisenha@ucsc.edu@sftp-cancer.sanger.ac.uk # Login to SFTP server then run these commands get /files/grch38/cosmic/v81/CosmicMutantExport.tsv.gz # Remove the 'NS' fields, search for the \t after to exclude the E'NS'ST transcripts. zcat CosmicMutantExport.tsv.gz | sed 's/NS\t/\t/g' > cosMut.tsv # Use a script to convert to bed format. cosmicToBed cosMut.tsv cosMut.bed @@ -6852,104 +6550,30 @@ # Use a script to convert to bed format. cosmicToBed cosMut.tsv cosMut.bed # This many lines were skipped, 134601 for not having genomic coordinate # Sort and convert to big bed using the .as file. sort -k1,1 -k2,2n cosMut.bed > sCosMut.bed bedToBigBed -type=bed8+31 -as=cosmicNew.as sCosMut.bed /hive/data/genomes/hg38/chrom.sizes cosMutHg38V82.bb -tab -extraIndex=name,cosmLabel # Link it up so the outside world can see it. cd /gbdb/hg38/cosmic/ ln -s /hive/data/outside/cosmic/hg38/v82/cosMutHg38V82.bb . ######################################################################### -2017-09-14: import of UCSC GENCODE group processing of GENCODE V27 (markd) - # edit hg/makeDb/outside/gencode/gencodeLoad.mk to set release and ensembl versions - - # download, build and load tables - mkdir -p /hive/data/genomes/hg38/bed/gencodeV27 - pushd /hive/data/genomes/hg38/bed/gencodeV27 - (time nice make -j 10 -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense. Results are in gencode-cmp.tsv - - # generate trackDb and joiner blurb - pushd kent/src/hg/makeDb/trackDb - ../../makeDb/outside/gencode/gencodeGenerateTrackDbs hg38 27 90 'Aug 2017' - - # Update human/hg38/wgEncodeGencodeSuper.html and update 'Release Notes' - # to describe new release. [ONLY if it's going to be pushed] - - # edit human/hg38/trackDb.wgEncode.ra to add new .ra file include - make DBS=hg38 - - # edit all.joiner to add ~/tmp/gencodeV27.joiner - # verify with: - pushd /hive/data/genomes/hg38/bed/gencodeV27 - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk joinerCheck - - # commit all - -######################################################################### -2018-03-07: update UCSC GENCODE V27 to include protein id (for VAI) and fix PAR tag - -# save existing data, move a -mkdir -p prev/pre-proteinId -mv data/gencode.tab prev/pre-proteinId/ -mv tables/wgEncodeGencodeAttrsV27.tab tables/wgEncodeGencodeTagV27.tab prev/pre-proteinId/ -mv loaded/wgEncodeGencodeAttrsV27.tab.loaded loaded/wgEncodeGencodeTagV27.tab.loaded prev/pre-proteinId/ - -# rebuild, getting protein ids. -make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk loadTables >&prot-ids.log - -# 2018-03-19: update search to include protein id -cd kent/src/hg/makeDb/trackDb -../../makeDb/outside/gencode/gencodeGenerateTrackDbs hg38 27 90 'Aug 2017' - -######################################################################### -2018-04-11: import of UCSC GENCODE group processing of GENCODE V28 (markd) - # edit hg/makeDb/outside/gencode/gencodeLoad.mk to set release and ensembl versions - - # download, build and load tables - mkdir -p /hive/data/genomes/hg38/bed/gencodeV28 - pushd /hive/data/genomes/hg38/bed/gencodeV28 - (time nice make -j 10 -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense. Results are in gencode-cmp.tsv - - # generate trackDb and joiner blurb - pushd kent/src/hg/makeDb/trackDb - ../../makeDb/outside/gencode/gencodeGenerateTrackDbs hg38 28 92 'Apr 2018' - - # Update human/hg38/wgEncodeGencodeSuper.html and update 'Release Notes' - # to describe new release. [ONLY if it's going to be pushed] - - # edit human/hg38/trackDb.wgEncode.ra to add new .ra file include - make DBS=hg38 - - # edit all.joiner to add ~/tmp/gencodeV28.joiner - # verify with: - pushd /hive/data/genomes/hg38/bed/gencodeV28 - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk joinerCheck - - # commit all - -######################################################################### # RepeatMasker Visualization track update (DONE - 2018-05-04 - ChrisL) screen -S rmskJoined.2018-05-04 mkdir /hive/data/genomes/hg38/bed/rmskJoined.2018-05-04 cd /hive/data/genomes/hg38/bed/rmskJoined.2018-05-04 ln -s ../repeatMasker/hg38.sorted.fa.out . ln -s ../repeatMasker/hg38.fa.align.gz . # this script points to the most recent RepeatMasker version: time (/scratch/data/RepeatMasker/util/rmToUCSCTables.pl \ -out hg38.sorted.fa.out -align hg38.fa.align.gz) > do.log 2>&1 & # no differences, forgot to remake rmsk files # so instead remake the rmsk track and try again mkdir /hive/data/genomes/hg38/bed/repeatMasker.2018-05-04 @@ -7005,150 +6629,15 @@ hgLoadBed -sqlTable=$HOME/kent/src/hg/lib/rmskJoined.sql \ -renameSqlTable -verbose=4 -tab \ -type=bed4+9 -as=$HOME/kent/src/hg/lib/rmskJoined.as hg38 \ rmskJoinedCurrent hg38.sorted.fa.join.tsv \ > loadJoined.log 2>&1 hgLoadSqlTab hg38 rmskAlignCurrent \ /cluster/home/chmalee/kent/src/hg/lib/rmskAlign.sql \ hg38.fa.align.tsv > loadAlign.log 2>&1 hgLoadOutJoined -verbose=2 -table=rmskOutCurrent hg38 hg38.sorted.fa.out > loadOut.log 2>&1 featureBits -countGaps hg38 rmskJoinedCurrent # 2796899855 bases of 3209286105 (87.150%) in intersection ######################################################################### -2018-09-05: import of UCSC GENCODE group processing of GENCODE V29 (markd) - 2018-11-09 this was the prerelease and the /hive/data/genomes/hg38/bed/gencodeV29 - was renamed to /hive/data/genomes/hg38/bed/gencodeV29.pre when the full release was - downloaded. - - # edit hg/makeDb/outside/gencode/gencodeLoad.mk to set release and ensembl versions - - # This is a pre-release from - # ftp://ftp.ebi.ac.uk/pub/databases/havana/gencode_pre/Gencode_human/release_29/ - - # download, build and load tables - mkdir -p /hive/data/genomes/hg38/bed/gencodeV29 - pushd /hive/data/genomes/hg38/bed/gencodeV29 - - # pre-release contained invalid biotype, fix for now - mv gencode.v29.chr_patch_hapl_scaff.annotation.gtf.gz gencode.v29.chr_patch_hapl_scaff.annotation.orig.gtf.gz - zcat gencode.v29.chr_patch_hapl_scaff.annotation.orig.gtf.gz | sed -e 's/"lincrna"/"lincRNA"/g' | gzip -c> gencode.v29.chr_patch_hapl_scaff.annotation.gtf.gz - - (time nice make -j 10 -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense. Results are in gencode-cmp.tsv - - # generate trackDb and joiner blurb - pushd ~/kent/src/hg/makeDb/trackDb - ../../makeDb/outside/gencode/gencodeGenerateTrackDbs hg38 29 94 'Sept 2018' - - # Edit human/hg38/wgEncodeGencodeSuper.html and update 'Release Notes' - # to describe new release. [ONLY if it's going to be pushed] - - # edit human/hg38/trackDb.wgEncode.ra to add new .ra file include - make DBS=hg38 - - # edit all.joiner to add ~/tmp/gencodeV29.joiner - # verify with: - pushd /hive/data/genomes/hg38/bed/gencodeV29 - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk joinerCheck - - # commit all - -######################################################################### -2018-11-09: import of UCSC GENCODE group processing of GENCODE V29 (markd) - # edit hg/makeDb/outside/gencode/gencodeLoad.mk to set release and ensembl versions - - # download, build and load tables - mkdir -p /hive/data/genomes/hg38/bed/gencodeV29 - pushd /hive/data/genomes/hg38/bed/gencodeV29 - - (time nice make -j 10 -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense. Results are in gencode-cmp.tsv - - # generate trackDb and joiner blurb - pushd ~/kent/src/hg/makeDb/trackDb - ../../makeDb/outside/gencode/gencodeGenerateTrackDbs hg38 29 94 'Oct 2018' - - # Edit human/hg38/wgEncodeGencodeSuper.html and update 'Release Notes' - # to describe new release. [ONLY if it's going to be pushed] - - # edit human/hg38/trackDb.wgEncode.ra to add new .ra file include - make DBS=hg38 - - # edit all.joiner to add ~/tmp/gencodeV29.joiner - # verify with: - pushd /hive/data/genomes/hg38/bed/gencodeV29 - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk joinerCheck - - # commit all - -######################################################################### -2019-04-08: import of UCSC GENCODE group processing of GENCODE V30 (markd) - # edit hg/makeDb/outside/gencode/gencodeLoad.mk to set release and ensembl versions - - # download, build and load tables - mkdir -p /hive/data/genomes/hg38/bed/gencodeV30 - pushd /hive/data/genomes/hg38/bed/gencodeV30 - - (time nice make -j 10 -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense. Results are in gencode-cmp.tsv - - # generate trackDb and joiner blurb - pushd ~/kent/src/hg/makeDb/trackDb - ../../makeDb/outside/gencode/gencodeGenerateTrackDbs hg38 30 96 'Apr 2019' - - # Edit human/hg38/wgEncodeGencodeSuper.html and update 'Release Notes' - # to describe new release. [ONLY if it's going to be pushed] - - # edit human/hg38/trackDb.gencode.ra to add new .ra file include - make DBS=hg38 - - ## only if being pushed to RR: - # edit all.joiner to add ~/tmp/gencodeV30.joiner - # verify with: - pushd /hive/data/genomes/hg38/bed/gencodeV30 - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk joinerCheck - - # commit all - -######################################################################### -2019-05-17: import of UCSC GENCODE group processing of GENCODE V31 (markd) - # import of pre-release, it will be promoted if there are no changes in the final release. - # edit hg/makeDb/outside/gencode/gencodeLoad.mk to set release and ensembl versions - - # download, build and load tables - mkdir -p /hive/data/genomes/hg38/bed/gencodeV31-pre - pushd /hive/data/genomes/hg38/bed/gencodeV31-pre - - (time nice make -j 10 -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk) >&build.1.out& - - # compare tables from previous release to see if number changed makes - # sense. Results are in gencode-cmp.tsv - - # generate trackDb and joiner blurb - pushd ~/kent/src/hg/makeDb/trackDb - ../../makeDb/outside/gencode/gencodeGenerateTrackDbs hg38 31 97 'June 2019' - - # Edit human/hg38/wgEncodeGencodeSuper.html and update 'Release Notes' - # to describe new release. [ONLY if it's going to be pushed] - - # edit human/hg38/trackDb.gencode.ra to add new .ra file include - make DBS=hg38 - - ## only if being pushed to RR: - # edit all.joiner to add ~/tmp/gencodeV31.joiner - # verify with: - pushd /hive/data/genomes/hg38/bed/gencodeV31 - make -f ~/kent/src/hg/makeDb/outside/gencode/gencodeLoad.mk joinerCheck - - # commit all - # if pushing public, add ticket and MARK QA READY - -#########################################################################