2fb68bdc7e7512791f7dfa748dc659c59bed62cb hiram Tue May 26 20:12:50 2026 -0700 cleanup and make MAF download files refs #31811 diff --git src/hg/makeDb/doc/vgp577way/vgp577way.txt src/hg/makeDb/doc/vgp577way/vgp577way.txt index 5320415f769..097debd80aa 100644 --- src/hg/makeDb/doc/vgp577way/vgp577way.txt +++ src/hg/makeDb/doc/vgp577way/vgp577way.txt @@ -1,57 +1,61 @@ ######################################################################## ### bringing in the Cactus 577-way alignments - April 2026 - Hiram ######################################################################## ### ### The maf files produced by Glenn come from the GI Prism system ### Requires the Prism VPN system in operation to get into it. ### Go to the 'emerald' machine, in the directory: ### /private/home/ghickey/dev/work/vgp-cactus/577way/ ### The single.maf.gz files available there as of 2026-04-12: ls *.single.maf.gz | grep -v Anc | xargs du -ksc | sort -n 4392699 vgp-577way-v1-european_river_lamprey.single.maf.gz +27378844 vgp-577way-v1-mexican_tetra.single.maf.gz 27465651 vgp-577way-v1-catshark.single.maf.gz 27956344 vgp-577way-v1-spotted_gar.single.maf.gz 28298848 vgp-577way-v1-zebrafish.single.maf.gz +33433099 vgp-577way-v1-coastal_tailed_frog.single.maf.gz 33562001 vgp-577way-v1-coelacanth.single.maf.gz 34730114 vgp-577way-v1-european_eel.single.maf.gz 36304442 vgp-577way-v1-tiny_cayenne_caecilian.single.maf.gz 44031743 vgp-577way-v1-three_spined_stickleback.single.maf.gz 44954739 vgp-577way-v1-clawed_frog.single.maf.gz 45852924 vgp-577way-v1-eastern_happy.single.maf.gz 62915764 vgp-577way-v1-brown_anole.single.maf.gz 95233208 test-vgp-577way-v1-gray_short_tailed_opossum.single.maf.gz 96201924 vgp-577way-v1-gray_short_tailed_opossum.single.maf.gz 105300666 vgp-577way-v1-chicken.single.maf.gz 109506066 vgp-577way-v1-zebra_finch.single.maf.gz 113391936 vgp-577way-v1-green_sea_turtle.single.maf.gz 123361778 vgp-577way-v1-emu.single.maf.gz 160086223 vgp-577way-v1-mm39.single.maf.gz 198519571 vgp-577way-v1-horseshoe_bat.single.maf.gz 203025701 vgp-577way-v1-hg38.single.maf.gz 203127845 vgp-577way-v1-hs1.single.maf.gz 203725553 vgp-577way-v1-dog.single.maf.gz 2001945735 total ls -og *.single.maf.gz | grep -v Anc | sed -e 's#^-rw-r--r-- 1 \+##;' | sort -n 4498122844 Apr 18 01:29 vgp-577way-v1-european_river_lamprey.single.maf.gz +28035936114 Apr 24 21:36 vgp-577way-v1-mexican_tetra.single.maf.gz 28124825995 Mar 20 10:07 vgp-577way-v1-catshark.single.maf.gz 28627295445 Mar 22 04:32 vgp-577way-v1-spotted_gar.single.maf.gz 28978019428 Mar 20 07:42 vgp-577way-v1-zebrafish.single.maf.gz +34235492910 Apr 27 21:21 vgp-577way-v1-coastal_tailed_frog.single.maf.gz 34367488940 Apr 18 10:24 vgp-577way-v1-coelacanth.single.maf.gz 35563635801 Mar 22 21:31 vgp-577way-v1-european_eel.single.maf.gz 37175748583 Apr 18 18:27 vgp-577way-v1-tiny_cayenne_caecilian.single.maf.gz 45088504216 Apr 13 01:51 vgp-577way-v1-three_spined_stickleback.single.maf.gz 46033651796 Mar 22 01:44 vgp-577way-v1-clawed_frog.single.maf.gz 46953393984 Apr 12 18:09 vgp-577way-v1-eastern_happy.single.maf.gz 64425741849 Mar 20 17:37 vgp-577way-v1-brown_anole.single.maf.gz 97518804112 Apr 20 09:32 test-vgp-577way-v1-gray_short_tailed_opossum.single.maf.gz 98510769939 Apr 17 22:16 vgp-577way-v1-gray_short_tailed_opossum.single.maf.gz 107827881545 Mar 17 19:39 vgp-577way-v1-chicken.single.maf.gz 112134211553 Mar 22 14:58 vgp-577way-v1-zebra_finch.single.maf.gz 116113341550 Apr 16 08:18 vgp-577way-v1-green_sea_turtle.single.maf.gz 126322459905 Mar 22 08:24 vgp-577way-v1-emu.single.maf.gz 163928292101 Apr 9 16:36 vgp-577way-v1-mm39.single.maf.gz 203284040383 Mar 23 14:15 vgp-577way-v1-horseshoe_bat.single.maf.gz @@ -174,62 +178,64 @@ CPU time in finished jobs: 3427s 57.11m 0.95h 0.04d 0.000 y IO & Wait Time: 185s 3.09m 0.05h 0.00d 0.000 y Average job time: 125s 2.08m 0.03h 0.00d Longest finished job: 690s 11.50m 0.19h 0.01d Submission to last job: 701s 11.68m 0.19h 0.01d ######################################################################## ### create iRows cd /hive/data/genomes/asmHubs/refseqBuild/GCF/037/176/765/GCF_037176765.1_rAnoSag1.mat/trackData/vgp577way mkdir iRows cd iRows ~/kent/src/hg/makeDb/doc/vgp577way/linkSizes.sh + 577 sizes + ~/kent/src/hg/makeDb/doc/vgp577way/mkNbeds.sh 577 nBeds 577 sizes 1154 total real 0m29.322s ### those scripts make up a large number of symLinks here, ### a directory nBedDir, a 'sizes' file and the 'nBeds' file: lrwxrwxrwx 1 90 Apr 5 12:03 GCA_003287225.2.len -> /hive/data/genomes/asmHubs/GCA/003/287/225/GCA_003287225.2/GCA_003287225.2.chrom.sizes.txt lrwxrwxrwx 1 90 Apr 5 12:03 GCA_005190385.3.len -> /hive/data/genomes/asmHubs/GCA/005/190/385/GCA_005190385.3/GCA_005190385.3.chrom.sizes.txt ... lrwxrwxrwx 1 35 Apr 5 12:03 hg38.len -> /hive/data/genomes/hg38/chrom.sizes lrwxrwxrwx 1 34 Apr 5 12:03 hs1.len -> /hive/data/genomes/hs1/chrom.sizes lrwxrwxrwx 1 35 Apr 5 12:03 mm39.len -> /hive/data/genomes/mm39/chrom.sizes -rw-rw-r-- 1 11506 Apr 5 12:03 sizes lrwxrwxrwx 1 27 Apr 5 12:04 GCA_003287225.2.bed -> nBedDir/GCA_003287225.2.bed lrwxrwxrwx 1 27 Apr 5 12:04 GCA_005190385.3.bed -> nBedDir/GCA_005190385.3.bed ### Note: as a result of the large number of symLinks in this directory, ### it is very slow to create a 'ls' listing in this directory. ls -S ../ucscMaf/*.maf > maf.list mkdir -p result # use the full path to the 2bit file for this operation: ~/kent/src/hg/makeDb/doc/vgp577way/mkIRowsJL.sh \ /hive/data/genomes/asmHubs/refseqBuild/GCF/037/176/765/GCF_037176765.1_rAnoSag1.mat/trackData/addMask/GCF_037176765.1_rAnoSag1.mat.masked.2bit > jobList -### 54g seems about right -para -ram=54g create jobList +### 64g seems about right +para -ram=64g create jobList para push ### when done para time > run.time cat run.time Completed: 29 of 29 jobs CPU time in finished jobs: 10508s 175.13m 2.92h 0.12d 0.000 y IO & Wait Time: 692s 11.54m 0.19h 0.01d 0.000 y Average job time: 386s 6.44m 0.11h 0.00d Longest finished job: 3092s 51.53m 0.86h 0.04d Submission to last job: 3398s 56.63m 0.94h 0.04d ######################################################################## ### construct bigMaf from iRows result mkdir /hive/data/genomes/asmHubs/refseqBuild/GCF/037/176/765/GCF_037176765.1_rAnoSag1.mat/trackData/vgp577way/bigMaf cd /hive/data/genomes/asmHubs/refseqBuild/GCF/037/176/765/GCF_037176765.1_rAnoSag1.mat/trackData/vgp577way/bigMaf @@ -376,45 +382,44 @@ sed -e "s/GC\([AF]\)\([0-9]\+\)v/GC\\1_\\2./g;" "${B}.tab" \ > ../${result} rm -f "${B}.tab" ' > runOne chmod +x runOne ### find optimum ram setting here para -ram=32g create jobList para push ### when finished para time > run.time ls result/*.bed | xargs cut -f2- \ - | $HOME/bin/x86_64/gnusort --parallel=32 -k1,1 -k2,2n > vgp577waySummary.bed + | $HOME/bin/x86_64/gnusort --parallel=32 -k1,1 -k2,2n | + gzip -c > vgp577waySummary.bed.gz ### makes a big file: -rw-rw-r-- 1 4157628035 Apr 24 14:41 vgp577waySummary.bed bedToBigBed -type=bed3+4 -as=$HOME/kent/src/hg/lib/mafSummary.as -tab \ - vgp577waySummary.bed ../../../*.chrom.sizes \ + vgp577waySummary.bed.gz ../../../*.chrom.sizes \ ../`pwd -P | sed -e "s#.*/GC\([AF]\)_#GC\1_#;" | cut -d"_" -f1-2`.vgp577waySummary.bb bigBedInfo ../*.vgp577waySummary.bb -### gzip the bed file: -gzip vgp577waySummary.bed +### the bed file was gzipped: -rw-rw-r-- 1 679264107 Apr 24 14:41 vgp577waySummary.bed.gz - ######################################################################## ### construct the trackDb cd /hive/data/genomes/asmHubs/refseqBuild/GCF/037/176/765/GCF_037176765.1_rAnoSag1.mat/trackData/vgp577way ~/kent/src/hg/makeDb/doc/vgp577way/mkTdb.sh ## this makes the file -rw-rw-r-- 1 39887 Apr 11 23:08 vgp577way.trackDb.txt ## symlink this track into the contrib directory: ### or, for hg38, hs1, mm39 - take it over to trackDb to incorporate the tracks ### and the bb symlinks go into, for example: ### mkdir -p /gbdb/mm39/vgp577way @@ -497,20 +502,50 @@ GCF_000001405.40 chrX:15551051-15610942 GCF_037176765.1 chr3:90678726-90740012 ### establish a saved session and put it into the session.txt file: cat sessionLink.txt GCA_949316315.1 https://genome-test.gi.ucsc.edu/s/Hiram/GCA_949316315.1.vgp577way GCA_964198595.1 https://genome-test.gi.ucsc.edu/s/Hiram/GCA_964198595.1.vgp577way GCF_902713615.1 https://genome-test.gi.ucsc.edu/s/Hiram/GCF_902713615.1.vgp577way GCA_009914755.4 https://genome-test.gi.ucsc.edu/s/Hiram/hs1.vgp577way GCF_000001405.40 https://genome-test.gi.ucsc.edu/s/Hiram/hg38.vgp577way GCF_037176765.1 https://genome-test.gi.ucsc.edu/s/Hiram/GCF_037176765.1.vgp577way ### then run: ./mkIndex.sh > index.html - ### It is now available as a link fro the index page: + ### It is now available as a link from the index page: https://hgwdev-hiram.gi.ucsc.edu/~hiram/VGP/vgp577way/ ######################################################################## +### clean up and construct download maf files: +rm -fr ucscMaf & +rm -fr split & +rm -fr bigMaf/*.bigMaf.gz & +cd iRows/result +# the -P 5 runs 5 gzips at a time in parallel +ls -S *.maf | xargs -P 5 -I{} gzip {} > ../../gzip.log 2>&1 & +wait +### when the gzip is finished +md5sum *.maf.gz > md5sum.txt + +######################################################################## +### when the rsync is done to hgdownload and the trackDb has been remade +### +pwd -P | sed -e "s#.*/GC\([AF]\)_#GC\\1_#;" | cut -d"_" -f1-2 + +og ../../contrib/vgp577way/`pwd -P | sed -e "s#.*/GC\([AF]\)_#GC\\1_#;" | cut -d"_" -f1-2`.*.b[bw] +rm ../../contrib/vgp577way/`pwd -P | sed -e "s#.*/GC\([AF]\)_#GC\\1_#;" | cut -d"_" -f1-2`.*.b[bw] + +ln -s `pwd`/iRows/result /hive/data/genomes/asmHubs/VGP/vgp577way/maf/`pwd -P | sed -e "s#.*/GC\([AF]\)_#GC\\1_#;" | cut -d"_" -f1-2` + +og /hive/data/genomes/asmHubs/VGP/vgp577way/maf + +rsync -L -P -a /hive/data/genomes/asmHubs/VGP/vgp577way/maf/ qateam@hgdownload1:/data/hubs/VGP/vgp577way/maf/ +rsync -L -P -a /hive/data/genomes/asmHubs/VGP/vgp577way/maf/ qateam@hgdownload3:/data/hubs/VGP/vgp577way/maf/ + +### refresh the trackDb, the hub files and push them out to hgdownload +ottoBuildGenArkHub.py \ + `pwd -P | sed -e "s#.*/GC\([AF]\)_#GC\\1_#;" | cut -d"_" -f1-2` +