9cedfa38c14068c79dec89f76c606ee22b3f931a max Wed Sep 30 15:02:37 2026 -0700 phasedVars: new subtrack hgdp1kSnv, a 17GB version of the 3.5TB gnomAD HGDP+1000G genotype VCF with only SNVs with AC>5 and only GT, so haplotype clustering can be shown up to 5Mbp, refs #37306 diff --git src/hg/makeDb/doc/hg38/varFreqs.txt src/hg/makeDb/doc/hg38/varFreqs.txt index 7c7601c5dc2..f0ec5164cdd 100644 --- src/hg/makeDb/doc/hg38/varFreqs.txt +++ src/hg/makeDb/doc/hg38/varFreqs.txt @@ -184,30 +184,42 @@ # Note: first attempt kept all fields -> 169GB, too large. This version keeps only continental groups. cd /hive/data/genomes/hg38/bed/varFreqs/hgdp1kFreq/ KEEP="INFO/AC,INFO/AF,INFO/AN,INFO/nhomalt,INFO/gnomad_AC,INFO/gnomad_AF,INFO/gnomad_AN,INFO/gnomad_AC_afr,INFO/gnomad_AF_a fr,INFO/gnomad_AN_afr,INFO/gnomad_AC_ami,INFO/gnomad_AF_ami,INFO/gnomad_AN_ami,INFO/gnomad_AC_amr,INFO/gnomad_AF_amr,INFO/g nomad_AN_amr,INFO/gnomad_AC_asj,INFO/gnomad_AF_asj,INFO/gnomad_AN_asj,INFO/gnomad_AC_eas,INFO/gnomad_AF_eas,INFO/gnomad_AN_ eas,INFO/gnomad_AC_fin,INFO/gnomad_AF_fin,INFO/gnomad_AN_fin,INFO/gnomad_AC_mid,INFO/gnomad_AF_mid,INFO/gnomad_AN_mid,INFO/ gnomad_AC_nfe,INFO/gnomad_AF_nfe,INFO/gnomad_AN_nfe,INFO/gnomad_AC_oth,INFO/gnomad_AF_oth,INFO/gnomad_AN_oth,INFO/gnomad_AC _sas,INFO/gnomad_AF_sas,INFO/gnomad_AN_sas,INFO/gnomad_popmax,INFO/gnomad_faf95_popmax" # This took days to complete, so asked Claude to make it parallel #bcftools view -G /gbdb/hg38/phasedVars/hgdp1k/gnomad.genomes.v3.1.2.hgdp_tgp.vcf.gz --threads 8 \ #| bcftools annotate -x "^${KEEP}" -Oz --threads 4 -o hgdp1k.freq.vcf.gz # use 30 threads, and chunks of 50 Mbp sh ~/kent/src/hg/makeDb/scripts/varFreqs/vcfFilterParallel.sh /gbdb/hg38/phasedVars/hgdp1k/gnomad.genomes.v3.1.2.hgdp_tgp.vcf.gz hgdp1k.freq.parallel.vcf.gz "$KEEP" 30 50 & tabix -p vcf hgdp1k.freq.vcf.gz +# phasedVars hgdp1kSnv: smaller genotype VCF for HGDP1k, Claude/Max, Sep 29 2026 +# The 3.5TB genotype VCF is too slow for haplotype clustering at higher zoom levels. +# Keep only SNVs with INFO/AC>5, only FORMAT/GT and INFO/AC,AN,AF. 60 jobs, 10 Mbp chunks. +# Test region chr20:40-41Mbp: 57698 variants -> 11310, 4.8MB +cd /hive/data/genomes/hg38/bed/varFreqs/hgdp1k/ +sh ~/kent/src/hg/makeDb/scripts/varFreqs/hgdp1kCommonSnvs.sh gnomad.genomes.v3.1.2.hgdp_tgp.vcf.gz hgdp1k.snvAc6.vcf.gz 60 10 > hgdp1kCommonSnvs.log 2>&1 +# took 45 minutes, 3.5TB -> 17GB +bcftools index -n gnomad.genomes.v3.1.2.hgdp_tgp.vcf.gz +# 189381961 +bcftools index -n hgdp1k.snvAc6.vcf.gz +# 34033926 + # Swefreq, Max, Feb 2026 # downloaded files from https://swefreq.nbis.se/dataset/SweGen/download # Access was approved through the website, but I emailed swefreq@scilifelab.se, it needed a reminder email # Also got email from adam.ameur@igp.uu.se with followup info and do-no-allow-downloads instruction cd /hive/data/genomes/hg38/bed/varFreqs/swefreq # Indigenomes, Max Jan 2026 # downloaded from https://clingen.igib.res.in/indigen/, used as-is cd /hive/data/genomes/hg38/bed/varFreqs/indigenomes/ # Japan Tommo 60k, Max Jan 2026 # downloaded from https://jmorp.megabank.tohoku.ac.jp/downloads cd /hive/data/genomes/hg38/bed/varFreqs/tommo61kjpn/ # copied urls from website wget -i urls.txt