f7cf3eee5dc2134d550c6b35b2fb250f65673341
max
  Tue Feb 3 01:55:53 2026 -0800
changes after code review, refs #37059

diff --git src/hg/makeDb/doc/hg38/varFreqs.txt src/hg/makeDb/doc/hg38/varFreqs.txt
index 0905138bf45..0cdd3582958 100644
--- src/hg/makeDb/doc/hg38/varFreqs.txt
+++ src/hg/makeDb/doc/hg38/varFreqs.txt
@@ -1,121 +1,109 @@
 # Mexico Biobank, Max, Nov 8 2025
 CrossMap.py vcf /gbdb/hg19/liftOver/hg19ToHg38.over.chain.gz /hive
 /data/genomes/hg19/bed/varFreqs/mexbb/MXBv2.vcf.gz /hive/data/genomes/hg38/p14Clean/hg38.p14.fa MXBv2.lift.hg19ToHg38.vcf && bgzip MXBv2.lift.hg19ToHg38.vcf && bcftools sort MXBv2.lift.hg19ToHg38.vcf -Oz -m 200G -T /data/tmp/ -o MXBv2.lift.hg19ToHg38.vcf.gz && tabix -p vcf MXBv2.lift.hg19ToHg38.vcf.gz
 
 # Mexico City Prospective study, Max Oct 28 2025
 cd /hive/data/genomes/hg38/bed/varFreqs/mcps/
 for i in `seq 1 22` X; do wget https://rgc-mcps.regeneron.com/downloads/20230130/chr$i.freq.vcf.gz; done
 for i in `seq 1 22` X; do wget https://rgc-mcps.regeneron.com/downloads/20230130/chr$i.freq.vcf.gz.tbi; done
 mv *vcf* vcf/
 bcftools concat  --threads 16  -Oz -o mcps.freq.vcf.gz vcf/chr{1..22}.freq.vcf.gz vcf/chrX.freq.vcf.gz
 # make normal AC and AF and AN fields for mouseovers
 zcat mcps.freq.vcf.gz | sed -e 's/_RAW//g' > mcps.fix.freq.vcf
 mv -f mcps.fix.freq.vcf mcps.freq.vcf
 bgzip mcps.freq.vcf
 tabix -p vcf mcps.freq.vcf.gz 
 
 # Regeneron million exomes, Max, Nov 3 2025
 cd /hive/data/genomes/hg38/bed/varFreqs/me
 for i in `seq 1 22` X Y; do wget https://rgc-research.regeneron.com/me/downloads/20231004/rgc_me_variant_frequencies_chr${i}_20231004.vcf.gz.tbi; done
 bcftools concat  --threads 10  -Oz -o rgc_me_freqs_20231004.vcf.gz rgc_me_variant_frequencies_chr{1..22}_20231004.vcf.gz  rgc_me_variant_frequencies_chrX_20231004.vcf.gz rgc_me_variant_frequencies_chrY_20231004.vcf.gz 
 zcat rgc_me_freqs_20231004.vcf.gz | sed -e 's/ALL_//g' > rgc_me_freqs_20231004.fix.vcf
 tabix -p vcf rgc_me_freqs_20231004.vcf.gz
 
 # GA south asia 100k pilot
 cd /hive/data/genomes/hg38/bed/varFreqs/ga100k/
 parallel -j 8 wget -q --no-check-certificate https://browser.genomeasia100k.org/service/web/download_files/{}.substitutions.annot.cont_withmaf.vcf.gz ::: {1..22} X Y
 # fix the header line, remove "FORMAT"
 for i in *.vcf.gz; do echo "zcat $i |   awk 'BEGIN{OFS=\"\\t\"} /^#CHROM/{NF=8; print; next} /^#/ {print; next} {NF=8; print}' |   bgzip -c > fixed/$i" >> cmds.txt; done
 parallel -j 8 < cmds.txt
 bcftools concat  --threads 16  -Oz -o ../ga100k.subst.vcf.gz fixed/{1..22}.substitutions.annot.cont_withmaf.vcf.gz
 # add indels
 wget -q --no-check-certificate https://browser.genomeasia100k.org/service/web/download_files/All.indels.annot.cont_withmaf.vcf.gz
 # index
 tabix -p vcf ../ga100k*.vcf.gz
 tabix -p vcf All*.vcf.gz
 
 # TOPMED Freeze 10
 cd /hive/data/genomes/hg38/bed/varFreqs/topmed/
 # need to download the VCFs manually, 22 VCFs, with one time links from https://bravo.sph.umich.edu/vcfs.html
 # grrrr...
 bcftools concat  --threads 10  -Oz -o topmed10.vcf.gz {1..22}.vcf.gz X.vcf.gz 
 tabix -p vcf topmed10.vcf.gz
 
 # Abraom brazil
 # get unique download link from https://abraom.ib.usp.br/download/index.php
 cd /hive/data/genomes/hg38/bed/varFreqs/abraom/
 wget 'https://abraom.ib.usp.br/download/download-files.php?fid=RklEMTIzNDU2&key=1762266466-key690a0d62348de0.22872232' -O abraom.tar
 tar xvfz abraom.tar
 ln -s  /hive/data/genomes/hg38/p14Clean/hg38.p14.fa
 samtools faidx hg38.p14.fa 
 python ~/kent/src/hg/makeDb/scripts/abraomToVcf.py SABE1171.Abraom.clean.tsv abraom.vcf hg38.p14.fa
 tabix -p vcf abraom.vcf.gz 
 
 # SGDP
 cd /hive/data/genomes/hg38/bed/varFreqs/sgp/
 CrossMap.py vcf /gbdb/hg19/liftOver/hg19ToHg38.over.chain.gz /hive/data/genomes/hg19/bed/varFreqs/sgdp/SGDP.nh2.vcf.gz hg38.p14.fa sgdp.hg38.nh2.vcf
 bgzip sgdp.hg38.nh2.vcf
 bcftools sort sgdp.hg38.nh2.vcf.gz -Oz -m 200G -T /data/tmp/ -o sgdp.hg38.nh2.sort.vcf.gz 
 mv sgdp.hg38.nh2.sort.vcf.gz SGDP.nh2.vcf.gz
 tabix -p vcf SGDP.nh2.vcf.gz
 
 # KOVA
 cd /hive/data/genomes/hg38/bed/varFreqs/sgp/
 # got tsv file via google drive link from 장인수 <insoo078@kribb.re.kr> 
 # VCF converter, written by Claude Opus 4.1 using 2 lines of example input
 python ~/kent/src/hg/makeDb/scripts/kovaToVcf.py 1_KOVA.v7.tsv.gz kova.v7.vcf
 bgzip kova.v7.vcf
 tabix -p vcf kova.v7.vcf.gz
 
 # NPM Singapore
 cd /hive/data/genomes/hg38/bed/varFreqs/npm/
 # downloaded data manually from chorus website, https://chorus.grids-platform.io/vcfdl
 bcftools concat  --threads 10  -Oz -o SG10K_Health_r5.3.2.sites.vcf.bgz  SG10K_Health_r5.3.2.sites.chr{1..22}.vcf.bgz SG10K_Health_r5.3.2.sites.chrX.vcf.bgz SG10K_Health_r5.3.2.sites.chrY.vcf.bgz 
 tabiv -p vcf SG10K_Health_r5.3.2.sites.vcf.bgz
 
 # Saudi 300 genomes
 cd /hive/data/genomes/hg38/bed/varFreqs/saudi
 wget https://figshare.com/ndownloader/files/51297884 -O 51297884.tsv.gz
 python3 ~/kent/src/hg/makeDb/scripts/saudiToVcf.py
 bgzip saudi.vcf
 tabix -p vcf saudi.vcf.gz
 
 # SFARI SPARK
 cd /hive/data/genomes/hg38/bed/varFreqs/sparkExomes/
 # used globus to download into vcf/
 sh ~/kent/src/hg/makeDb/scripts/sparkMergeVcfAddCounts.sh vcf/SPARK.iWES_v3.2024_08.deepvariant 8
 bcftools norm -m-  SPARK.iWES_v3.2024_08.deepvariant.sites.vcf.gz -Oz > SPARK.iWES_v3.2024_08.deepvariant.norm.vcf.gz && tabix -p vcf SPARK.iWES_v3.2024_08.deepvariant.norm.vcf.gz
 
 cd /hive/data/genomes/hg38/bed/varFreqs/sparkWgs/
 # used globus to download into vcf/
 sh ~/kent/src/hg/makeDb/scripts/sparkMergeVcfAddCounts.sh vcf/wgs_12519_genome.deepvariant 8
 bcftools norm -m-  wgs_12519_genome.deepvariant.sites.vcf.gz -Oz > wgs_12519_genome.deepvariant.norm.vcf.gz
 tabix -p vcf wgs_12519_genome.deepvariant.norm.vcf.gz
 
 # NCBI ALFA bigBed to VCF, Max Jan 26 2026
 # Source: ALFA R4 bigBed files, 904M variants, output 163M with non-zero AF
 cd /hive/data/genomes/hg38/bed/varFreqs/alfa
-mkdir -p parallel_out
-# run 10 parallel jobs, each processing 2-3 chromosomes
-python ~/kent/src/hg/makeDb/scripts/alfa_to_vcf.py --bigbed ALFA_GLB.bb --output parallel_out/part01.vcf --zero-file parallel_out/part01_zero.txt --chrom chr1 chr13 chr21 &> parallel_out/part01.log &
-python ~/kent/src/hg/makeDb/scripts/alfa_to_vcf.py --bigbed ALFA_GLB.bb --output parallel_out/part02.vcf --zero-file parallel_out/part02_zero.txt --no-header --chrom chr2 chr16 chr22 &> parallel_out/part02.log &
-python ~/kent/src/hg/makeDb/scripts/alfa_to_vcf.py --bigbed ALFA_GLB.bb --output parallel_out/part03.vcf --zero-file parallel_out/part03_zero.txt --no-header --chrom chr3 chr17 &> parallel_out/part03.log &
-python ~/kent/src/hg/makeDb/scripts/alfa_to_vcf.py --bigbed ALFA_GLB.bb --output parallel_out/part04.vcf --zero-file parallel_out/part04_zero.txt --no-header --chrom chr4 chr20 &> parallel_out/part04.log &
-python ~/kent/src/hg/makeDb/scripts/alfa_to_vcf.py --bigbed ALFA_GLB.bb --output parallel_out/part05.vcf --zero-file parallel_out/part05_zero.txt --no-header --chrom chr5 chr19 chrY &> parallel_out/part05.log &
-python ~/kent/src/hg/makeDb/scripts/alfa_to_vcf.py --bigbed ALFA_GLB.bb --output parallel_out/part06.vcf --zero-file parallel_out/part06_zero.txt --no-header --chrom chr6 chr18 &> parallel_out/part06.log &
-python ~/kent/src/hg/makeDb/scripts/alfa_to_vcf.py --bigbed ALFA_GLB.bb --output parallel_out/part07.vcf --zero-file parallel_out/part07_zero.txt --no-header --chrom chr7 chr15 &> parallel_out/part07.log &
-python ~/kent/src/hg/makeDb/scripts/alfa_to_vcf.py --bigbed ALFA_GLB.bb --output parallel_out/part08.vcf --zero-file parallel_out/part08_zero.txt --no-header --chrom chr8 chr12 chrM &> parallel_out/part08.log &
-python ~/kent/src/hg/makeDb/scripts/alfa_to_vcf.py --bigbed ALFA_GLB.bb --output parallel_out/part09.vcf --zero-file parallel_out/part09_zero.txt --no-header --chrom chr9 chr11 &> parallel_out/part09.log &
-python ~/kent/src/hg/makeDb/scripts/alfa_to_vcf.py --bigbed ALFA_GLB.bb --output parallel_out/part10.vcf --zero-file parallel_out/part10_zero.txt --no-header --chrom chr10 chr14 chrX &> parallel_out/part10.log &
-wait
-cat parallel_out/part*.vcf > ALFA_merged.vcf
-cat parallel_out/part*_zero.txt > ALFA_zero.txt
-grep '^#' ALFA_merged.vcf > ALFA_sorted.vcf
-grep -v '^#' ALFA_merged.vcf | sort -k1,1V -k2,2n >> ALFA_sorted.vcf
-bgzip -f ALFA_sorted.vcf && mv -f ALFA_sorted.vcf.gz ALFA.vcf.gz && tabix -p vcf ALFA.vcf.gz
-rm -f ALFA_merged.vcf parallel_out/*.vcf parallel_out/*_zero.txt
-ln -sf /hive/data/genomes/hg38/bed/varFreqs/alfa/ALFA.vcf.gz /gbdb/hg38/varFreqs/alfa/
-ln -sf /hive/data/genomes/hg38/bed/varFreqs/alfa/ALFA.vcf.gz.tbi /gbdb/hg38/varFreqs/alfa/
+python3 alfa_to_vcf.py --out ALFA.vcf --zero-af-file ALFA_zero.txt
+# Compress and index
+bgzip ALFA.vcf
+tabix -p vcf ALFA.vcf.gz
 # Final: 2.7GB, 163M variants (146M SNPs, 17M indels), ALFA_zero.txt has 26GB of zero-AF variants
 
+# Australia, Max, Jan 2026
+# received files from m.hobbs@garvan.org.au
+cd /hive/data/genomes/hg38/bed/varFreqs/mgrb/
+bcftools norm -f hg38.fa -m-any MGRB.phase3.GRCh38.vcf.gz -o MGRB.phase3.GRCh38.norm.vcf.gz
+tabix MGRB.phase3.GRCh38.norm.vcf.gz