de609b7489963f32ad474619cb3dff0283ef7e76 braney Sun Aug 23 15:13:46 2026 -0700 vcf: bound the genotype allele index by the field it is stored in, refs #38155 parseAlleleIx checked the index against alleleCount only. A record can hold up to VCF_MAX_INFO alleles, so an index of 128 or more could pass that check and then narrow on the way into the signed char field. The narrowed value was sometimes another real allele of the record: with 260 ALT alleles, index 260 came out as 4. The parser then reported a genotype the VCF never named. The check now also rejects an index above SCHAR_MAX, so an index too large for the field reads as missing data. SCHAR_MAX and not CHAR_MAX, because CHAR_MAX is 255 on the unsigned char platforms that the field is declared signed for. Every other assignment to hapIxA and hapIxB in this file is a literal in the range -1 to 2, so parseAlleleIx was the only path that could carry an out-of-range value. New test vcfParseManyAlleles, with a record of 260 ALT alleles. Without the fix, GT 128/1 reads as -128/1, 130/130 as -126/-126, and 260/260 as 4/4. diff --git src/lib/tests/makefile src/lib/tests/makefile index 879bd33e16c..78760d99568 100644 --- src/lib/tests/makefile +++ src/lib/tests/makefile @@ -1,417 +1,425 @@ kentSrc = ../.. include ../../inc/common.mk MYLIBDIR = ../../lib/${MACHTYPE} MYLIBS = ${MYLIBDIR}/jkweb.a BIN_DIR = bin/${MACHTYPE} pipelineTester = ${BIN_DIR}/pipelineTester test: errCatchTest htmlPageTest htmlExpandUrlTest pipelineTests dyStringTest \ mimeTests base64Tests quotedPTests safeTest hashTest fetchUrlTest gff3Test \ tabixTest vcfTest hacTreeTest mmHashTest testSumDoubles jsonQueryTest \ dnaCodonTest rm -r output fetchUrlTest testSumDoubles @echo tested all mkdirs: ${MKDIR} output ${BIN_DIR} testSumDoubles: testSumDoubles.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o ./testSumDoubles testSumDoubles.o ${MYLIBS} ${L} dnaCodonTest: dnaCodonTest.o ${MYLIBS} mkdirs @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/dnaCodonTest dnaCodonTest.o ${MYLIBS} ${L} ${STRIP} ${BIN_DIR}/dnaCodonTest${EXE} ${BIN_DIR}/dnaCodonTest > output/dnaCodonTest diff expected/dnaCodonTest output/dnaCodonTest errCatchTest: errCatchTest.o ${MYLIBS} mkdirs @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/errCatchTest errCatchTest.o ${MYLIBS} ${L} ${STRIP} ${BIN_DIR}/errCatchTest${EXE} ${BIN_DIR}/errCatchTest secret > output/errCatch.good diff expected/errCatch.good output/errCatch.good ${BIN_DIR}/errCatchTest bad > output/errCatch.bad diff expected/errCatch.bad output/errCatch.bad htmlExpandUrlTest: htmlExpandUrlTest.o ${MYLIBS} mkdirs @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/htmlExpandUrlTest htmlExpandUrlTest.o ${MYLIBS} ${L} ${STRIP} ${BIN_DIR}/htmlExpandUrlTest${EXE} ${BIN_DIR}/htmlExpandUrlTest > output/htmlExpandUrlTest 2>&1 diff expected/htmlExpandUrlTest output/htmlExpandUrlTest htmlPageTest: htmlPageTest.o ${MYLIBS} mkdirs @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/htmlPageTest htmlPageTest.o ${MYLIBS} ${L} ${STRIP} ${BIN_DIR}/htmlPageTest${EXE} ${BIN_DIR}/htmlPageTest input/google.html > output/google.out diff expected/google.out output/google.out pipelineTests: pipelineWrite pipelineWriteMult pipelineWriteFd \ pipelineRead pipelineReadMult pipelineReadFd pipelineReadMem \ pipelineExitCode pipelineExitCode2X pipelineWriteErr pipelineExecError pipelineSigpipe \ pipelineTimeout pipelineWrite: ${pipelineTester} mkdirs ${pipelineTester} -write -pipeData=input/simple1.txt -otherEnd=output/$@.out.gz "gzip -1" gunzip -c output/$@.out.gz > output/$@.out diff -b input/simple1.txt output/$@.out # add come junk to make sure output gets truncated pipelineWriteMult: ${pipelineTester} mkdirs cat input/google.html > output/$@.wc ${pipelineTester} -write -pipeData=input/simple1.txt -otherEnd=output/$@.wc "gzip -1" "gzip -dc" "wc" diff -b expected/simple1.wc output/$@.wc pipelineWriteFd: ${pipelineTester} mkdirs ${pipelineTester} -fdApi -write -pipeData=input/simple1.txt -otherEnd=output/$@.out.gz "gzip -1" gunzip -c output/$@.out.gz > output/$@.out diff -b input/simple1.txt output/$@.out pipelineRead: ${pipelineTester} mkdirs gzip -1c input/simple1.txt >output/$@.in.gz ${pipelineTester} -otherEnd=output/$@.in.gz -pipeData=output/$@.out "gzip -dc" diff -b input/simple1.txt output/$@.out pipelineReadMult: ${pipelineTester} mkdirs ${pipelineTester} -pipeData=output/$@.wc -otherEnd=input/simple1.txt "gzip -1" "gzip -dc" "wc" diff -b expected/simple1.wc output/$@.wc pipelineReadFd: ${pipelineTester} mkdirs gzip -1c input/simple1.txt >output/$@.in.gz ${pipelineTester} -fdApi -otherEnd=output/$@.in.gz -pipeData=output/$@.out "gzip -dc" diff -b input/simple1.txt output/$@.out pipelineReadMem: ${pipelineTester} mkdirs gzip -1c input/simple1.txt >output/$@.in.gz ${pipelineTester} -memApi -otherEnd=output/$@.in.gz -pipeData=output/$@.out "gzip -dc" diff -b input/simple1.txt output/$@.out # make sure pipe exit code makes it back pipelineExitCode: ${pipelineTester} ${pipelineTester} -exitCode=13 "sh -c 'exit 13'" # this failed when test was run twise in same process pipelineExitCode2X: ${pipelineTester} ${pipelineTester} -executeTwice -exitCode=13 "sh -c 'exit 13'" # test redirecting stderr, see that two process can write stderr, but only # the second's stdout should make it to the end of the pipe. Since order # of writes to stderr is determined by process scheduling and when a process # terminates due to SIGPIPE, just check that stderr was not empty, don't # check contents. pipelineWriteErr: ${pipelineTester} mkdirs ${pipelineTester} -write -otherEnd=output/$@.out -stderr=output/$@.err "sh -c 'echo OUT; echo ERR >&2'" "sh -c 'echo OUT2; echo ERR2 >&2'" diff -b expected/$@.out output/$@.out test -s output/$@.err # exec a non-existent program pipelineExecError: ${pipelineTester} mkdirs if ${pipelineTester} -write -stderr=output/$@.err "./thatDoesNotCompute" 2> output/$@.parent.err ; then false else true ; fi diff -b expected/$@.err output/$@.err diff -b expected/$@.parent.err output/$@.parent.err # test setting SIGPIPE by generating lots of output and then prematurely closing the # pipe pipelineSigpipe: ${pipelineTester} mkdirs ${pipelineTester} -sigpipe -maxNumLines=3 -pipeData=/dev/null "awk 'BEGIN {while (1) {print "foo"}}'" pipelineTimeout: ${pipelineTester} mkdirs if ${pipelineTester} -timeout=2 "bash -c 'sleep 20'" 2> output/$@.parent.err ; then false else true ; fi diff -b expected/$@.parent.err output/$@.parent.err ${pipelineTester}: pipelineTester.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${pipelineTester} pipelineTester.o ${MYLIBS} ${L} dyStringTest: ${BIN_DIR}/dyStringTester mkdirs ${BIN_DIR}/dyStringTester ${BIN_DIR}/dyStringTester: mkdirs dyStringTester.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/dyStringTester dyStringTester.o ${MYLIBS} ${L} mimeTests: mime1 mime2 mime3 mime4 mimeBin mime5 mimeAltHead mimeAutoBoundary mimeBlat ${BIN_DIR}/mimeTester: mkdirs mimeTester.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/mimeTester mimeTester.o ${MYLIBS} ${L} mime1: ${BIN_DIR}/mimeTester mkdirs ${BIN_DIR}/mimeTester < input/$@.txt > output/$@.out diff expected/$@.out output/$@.out mime2: ${BIN_DIR}/mimeTester mkdirs ${BIN_DIR}/mimeTester < input/$@.txt > output/$@.out diff expected/$@.out output/$@.out mime3: ${BIN_DIR}/mimeTester mkdirs ${BIN_DIR}/mimeTester < input/$@.txt > output/$@.out diff expected/$@.out output/$@.out mime4: ${BIN_DIR}/mimeTester mkdirs ${BIN_DIR}/mimeTester < input/$@.txt > output/$@.out diff expected/$@.out output/$@.out mimeBin: ${BIN_DIR}/mimeTester mkdirs ${BIN_DIR}/mimeTester < input/$@.txt > output/$@.out diff expected/$@.out output/$@.out mime5: ${BIN_DIR}/mimeTester mkdirs ${BIN_DIR}/mimeTester < input/$@.txt > output/$@.out diff expected/$@.out output/$@.out mimeAltHead: ${BIN_DIR}/mimeTester mkdirs ${BIN_DIR}/mimeTester -altHeader='CONTENT_TYPE=multipart/form-data; boundary=----------0xKhTmLbOuNdArY' < input/$@.txt > output/$@.out diff expected/$@.out output/$@.out mimeAutoBoundary: ${BIN_DIR}/mimeTester mkdirs ${BIN_DIR}/mimeTester -autoBoundary < input/$@.txt > output/$@.out diff expected/$@.out output/$@.out mimeBlat: ${BIN_DIR}/mimeTester mkdirs ${BIN_DIR}/mimeTester -altHeader='CONTENT_TYPE=multipart/form-data; boundary=----------0xKhTmLbOuNdArY' < input/$@.txt > output/$@.out diff expected/$@.out output/$@.out mimeSeries: ${BIN_DIR}/mimeTester mkdirs ${BIN_DIR}/mimeTester -sizeSeries=3000 ${BIN_DIR}/htmlMimeTest: mkdirs htmlMimeTest.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/htmlMimeTest htmlMimeTest.o ${MYLIBS} ${L} htmlMime1: ${BIN_DIR}/htmlMimeTest mkdirs ${BIN_DIR}/htmlMimeTest https://hgwdev.gi.ucsc.edu/cgi-bin/hgBlat input/htmlMime.txt 3490 3502 > output/$@.out diff expected/$@.out output/$@.out base64Tests: base64Encode base64Decode ${BIN_DIR}/testBase64: mkdirs testBase64.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/testBase64 testBase64.o ${MYLIBS} ${L} base64Encode: ${BIN_DIR}/testBase64 mkdirs ${BIN_DIR}/testBase64 'My Test String' > output/$@.out diff expected/$@.out output/$@.out base64Decode: ${BIN_DIR}/testBase64 mkdirs ${BIN_DIR}/testBase64 'TXkgVGVzdCBTdHJpbmc=' > output/$@.out diff expected/$@.out output/$@.out quotedPTests: quotedPEncode quotedPDecode ${BIN_DIR}/testQuotedP: mkdirs testQuotedP.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/testQuotedP testQuotedP.o ${MYLIBS} ${L} quotedPEncode: ${BIN_DIR}/testQuotedP mkdirs ${BIN_DIR}/testQuotedP 'taxes are quite high ' > output/$@.out diff expected/$@.out output/$@.out quotedPDecode: ${BIN_DIR}/testQuotedP mkdirs ${BIN_DIR}/testQuotedP 'taxes=20are=20quite=20high=20=' > output/$@.out diff expected/$@.out output/$@.out ${BIN_DIR}/mimeDecodeTest: mkdirs mimeDecodeTest.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/mimeDecodeTest mimeDecodeTest.o ${MYLIBS} ${L} mimeDecodeTest: ${BIN_DIR}/mimeDecodeTest mkdirs ${BIN_DIR}/mimeDecodeTest -cid -autoBoundary output < input/$@.txt diff expected/noName1.html output/noName1.html ${BIN_DIR}/safeTester: mkdirs safeTester.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/safeTester safeTester.o ${MYLIBS} ${L} safeTest: ${BIN_DIR}/safeTester mkdirs ${BIN_DIR}/safeTester hashTest: hashTest1 ${BIN_DIR}/testHash: mkdirs testHash.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/testHash testHash.o ${MYLIBS} ${L} hashTest1: ${BIN_DIR}/testHash mkdirs ${BIN_DIR}/testHash input/$@.txt > output/$@.out diff expected/$@.out output/$@.out ${BIN_DIR}/testQuotedString: mkdirs testQuotedString.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/testQuotedString testQuotedString.o ${MYLIBS} ${L} testQuotedString: ${BIN_DIR}/testQuotedString mkdirs ${BIN_DIR}/testQuotedString -verbose=2 quote this\\ following miniBlat: mkdirs miniBlat.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o miniBlat miniBlat.o ${MYLIBS} ${L} fetchUrlTest: mkdirs fetchUrlTest.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o fetchUrlTest fetchUrlTest.o ${MYLIBS} ${L} fetchUrlViaUdcTest: mkdirs fetchUrlViaUdcTest.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o fetchUrlViaUdcTest fetchUrlViaUdcTest.o ${MYLIBS} ${L} ## # gff3 tests ## gff3Tester=${BIN_DIR}/gff3Tester gff3Test: gff3SacCerTest gff3ErrorCasesTest gff3DiscontiousTest # FIXME: doesn't work yet # gff3SpecialCasesTest gff3SacCerTest: ${gff3Tester} mkdirs ${gff3Tester} input/sacCerTest.gff3 output/$@.out diff expected/$@.out output/$@.out gff3SpecialCasesTest: ${gff3Tester} mkdirs ${gff3Tester} input/specialCasesTest.gff3 output/$@.out diff expected/$@.out output/$@.out gff3ErrorCasesTest: ${gff3Tester} mkdirs if ${gff3Tester} input/errorCasesTest.gff3 /dev/null >output/$@.err 2>&1 ; then exit 0 else exit 1; fi diff expected/$@.err output/$@.err gff3DiscontiousTest: ${gff3Tester} mkdirs ${gff3Tester} input/discontinuous.gff3 output/$@.out diff expected/$@.out output/$@.out ${BIN_DIR}/gff3Tester: gff3Tester.o ${MYLIBS} ${MKDIR} ${BIN_DIR} ${CC} ${COPT} -o ${BIN_DIR}/gff3Tester gff3Tester.o ${MYLIBS} ${L} # lineFile's tabix support: tabixTester=${BIN_DIR}/tabixFetch tabixTest: tabixFetch1kGNoGenotypes tabixFetch1kGWithGenotypes tabixFetch1kGNoGenotypes: ${tabixTester} mkdirs ${tabixTester} input/YRI.trio.2010_06.novelsequences.sites.vcf.gz 2:26790860-194631353 > output/$@.out diff expected/$@.out output/$@.out tabixFetch1kGWithGenotypes: ${tabixTester} mkdirs ${tabixTester} input/YRI.low_coverage.2010_07_excerpt.genotypes.vcf.gz 2:26793738-26794385 > output/$@.out diff expected/$@.out output/$@.out ${BIN_DIR}/tabixFetch: tabixFetch.o ${MYLIBS} ${MKDIR} ${BIN_DIR} ${CC} ${COPT} -o ${BIN_DIR}/tabixFetch tabixFetch.o ${MYLIBS} ${L} # vcf: vcfTester=${BIN_DIR}/vcfParseTest vcfTest: vcfParse1kGNoGenotypes vcfParse1kGWithGenotypes vcfParseOldV3 \ vcfHeader1kGNoGenotypes vcfHeader1kGWithGenotypes vcfHeaderOldV3 \ - vcfParseBadGenotypeIx + vcfParseBadGenotypeIx vcfParseManyAlleles vcfParse1kGNoGenotypes: ${vcfTester} mkdirs ${vcfTester} input/YRI.trio.2010_06.novelsequences.sites.vcf.gz 2 26790859 194631353 > output/$@.out diff expected/$@.out output/$@.out vcfParse1kGWithGenotypes: ${vcfTester} mkdirs ${vcfTester} input/YRI.low_coverage.2010_07_excerpt.genotypes.vcf.gz 2 26793737 26794385 > output/$@.out diff expected/$@.out output/$@.out vcfParseOldV3: ${vcfTester} mkdirs ${vcfTester} input/20091110_pilot1_vcf_merged_call_sets_YRI.2and3_way.vcf.gz 1 3000 50000 >& output/$@.out diff expected/$@.out output/$@.out # Regression tests for the tabix header-read path (htslib >= 1.21 tbx_readrec # strips meta_char lines, so the VCF header must be read off the htsFile # directly, not via the tabix iterator). These check version, def counts, and # sample IDs -- all of which silently degrade if the header parser sees nothing. vcfHeader1kGNoGenotypes: ${vcfTester} mkdirs ${vcfTester} -headerOnly input/YRI.trio.2010_06.novelsequences.sites.vcf.gz > output/$@.out diff expected/$@.out output/$@.out vcfHeader1kGWithGenotypes: ${vcfTester} mkdirs ${vcfTester} -headerOnly input/YRI.low_coverage.2010_07_excerpt.genotypes.vcf.gz > output/$@.out diff expected/$@.out output/$@.out vcfHeaderOldV3: ${vcfTester} mkdirs ${vcfTester} -headerOnly input/20091110_pilot1_vcf_merged_call_sets_YRI.2and3_way.vcf.gz > output/$@.out diff expected/$@.out output/$@.out # A GT allele index that this record has no allele for must parse as missing data, so that # every caller sees either a real allele or missing data. vcfParseBadGenotypeIx: ${vcfTester} mkdirs ${vcfTester} -genotypes input/badGenotypeIx.vcf.gz chr1 0 10000 > output/$@.out diff expected/$@.out output/$@.out +# A record can have more alleles than fit in the signed char that holds a genotype's allele +# index. An index too large for the field must also parse as missing data, instead of being +# silently narrowed to some other value. input/manyAlleles.vcf.gz has 260 ALT alleles, so +# index 128 would narrow to a negative value and index 260 to 4, a real allele of that record. +vcfParseManyAlleles: ${vcfTester} mkdirs + ${vcfTester} -genotypes input/manyAlleles.vcf.gz chr1 0 10000 > output/$@.out + diff expected/$@.out output/$@.out + ${BIN_DIR}/vcfParseTest: vcfParseTest.o ${MYLIBS} ${MKDIR} ${BIN_DIR} ${CC} ${COPT} -o ${BIN_DIR}/vcfParseTest vcfParseTest.o ${MYLIBS} ${L} # hacTree: hacTreeTester=${BIN_DIR}/hacTreeTest hacTreeTest: ${hacTreeTester} mkdirs ${hacTreeTester} input/$@.txt output/$@.out diff expected/$@.out output/$@.out ${BIN_DIR}/hacTreeTest: hacTreeTest.o ${MYLIBS} ${MKDIR} ${BIN_DIR} ${CC} ${COPT} -o ${BIN_DIR}/hacTreeTest hacTreeTest.o ${MYLIBS} ${L} # mmHash: mmHashTester=${BIN_DIR}/mmHashTest mmHashTest: ${mmHashTester} mkdirs ${mmHashTester} input/$@.txt output/$@.mmh output/$@.out diff expected/$@.out output/$@.out cmp expected/$@.mmh output/$@.mmh ${BIN_DIR}/mmHashTest: mmHashTest.o ${MYLIBS} ${MKDIR} ${BIN_DIR} ${CC} ${COPT} -o ${BIN_DIR}/mmHashTest mmHashTest.o ${MYLIBS} ${L} # udc (not part of the top-level test target at this point): udcTest: udcTest.o ${MYLIBS} mkdirs @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/udcTest udcTest.o ${MYLIBS} ${L} ${BIN_DIR}/udcTest # udc (not part of the top-level test target at this point): udcCacheSizesCheck: udcCacheSizesCheck.o ${MYLIBS} mkdirs @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/udcCacheSizesCheck udcCacheSizesCheck.o ${MYLIBS} ${L} ${BIN_DIR}/udcCacheSizesCheck testOutOfMem: testOutOfMem.o ${MYLIBS} mkdirs @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/testOutOfMem testOutOfMem.o ${MYLIBS} ${L} # we expect this to errAbort because we allocate one byte too much beyond limit -${BIN_DIR}/testOutOfMem 100000 1 clean: rm -rf *.o bin output *.tmp mimeTester.tmp mimeTester.out fetchUrlTest fetchUrlViaUdcTest ${BIN_DIR}/testDecodedString: mkdirs testDecodedString.o ${MYLIBS} @${MKDIR} $(dir $@) ${CC} ${COPT} -o ${BIN_DIR}/testDecodedString testDecodedString.o ${MYLIBS} ${L} testDecodedString: ${BIN_DIR}/testDecodedString mkdirs ${BIN_DIR}/testDecodedString -verbose=2 quote this\\ following # jsonQuery: jsonQueryTester=${BIN_DIR}/jsonQueryTest jsonQueryTest: ${jsonQueryTester} mkdirs ${jsonQueryTester} input/$@Json.txt input/$@Path.txt output/$@.out diff expected/$@.out output/$@.out ${BIN_DIR}/jsonQueryTest: jsonQueryTest.o ${MYLIBS} ${MKDIR} ${BIN_DIR} ${CC} ${COPT} -o ${BIN_DIR}/jsonQueryTest jsonQueryTest.o ${MYLIBS} ${L}