6cdc3398485dd66c0a16e0b0ccaa840911f87308
braney
  Thu Aug 13 14:47:48 2026 -0700
geneReviews otto: go back to the exact coordinate check, refs #38098

Reverts the span-overlap comparison from e7b98d8286d. A wholesale shift in the
source coordinates, such as the one the July 15 hg19 knownGene rebuild caused,
should stop the job and have someone look at the new data. Loosening the test
would let the next one through unseen. The right response to a change like that
is to review it and install it by hand.

For geneReviews a row is again unchanged only when chrom, chromStart, chromEnd
and name all match. A comment on the test says why it is strict, so the next
person does not read the strictness as an oversight.

Kept from the reverted commit: LC_ALL=C, and a guard so zero rows in common
fails with a message rather than an awk divide-by-zero.

diff --git src/hg/utils/otto/geneReviews/validateGeneReviews.sh src/hg/utils/otto/geneReviews/validateGeneReviews.sh
index 71672089b7d..b792c513258 100755
--- src/hg/utils/otto/geneReviews/validateGeneReviews.sh
+++ src/hg/utils/otto/geneReviews/validateGeneReviews.sh
@@ -1,65 +1,52 @@
 #!/bin/sh -e
 
 # Pin the locale so sort and join agree, whether run by cron or by hand.
 LC_ALL=C
 export LC_ALL
 
 db=$1
 tooMuch=0.1000   # how much change (either gain or loss) is too much
 
-tab=`printf '\t'`
-
-# Write one line per gene per chromosome: "gene|chrom", lowest start, highest end.
-function geneSpans() {
-hgsql -N $db -e \
-    "select name, chrom, min(chromStart), max(chromEnd) from $1 group by name, chrom" \
-    | awk -F'\t' '{OFS="\t"; print $1"|"$2, $3, $4}' | sort -t "$tab" -k1,1
-}
+# The comparison below is deliberately strict: for geneReviews a row counts as
+# unchanged only if chrom, chromStart, chromEnd and name all match.  When the
+# source data shifts wholesale, as it does after a knownGene rebuild, this is
+# meant to fail and have someone look at the new data before it is installed.
+# See #38098.  To let a reviewed change through, install it by hand rather than
+# loosening the test.
 
 for i in `cat ../geneReviews.tables`
 do 
-    f=$i"New"
     if  test $i == "geneReviews"
     then
-        # hg19 and hg18 take their coordinates from knownGene, so a knownGene
-        # rebuild moves the ends of most genes by a few bases.  Comparing
-        # coordinates exactly then counts nearly every gene as changed and the
-        # run fails even though the data is fine.  Compare the overall span of
-        # each gene on each chromosome instead, and call a gene unchanged when
-        # its old and new spans overlap.
-        geneSpans $i > $i.out
-        geneSpans $f > $f.out
-        oldCount=`cat $i.out | wc -l`
-        newCount=`cat $f.out | wc -l`
-        # join gives gene|chrom, oldStart, oldEnd, newStart, newEnd
-        common=`join -t "$tab" $i.out $f.out | awk -F'\t' '$2 < $5 && $4 < $3' | wc -l`
-        onlyOld=$((oldCount - common))
-        onlyNew=$((newCount - common))
+    fields='chrom, chromStart, chromEnd, name'
     else
-        echo "select * from $i" |  hgsql $db | tail -n +2 | sort > $i.out
-        echo "select * from $f" |hgsql $db | tail -n +2 | sort > $f.out
+        fields='*'
+    fi
+
+    echo "select $fields from $i" |  hgsql $db | tail -n +2 | sort > $i.out
+    f=$i"New"
+    echo "select $fields from $f" |hgsql $db | tail -n +2 | sort > $f.out
     oldCount=`cat $i.out | wc -l`
     newCount=`cat $f.out | wc -l`
     common=`join -t '\001'  $i.out $f.out | wc -l`
     onlyOld=`join -t '\001' -v 1 $i.out $f.out | wc -l`
     onlyNew=`join -t '\001' -v 2 $i.out $f.out | wc -l`
-    fi
     echo $i $newCount "-" $onlyNew "=" $common "=" $oldCount "-" $onlyOld  
     rm $i.out $f.out
 done > newGeneReviews$db.stats
 
 cat newGeneReviews$db.stats | awk -v db=$db -v tooMuch=$tooMuch ' 
 {
     if ($6 == 0)
 	{
 	print "validate on " db "." $1 " failed: no rows in common";
 	exit 1
 	}
     if (($4/$6 > tooMuch) || ($10/$6 > tooMuch))
 	{
 	print "validate on " db "." $1 " failed:" $4,$6,$4/$6,$10,$6,$10/$6;
 	exit 1
 	}
 }'
 
 exit 0