74f5343d5f22428477778d20a5dc6e9e39c213fd
braney
  Thu Aug 13 14:36:56 2026 -0700
geneReviews otto: pin the locale so a hand-run matches the cron run, refs #38098

The NCBI GeneReviews files are Latin-1. GRtitle_shortname_NBKid.txt holds two
titles with high bytes, "Cantu syndrome" and "Stromme Syndrome". Under a UTF-8
locale GNU grep decides the file is binary, drops those two lines and writes a
note to stderr that the wrapper's mail never shows. cron runs with no locale set
and is safe, but a hand-run from a login shell quietly loses two disease titles
and three geneReviewsDetail rows.

Set LC_ALL=C in all three scripts. This also keeps sort and join in agreement in
validateGeneReviews.sh whichever way the job is started.

diff --git src/hg/utils/otto/geneReviews/validateGeneReviews.sh src/hg/utils/otto/geneReviews/validateGeneReviews.sh
index 5b060479dcd..71672089b7d 100755
--- src/hg/utils/otto/geneReviews/validateGeneReviews.sh
+++ src/hg/utils/otto/geneReviews/validateGeneReviews.sh
@@ -1,61 +1,65 @@
 #!/bin/sh -e
 
+# Pin the locale so sort and join agree, whether run by cron or by hand.
+LC_ALL=C
+export LC_ALL
+
 db=$1
 tooMuch=0.1000   # how much change (either gain or loss) is too much
 
 tab=`printf '\t'`
 
 # Write one line per gene per chromosome: "gene|chrom", lowest start, highest end.
 function geneSpans() {
 hgsql -N $db -e \
     "select name, chrom, min(chromStart), max(chromEnd) from $1 group by name, chrom" \
     | awk -F'\t' '{OFS="\t"; print $1"|"$2, $3, $4}' | sort -t "$tab" -k1,1
 }
 
 for i in `cat ../geneReviews.tables`
 do
     f=$i"New"
     if  test $i == "geneReviews"
     then
         # hg19 and hg18 take their coordinates from knownGene, so a knownGene
         # rebuild moves the ends of most genes by a few bases.  Comparing
         # coordinates exactly then counts nearly every gene as changed and the
         # run fails even though the data is fine.  Compare the overall span of
         # each gene on each chromosome instead, and call a gene unchanged when
         # its old and new spans overlap.
         geneSpans $i > $i.out
         geneSpans $f > $f.out
         oldCount=`cat $i.out | wc -l`
         newCount=`cat $f.out | wc -l`
         # join gives gene|chrom, oldStart, oldEnd, newStart, newEnd
         common=`join -t "$tab" $i.out $f.out | awk -F'\t' '$2 < $5 && $4 < $3' | wc -l`
         onlyOld=$((oldCount - common))
         onlyNew=$((newCount - common))
     else
         echo "select * from $i" |  hgsql $db | tail -n +2 | sort > $i.out
         echo "select * from $f" |hgsql $db | tail -n +2 | sort > $f.out
         oldCount=`cat $i.out | wc -l`
         newCount=`cat $f.out | wc -l`
         common=`join -t '\001'  $i.out $f.out | wc -l`
         onlyOld=`join -t '\001' -v 1 $i.out $f.out | wc -l`
         onlyNew=`join -t '\001' -v 2 $i.out $f.out | wc -l`
     fi
     echo $i $newCount "-" $onlyNew "=" $common "=" $oldCount "-" $onlyOld
     rm $i.out $f.out
 done > newGeneReviews$db.stats
 
 cat newGeneReviews$db.stats | awk -v db=$db -v tooMuch=$tooMuch '
 {
     if ($6 == 0)
 	{
 	print "validate on " db "." $1 " failed: no rows in common";
 	exit 1
 	}
     if (($4/$6 > tooMuch) || ($10/$6 > tooMuch))
 	{
 	print "validate on " db "." $1 " failed:" $4,$6,$4/$6,$10,$6,$10/$6;
 	exit 1
 	}
 }'
 
 exit 0