74f5343d5f22428477778d20a5dc6e9e39c213fd
braney
  Thu Aug 13 14:36:56 2026 -0700
geneReviews otto: pin the locale so a hand-run matches the cron run, refs #38098

The NCBI GeneReviews files are Latin-1. GRtitle_shortname_NBKid.txt holds two
titles with high bytes, "Cantu syndrome" and "Stromme Syndrome". Under a UTF-8
locale GNU grep decides the file is binary, drops those two lines and writes a
note to stderr that the wrapper's mail never shows. cron runs with no locale set
and is safe, but a hand-run from a login shell quietly loses two disease titles
and three geneReviewsDetail rows.

Set LC_ALL=C in all three scripts. This also keeps sort and join in agreement in
validateGeneReviews.sh whichever way the job is started.

diff --git src/hg/utils/otto/geneReviews/buildGeneReviews.sh src/hg/utils/otto/geneReviews/buildGeneReviews.sh
index 01df03328c1..c3bf3db7510 100755
--- src/hg/utils/otto/geneReviews/buildGeneReviews.sh
+++ src/hg/utils/otto/geneReviews/buildGeneReviews.sh
@@ -1,109 +1,116 @@
 #!/bin/sh
 set -e
 # processing raw data file from GENEREVIEWS
 
+# The NCBI files are Latin-1, not UTF-8.  Under a UTF-8 locale grep treats them
+# as binary and silently drops the lines that hold the high bytes, so two
+# disease titles go missing.  cron runs with no locale set and is safe, but a
+# hand-run from a login shell is not.  Pin the locale so both behave the same.
+LC_ALL=C
+export LC_ALL
+
 geneDiseaseFile="geneReviewsGeneDiseases.tab"
 
 function createGeneReviewTables() { 
 #Create geneReviews table and bigBed
 
 # Load the internal working table geneReviewsGrshortNBKid to hg38/hg19/hg18
 hgsql $1 -e 'drop table if exists geneReviewsGrshortNBKidNew'
 hgsql $1 -e 'create table geneReviewsGrshortNBKidNew select * from geneReviewsGrshortNBKid limit 0'
 hgsql $1 -e \
 'load data local infile "geneReviewsGrshortNBKid.tab" into table geneReviewsGrshortNBKidNew'
 
 # Load the internal working table geneReviewsGrshortTitleNBKid to
 # hg38/19/18
 hgsql $1 -e 'drop table if exists geneReviewsGrshortTitleNBKidNew'
 hgsql $1 -e 'create table geneReviewsGrshortTitleNBKidNew select * from geneReviewsGrshortTitleNBKid limit 0'
 hgsql $1 -e \
 'load data local infile "geneReviewsGrshortTitleNBKid.tab" into table geneReviewsGrshortTitleNBKidNew'
 
 # for each refGen in grRefGene.lst, create a non-overlapping bed row. 
 rm -f geneReviews.$1.tab
 cat grRefGene.lst | while read G
     do
     if [ "$1" = "hg38" ]
     then
         hgsql $1 -N -e \
             "SELECT e.chrom,e.txStart,e.txEnd,e.name2 \
             FROM gencodeAnnotV35 e where e.name2 ='${G}' \
             ORDER BY e.chrom,e.txStart;" > temp.in
     else
         hgsql $1 -N -e \
             "SELECT e.chrom,e.txStart,e.txEnd,j.geneSymbol \
             FROM knownGene e, kgXref j WHERE e.name = j.kgID AND \
             j.geneSymbol ='${G}' ORDER BY e.chrom,e.txStart;" > temp.in
     fi
     bedRemoveOverlap temp.in temp.out
     cat temp.out >> geneReviews.$1.tab
     done
 rm temp.*
 
 # load the collapsed bed4 file to database
 hgsql $1 -e 'drop table if exists geneReviewsNew'
 
 # NOTE: keeping table for backwards compatibility and search (for now)
 hgLoadBed $1 geneReviewsNew geneReviews.$1.tab
 
 # Create big bed file with diseases for mouseover
 db=$1
 perl ../geneRevsAddDiseases.pl $geneDiseaseFile geneReviews.$db.tab > geneReviewsExt.$db.tab
 bedSort geneReviewsExt.$db.tab  stdout | uniq > geneReviewsExt.$db.bed
 gbdb="/gbdb/$db/geneReviews"
 mkdir -p $gbdb
 
 # validate
 oldLc=`bigBedToBed $gbdb/geneReviews.bb stdout | wc -l`
 newLc=`wc -l < geneReviewsExt.$db.bed`
 echo rowcount: old $oldLc new: $newLc
 if [ $oldLc -ne 0 ]; then
     echo $oldLc $newLc | \
         awk '{if (($2-$1)/$1 > 0.1) {printf "validate $db GENE REVIEWS failed: old count: %d, new count: %d\n", $1,$2; exit 1;}}'
 fi
 
 # Build the bigBed, but leave /gbdb pointing at the previous build for now.
 # checkGeneReviews.sh moves the link after validation passes, so the browser
 # image and the geneReviews/geneReviewsDetail tables always come from the same
 # build.
 bedToBigBed -tab -type=bed9+2 -as=../geneReviews.as geneReviewsExt.$db.bed \
          /hive/data/genomes/$db/chrom.sizes geneReviews.$db.bb
 
 # Create and load geneReviewsDetail table
 hgsql $1 -N -e \
     "SELECT  s.geneSymbol, s.grShort, t.NBKid, t.grTitle \
         FROM geneReviewsGrshortNBKidNew s, geneReviewsGrshortTitleNBKidNew t \
         WHERE s.grShort = t.grShort ORDER BY s.geneSymbol;" > geneReviewsDetail.tab
 
 hgsql $1 -e 'drop table if exists geneReviewsDetailNew'
 hgsql $1 -e 'create table geneReviewsDetailNew select * from geneReviewsDetail limit 0'
 hgsql $1 -e \
 'load data local infile "geneReviewsDetail.tab" into table geneReviewsDetailNew'
 }
 
 ####### main ##########
 
 # Create gene to disease file for extended BED mouseover
 perl ../geneRevsListDiseases.pl NBKid_shortname_genesymbol.txt GRtitle_shortname_NBKid.txt > \
         $geneDiseaseFile
 
 # Create tab files for internal geneReviewsGrshortNBKid and  geneReviewsGrshortTitleNBKid tables
 cat NBKid_shortname_genesymbol.txt | grep -v "^#" \
     | awk  '{FS="\t"} {OFS="\t"} {if ($3!="Not applicable") print $3,$2,$1}' \
     | sort -k1 > geneReviewsGrshortNBKid.tab
 grep -v "^#" GRtitle_shortname_NBKid.txt | sort -k1 > geneReviewsGrshortTitleNBKid.tab
 
 # Generate a list of refSeq genes that have geneReview associated with it.
 cat geneReviewsGrshortNBKid.tab | awk -F'\t' '{printf  "%s\n", $1}' \
     | sort  | uniq  > grRefGene.lst
 
 createGeneReviewTables "hg38"
 createGeneReviewTables "hg19"
 createGeneReviewTables "hg18"
 
 exit 0