d6d6967d21a36badceed77f26525ec10c8540ebf
jcasper
  Tue Apr 5 17:40:55 2022 -0700
Updates to the DECIPHER pipeline, first for a manual build, refs #29130

diff --git src/hg/utils/otto/decipher/validateDecipher.sh src/hg/utils/otto/decipher/validateDecipher.sh
index 38e5944..0fd8a1c 100755
--- src/hg/utils/otto/decipher/validateDecipher.sh
+++ src/hg/utils/otto/decipher/validateDecipher.sh
@@ -1,38 +1,43 @@
-#!/bin/sh -e
+#!/bin/bash
+set -beEu -o pipefail
 
-db=$1
-tooMuch=0.1000   # how much change (either gain or loss) is too much
+db=hg38
+tooMuch=0.1000
 
 for i in `cat ../decipher.tables`
 do
     fields='*'
-#    if  test $i == "decipherRaw"
-#    then
-#	fields='id,start,end,chr,mean_ratio,classification_type'
-#	#fields='id,start,end,chr,mean_ratio'
-#    else
-#        fields='*'
-#    fi
-
     echo "select $fields from $i" |  hgsql $db | tail -n +2 | sort > $i.out
     f=$i"New"
     echo "select $fields from $f" |hgsql $db | tail -n +2 | sort > $f.out
     oldCount=`cat $i.out | wc -l`
     newCount=`cat $f.out | wc -l`
     common=`join -t '\001'  $i.out $f.out | wc -l`
     onlyOld=`join -t '\001' -v 1 $i.out $f.out | wc -l`
     onlyNew=`join -t '\001' -v 2 $i.out $f.out | wc -l`
     echo $i $newCount "-" $onlyNew "=" $common "=" $oldCount "-" $onlyOld
-    rm $i.out $f.out
+    rm "$i.out" "$f.out"
 done > newDecipher.stats
 
+# bigBed validation
+
+oldBed=`mktemp decipherValidateOldXXX.bed`
+bigBedToBed ../release/hg38/decipherCnv.bb $oldBed
+oldCount=`cat $oldBed | wc -l`
+newCount=`cat decipherCnv.bed | wc -l`
+common=`comm -12 <( sort $oldBed) <(sort decipherCnv.bed) | wc -l`
+onlyOld=`comm -23 <(sort $oldBed) <(sort decipherCnv.bed) | wc -l`
+onlyNew=`comm -13 <(sort $oldBed) <(sort decipherCnv.bed) | wc -l`
+echo "decipherCnv.bed" $newCount "-" $onlyNew "=" $common "=" $oldCount "-" $onlyOld >> newDecipher.stats
+rm "$oldBed"
+
 cat newDecipher.stats | awk -v db=$db -v tooMuch=$tooMuch '
 {
     if (($4/$6 > tooMuch) || ($10/$6 > tooMuch))
         {
-	print "validate on " db "." $1 " failed:" $4,$6,$4/$6,$10,$6,$10/$6;
+        print "validate on " db "." $1 " failed: only new " $4 "/" $6 "=" $4/$6 ", only old " $10 "/" $6 "=" $10/$6;
         exit 1
         }
 }'
 
 exit 0