0a48f8c6284efb6756f1d2ba23694b58c9aa6f47 lrnassar Tue Aug 18 12:01:13 2026 -0700 Update ENIGMA BRCA1/BRCA2 hub scripts and data to CSpec v1.2. refs #38130 Rebuilds the BRCAsplicing and BRCAfunctionalAssays tracks from the v1.2 specification tables downloaded from the ClinGen CSpec registry. Adds exportV12Sheets.py (xlsx to text with merged-cell expansion) and convertTable4toFlat.py (converts the v1.2 visual Table 4 layout back to the flat format the track script consumes, including NMD-boundary PTC sub-ranges). Build scripts now write to a versioned dir instead of overwriting the files the public hub serves, and the hgSearch coordinate scraper was fixed for the current page format and made to fail loudly on a missed lookup. Also updates the vcepVersions monitor regex for the corrected BRCA1/BRCA2 wording on the hub description page. diff --git src/hg/makeDb/scripts/enigma/BRCAfunctionalAssays.py src/hg/makeDb/scripts/enigma/BRCAfunctionalAssays.py index 6731b643bae..3352890133e 100644 --- src/hg/makeDb/scripts/enigma/BRCAfunctionalAssays.py +++ src/hg/makeDb/scripts/enigma/BRCAfunctionalAssays.py @@ -1,74 +1,74 @@ import subprocess def bash(cmd): """Run the cmd in bash subprocess""" try: rawBashOutput = subprocess.run(cmd, check=True, shell=True,\ stdout=subprocess.PIPE, universal_newlines=True, stderr=subprocess.STDOUT) bashStdoutt = rawBashOutput.stdout except subprocess.CalledProcessError as e: raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) return(bashStdoutt) -rawFilePath = "/hive/data/inside/enigmaTracksData/CSpec_BRCA12ACMG_Rules-Specifications_V1.1_Table-9_2023-11-22-1.txt" -bash("tail -n +4 "+rawFilePath+" > /hive/data/inside/enigmaTracksData/CSpecDataMinusHeader.txt") -rawFileNoHeader = open('/hive/data/inside/enigmaTracksData/CSpecDataMinusHeader.txt','r', encoding='latin-1') -varsToConvertToVcf = open('/hive/data/inside/enigmaTracksData/varsToConvert.txt','w') +rawFilePath = "/hive/data/inside/enigmaTracksData/v1.2/CSpec_BRCA12ACMG_Rules-Specifications_V1.2_Table-9.txt" +bash("tail -n +4 "+rawFilePath+" > /hive/data/inside/enigmaTracksData/v1.2/CSpecDataMinusHeader.txt") +rawFileNoHeader = open('/hive/data/inside/enigmaTracksData/v1.2/CSpecDataMinusHeader.txt','r', encoding='utf-8') +varsToConvertToVcf = open('/hive/data/inside/enigmaTracksData/v1.2/varsToConvert.txt','w') def assignRGBcolor(assignedCode): if assignedCode == "PS3": itemRgb = '0,100,120' #Dark blue elif assignedCode == "BS3": itemRgb = '0,200,240' #Light blue elif assignedCode == "None": itemRgb = '0,0,0' #Black else: print("Error, no code match: "+assignedCode) return(itemRgb) #itemRgb = '0,128,0' #Green for line in rawFileNoHeader: line = line.rstrip().split("\t") if line[0].startswith("BRCA1"): varsToConvertToVcf.write("NM_007294.4:"+line[1]+"\n") elif line[0].startswith("BRCA2"): varsToConvertToVcf.write("NM_000059.4:"+line[1]+"\n") else: print("ERROR! Missing transcript") varsToConvertToVcf.close() rawFileNoHeader.close() -bash("hgvsToVcf -noLeftShift hg38 /hive/data/inside/enigmaTracksData/varsToConvert.txt /hive/data/inside/enigmaTracksData/tempVcfFile") +bash("hgvsToVcf -noLeftShift hg38 /hive/data/inside/enigmaTracksData/v1.2/varsToConvert.txt /hive/data/inside/enigmaTracksData/v1.2/tempVcfFile") -tempVcfFile = open('/hive/data/inside/enigmaTracksData/tempVcfFile','r') +tempVcfFile = open('/hive/data/inside/enigmaTracksData/v1.2/tempVcfFile','r') vcfVarCoords = {} for line in tempVcfFile: line = line.rstrip().split("\t") if line[0].startswith("#"): continue else: #Have the dic be the hgnsc as key (e.g. NM_007294.3:c.1081T>C) with a list of chr + vcfLocation #Also subtracting 1 to make a chr chromStart chromEnd format vcfVarCoords[line[2]] = [line[0],str(int(line[1])-1),line[1]] tempVcfFile.close() -rawFileNoHeader = open('/hive/data/inside/enigmaTracksData/CSpecDataMinusHeader.txt','r', encoding='latin-1') -outputBedFile = open("/hive/data/inside/enigmaTracksData/outputBedFile.bed",'w') +rawFileNoHeader = open('/hive/data/inside/enigmaTracksData/v1.2/CSpecDataMinusHeader.txt','r', encoding='utf-8') +outputBedFile = open("/hive/data/inside/enigmaTracksData/v1.2/outputBedFile.bed",'w') #Reiterate through the file matching coordinates now for line in rawFileNoHeader: line = line.rstrip("\n").split("\t") geneSymbol = line[0] if line[0].startswith("BRCA1"): nmAccession = "NM_007294.4" elif line[0].startswith("BRCA2"): nmAccession = "NM_000059.4" else: print("Error: Line didn't start with BRCA*") nameToMatch = nmAccession+":"+line[1] _mouseOver = geneSymbol+"<br><b>HGVSc:</b> "+nmAccession+\ ":"+line[1]+"<br><b>HGVSp:</b> "+line[2]+\ @@ -79,32 +79,32 @@ if nameToMatch in vcfVarCoords.keys(): lineToWrite = ("\t".join(vcfVarCoords[nameToMatch])+"\t"+line[1]+\ "\t0\t.\t"+"\t".join(vcfVarCoords[nameToMatch][1:])+\ "\t"+itemRGB+"\t"+nmAccession+"\t"+line[0]+"\t"+\ "\t".join(line[2:])+"\t"+_mouseOver+"\n") outputBedFile.write(lineToWrite) else: print("PROBLEM! A key did not match.") print(nameToMatch) outputBedFile.close() rawFileNoHeader.close() -bash("bedSort /hive/data/inside/enigmaTracksData/outputBedFile.bed \ -/hive/data/inside/enigmaTracksData/outputBedFile.bed") +bash("bedSort /hive/data/inside/enigmaTracksData/v1.2/outputBedFile.bed \ +/hive/data/inside/enigmaTracksData/v1.2/outputBedFile.bed") startOfAsFile="""table BRCAfunctionalAssays "BRCA1 and BRCA2 functional assay results reviewed for application of PS3 and BS3 codes" ( string chrom; "Reference sequence chromosome or scaffold" uint chromStart; "Start position in chromosome" uint chromEnd; "End position in chromosome" string name; "HGVS Nucleotide" uint score; "Not used, all 0" char[1] strand; "Not used, all ." uint thickStart; "Same as chromStart" uint thickEnd; "Same as chromEnd" uint reserved; "RGB value (use R,G,B string in input file)" string NMaccession; "RefSeq NM Accession" string geneSymbol; "Gene symbol" @@ -117,37 +117,39 @@ name = [] for i in range(14): if line2[i] != "": line2[i] = line2[i].replace('"','') name.append(line1[i]+" - "+line2[i]) else: name.append(line1[i]) n=0 for i in name[2:]: n+=1 asFileAddition = " lstring extraField"+str(n)+';\t"'+i+'"\n' startOfAsFile = startOfAsFile+asFileAddition startOfAsFile = startOfAsFile+" string _mouseOver;"+'\t"'+'Field only used as mouseOver'+'"\n' -asFileOutput = open("/hive/data/inside/enigmaTracksData/BRCAfunctionalAssays.as","w") +asFileOutput = open("/hive/data/inside/enigmaTracksData/v1.2/BRCAfunctionalAssays.as","w") for line in startOfAsFile.split("\n"): if "_mouseOver" in line: asFileOutput.write(line+"\n )") else: asFileOutput.write(line+"\n") asFileOutput.close() -bash("bedToBigBed -as=/hive/data/inside/enigmaTracksData/BRCAfunctionalAssays.as -type=bed9+15 -tab \ -/hive/data/inside/enigmaTracksData/outputBedFile.bed /cluster/data/hg38/chrom.sizes \ -/hive/data/inside/enigmaTracksData/BRCAfunctionalAssaysHg38.bb") +bash("bedToBigBed -as=/hive/data/inside/enigmaTracksData/v1.2/BRCAfunctionalAssays.as -type=bed9+15 -tab \ +/hive/data/inside/enigmaTracksData/v1.2/outputBedFile.bed /cluster/data/hg38/chrom.sizes \ +/hive/data/inside/enigmaTracksData/v1.2/BRCAfunctionalAssaysHg38.bb") -bash("liftOver -bedPlus=9 -tab /hive/data/inside/enigmaTracksData/outputBedFile.bed \ +bash("liftOver -bedPlus=9 -tab /hive/data/inside/enigmaTracksData/v1.2/outputBedFile.bed \ /hive/data/genomes/hg38/bed/liftOver/hg38ToHg19.over.chain.gz \ -/hive/data/inside/enigmaTracksData/outputBedFileHg19.bed /hive/data/inside/enigmaTracksData/unmapped.bed") +/hive/data/inside/enigmaTracksData/v1.2/outputBedFileHg19.bed /hive/data/inside/enigmaTracksData/v1.2/unmapped.bed") -bash("bedToBigBed -as=/hive/data/inside/enigmaTracksData/BRCAfunctionalAssays.as -type=bed9+15 -tab \ -/hive/data/inside/enigmaTracksData/outputBedFileHg19.bed /cluster/data/hg19/chrom.sizes \ -/hive/data/inside/enigmaTracksData/BRCAfunctionalAssaysHg19.bb") +bash("bedToBigBed -as=/hive/data/inside/enigmaTracksData/v1.2/BRCAfunctionalAssays.as -type=bed9+15 -tab \ +/hive/data/inside/enigmaTracksData/v1.2/outputBedFileHg19.bed /cluster/data/hg19/chrom.sizes \ +/hive/data/inside/enigmaTracksData/v1.2/BRCAfunctionalAssaysHg19.bb") -bash("ln -sf /hive/data/inside/enigmaTracksData/BRCAfunctionalAssaysHg38.bb /gbdb/hg38/bbi/enigma/BRCAfunctionalAssays.bb") -bash("ln -sf /hive/data/inside/enigmaTracksData/BRCAfunctionalAssaysHg19.bb /gbdb/hg19/bbi/enigma/BRCAfunctionalAssays.bb") +# The /gbdb symlinks and the hub point at the fixed filenames one level up in +# /hive/data/inside/enigmaTracksData/ - the public hub serves through that chain, so +# this script builds into the versioned v1.2/ dir and the verified .bb files are +# copied onto the staging filenames at release time (see makedoc).