0a48f8c6284efb6756f1d2ba23694b58c9aa6f47 lrnassar Tue Aug 18 12:01:13 2026 -0700 Update ENIGMA BRCA1/BRCA2 hub scripts and data to CSpec v1.2. refs #38130 Rebuilds the BRCAsplicing and BRCAfunctionalAssays tracks from the v1.2 specification tables downloaded from the ClinGen CSpec registry. Adds exportV12Sheets.py (xlsx to text with merged-cell expansion) and convertTable4toFlat.py (converts the v1.2 visual Table 4 layout back to the flat format the track script consumes, including NMD-boundary PTC sub-ranges). Build scripts now write to a versioned dir instead of overwriting the files the public hub serves, and the hgSearch coordinate scraper was fixed for the current page format and made to fail loudly on a missed lookup. Also updates the vcepVersions monitor regex for the corrected BRCA1/BRCA2 wording on the hub description page. diff --git src/hg/makeDb/scripts/enigma/BRCAsplicing.py src/hg/makeDb/scripts/enigma/BRCAsplicing.py index 3ef06d56731..1ad55412196 100644 --- src/hg/makeDb/scripts/enigma/BRCAsplicing.py +++ src/hg/makeDb/scripts/enigma/BRCAsplicing.py @@ -1,172 +1,181 @@ import re import subprocess def bash(cmd): """Run the cmd in bash subprocess""" try: rawBashOutput = subprocess.run(cmd, check=True, shell=True,\ stdout=subprocess.PIPE, universal_newlines=True, stderr=subprocess.STDOUT) bashStdoutt = rawBashOutput.stdout except subprocess.CalledProcessError as e: raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) return(bashStdoutt) -rawFilePath = "/hive/data/inside/enigmaTracksData/Anna_CSpec_BRCA12ACMG-Rules-Specifications_V1.1Table-4_2023-11-22.txt" +rawFilePath = "/hive/data/inside/enigmaTracksData/v1.2/Table4_V1.2_flat.txt" def assignRGBcolor(lineToCheck): if "DEL" in lineToCheck[4]: itemRgb = '180,53,53' #Red elif "DUP" in lineToCheck[4]: itemRgb = '117,136,214' #Blue elif "RNA" in lineToCheck[5]: itemRgb = '146,64,190' #Black else: itemRgb = '0,0,0' return(itemRgb) def checkExtraCarrotInPosition(position): if position[len(position)-1] == ">": correctPosition = position[0:len(position)-1] else: correctPosition = position return(correctPosition) rawFile = open(rawFilePath,'r') -outputBedFile = open("/hive/data/inside/enigmaTracksData/outputBedFile.bed",'w', encoding='latin-1') +outputBedFile = open("/hive/data/inside/enigmaTracksData/v1.2/outputBedFile.bed",'w', encoding='utf-8') for line in rawFile: line = line.rstrip("\n").split("\t") if line[0].startswith("Gene") or line[0]== "": continue elif line == "": continue else: # print(line) itemRgb = assignRGBcolor(line) NMacc = line[1] if line[0] == "BRCA1": strand = "-" chrom = "chr17" else: strand = "+" chrom = "chr13" if len(line[3].split("c")) > 2: firstPos=line[3].split(".")[1].split("-c")[0] secondPos=line[3].split("-c.")[1] + chromStart = None + chromEnd = None queryPosition = bash("curl https://hgwdev.gi.ucsc.edu/cgi-bin/hgSearch?search="+NMacc+"%3Ac"+firstPos) for resultsLine in queryPosition.split("\n"): - if resultsLine.startswith("<script"): + if "position=" in resultsLine: if strand == "-": chromEnd = str(int(resultsLine.split("position=")[1].split("-")[0].split(":")[1])+5) elif strand == "+": chromStart = str(int(resultsLine.split("position=")[1].split("-")[0].split(":")[1])+4) queryPosition = bash("curl https://hgwdev.gi.ucsc.edu/cgi-bin/hgSearch?search="+NMacc+"%3Ac"+secondPos) for resultsLine in queryPosition.split("\n"): - if resultsLine.startswith("<script"): + if "position=" in resultsLine: if strand == "-": chromStart = str(int(resultsLine.split("position=")[1].split("-")[0].split(":")[1])+4) elif strand == "+": chromEnd = str(int(resultsLine.split("position=")[1].split("-")[0].split(":")[1])+5) + if chromStart is None or chromEnd is None: + raise RuntimeError("hgSearch returned no position for "+NMacc+" "+line[3]) else: p = re.compile('[0-9]') if "-" in line[3]: #c.-20-1G> if len(line[3].split("-")) > 2: pos = "-"+line[3].split("-")[1] adjustment = p.findall(line[3].split("-")[2]) adjustment = "-"+adjustment[0] elif line[3].startswith("c.-"): #c.-20+2T> pos = "-"+line[3].split("-")[1].split("+")[0] adjustment = p.findall(line[3].split("+")[1]) adjustment = adjustment[0] else: #c.20-2T> pos = line[3].split("c.")[1].split("-")[0] adjustment = p.findall(line[3].split("-")[1]) adjustment = "-"+adjustment[0] elif "+" in line[3]: #c.475+1G> pos = line[3].split("c.")[1].split("+")[0] adjustment = p.findall(line[3].split("+")[1])[0] else: #c.1 pos = line[3].split('c.')[1] adjustment = 0 + position = None queryPosition = bash("curl https://hgwdev.gi.ucsc.edu/cgi-bin/hgSearch?search="+NMacc+"%3Ac"+pos) for resultsLine in queryPosition.split("\n"): - if resultsLine.startswith("<script"): + if "position=" in resultsLine: position = int(resultsLine.split("position=")[1].split("-")[0].split(":")[1])+5 + if position is None: + raise RuntimeError("hgSearch returned no position for "+NMacc+" "+line[3]) if strand == "-": position = int(position)-int(adjustment) elif strand == "+": position = int(position)+int(adjustment) chromStart = str(position-1) chromEnd = str(position) varType = line[4].replace('"','') ACMGcode = line[5].replace('"','') observation = line[6].replace("?","delta").replace('"','') correctedPosition = checkExtraCarrotInPosition(line[3]) _mouseOver = "<b>Transcript:</b> "+line[1]+"<br><b>Exon:</b> "+line[2]+\ "<br><b>Position:</b> "+correctedPosition+"<br><b>Var Type:</b> "+varType+\ "<br><b>ACMG Code: </b>"+ACMGcode outputBedFile.write(chrom+"\t"+chromStart+"\t"+chromEnd+"\t"+correctedPosition+"\t0\t"+\ strand+"\t"+chromStart+"\t"+chromEnd+"\t"+itemRgb+"\t"+\ "\t".join(line[:4])+"\t"+varType+"\t"+\ ACMGcode+"\t"+observation+\ "\t"+line[7]+"\t"+_mouseOver+"\n") rawFile.close() outputBedFile.close() -bash("bedSort /hive/data/inside/enigmaTracksData/outputBedFile.bed \ -/hive/data/inside/enigmaTracksData/outputBedFile.bed") +bash("bedSort /hive/data/inside/enigmaTracksData/v1.2/outputBedFile.bed \ +/hive/data/inside/enigmaTracksData/v1.2/outputBedFile.bed") startOfAsFile="""table BRCAsplicing -"BRCA1 and BRCA2 variant codes according to PVS1 decision trees (ENIGMA specifications version 1.1.0)" +"BRCA1 and BRCA2 variant codes according to PVS1 decision trees (ENIGMA specifications version 1.2)" ( string chrom; "Reference sequence chromosome or scaffold" uint chromStart; "Start position in chromosome" uint chromEnd; "End position in chromosome" string name; "Var location" uint score; "Not used, all 0" char[1] strand; "- or +" uint thickStart; "Same as chromStart" uint thickEnd; "Same as chromEnd" uint reserved; "RGB value (use R,G,B string in input file)" """ line1 = bash("head -1 "+rawFilePath) line1 = line1.rstrip("\n").split("\t") name = [] for i in range(8): asFileAddition = " lstring extraField"+str(i)+';\t"'+line1[i]+'"\n' startOfAsFile = startOfAsFile+asFileAddition startOfAsFile = startOfAsFile+" string _mouseOver;"+'\t"'+'Field only used as mouseOver'+'"\n' -asFileOutput = open("/hive/data/inside/enigmaTracksData/BRCAsplicing.as","w") +asFileOutput = open("/hive/data/inside/enigmaTracksData/v1.2/BRCAsplicing.as","w") for line in startOfAsFile.split("\n"): if "_mouseOver" in line: asFileOutput.write(line+"\n )") else: asFileOutput.write(line+"\n") asFileOutput.close() -bash("bedToBigBed -as=/hive/data/inside/enigmaTracksData/BRCAsplicing.as -type=bed9+9 -tab \ -/hive/data/inside/enigmaTracksData/outputBedFile.bed /cluster/data/hg38/chrom.sizes \ -/hive/data/inside/enigmaTracksData/BRCAsplicingHg38.bb") +bash("bedToBigBed -as=/hive/data/inside/enigmaTracksData/v1.2/BRCAsplicing.as -type=bed9+9 -tab \ +/hive/data/inside/enigmaTracksData/v1.2/outputBedFile.bed /cluster/data/hg38/chrom.sizes \ +/hive/data/inside/enigmaTracksData/v1.2/BRCAsplicingHg38.bb") -bash("liftOver -bedPlus=9 -tab /hive/data/inside/enigmaTracksData/outputBedFile.bed \ +bash("liftOver -bedPlus=9 -tab /hive/data/inside/enigmaTracksData/v1.2/outputBedFile.bed \ /hive/data/genomes/hg38/bed/liftOver/hg38ToHg19.over.chain.gz \ -/hive/data/inside/enigmaTracksData/outputBedFileHg19.bed /hive/data/inside/enigmaTracksData/unmapped.bed") +/hive/data/inside/enigmaTracksData/v1.2/outputBedFileHg19.bed /hive/data/inside/enigmaTracksData/v1.2/unmapped.bed") -bash("bedToBigBed -as=/hive/data/inside/enigmaTracksData/BRCAsplicing.as -type=bed9+9 -tab \ -/hive/data/inside/enigmaTracksData/outputBedFileHg19.bed /cluster/data/hg19/chrom.sizes \ -/hive/data/inside/enigmaTracksData/BRCAsplicingHg19.bb") +bash("bedToBigBed -as=/hive/data/inside/enigmaTracksData/v1.2/BRCAsplicing.as -type=bed9+9 -tab \ +/hive/data/inside/enigmaTracksData/v1.2/outputBedFileHg19.bed /cluster/data/hg19/chrom.sizes \ +/hive/data/inside/enigmaTracksData/v1.2/BRCAsplicingHg19.bb") -bash("ln -sf /hive/data/inside/enigmaTracksData/BRCAsplicingHg38.bb /gbdb/hg38/bbi/enigma/BRCAsplicing.bb") -bash("ln -sf /hive/data/inside/enigmaTracksData/BRCAsplicingHg19.bb /gbdb/hg19/bbi/enigma/BRCAsplicing.bb") +# The /gbdb symlinks and the hub point at the fixed filenames one level up in +# /hive/data/inside/enigmaTracksData/ - the public hub serves through that chain, so +# this script builds into the versioned v1.2/ dir and the verified .bb files are +# copied onto the staging filenames at release time (see makedoc).