ea7173bd838a0f4795a1222eb5b49c85defbc4cf max Mon Jul 3 05:13:11 2023 -0700 the proper fix for hgGeneGraph unicode problems, taking back some of the earlier changes, refs #31563 diff --git src/hg/hgGeneGraph/hgGeneGraph src/hg/hgGeneGraph/hgGeneGraph index 2fdac98..34adb6a 100755 --- src/hg/hgGeneGraph/hgGeneGraph +++ src/hg/hgGeneGraph/hgGeneGraph @@ -19,51 +19,54 @@ # dark blue, dashed = only low-throughput data # dark blue, thickness = low-throughput data + text # dark blue + dashed = only pathway data # code review # - os.system is not a security risk here, no variables go into the cmd line # - mysql statements are not escaped, instead all CGI vars are checked for non-alpha letters # hgFixed tables required for this script: ggLink (main table with gene-gene links), # ggLinkEvent (details about link), ggEventDb (details about links from databases), # ggEventText (details about links from text mining), ggDoc (details about documents for ggEventText) # ggGeneName (symbols), ggGeneClass (HPRD/Panther class) # these are default python modules on python 2.7, no errors expected here -import sys, cgi, os, string, urllib.request, urllib.parse, urllib.error, operator, hashlib, codecs +import sys, cgi, os, string, urllib.request, urllib.parse, urllib.error, operator, hashlib from sys import exit from collections import defaultdict, namedtuple from os.path import * import cgitb cgitb.enable() # import the UCSC-specific library sys.path.append(join(dirname(__file__), "pyLib")) try: from hgLib3 import cgiArgs, cgiSetup, cgiString, printContentType, printMenuBar, \ sqlConnect, sqlQuery, errAbort, cfgOption, runCmd, cgiGetAll, printHgcHeader, \ printHgcSection, getNonce, getCspMetaHeader, jsOnEventById, \ jsInlineFinish, webStartGbNoBanner, htmlPageEnd, hConnectCentral, \ - sqlTableExists, readSmallFile + sqlTableExists, readSmallFile, forceUnicode except: print("Content-type: text/html\n") print("Cannot find the directory cgi-bin/pyLib in Apache. This is an installation error.") print("All all parts of cgi-bin installed? Did you do 'make' in kent/src/hg/pyLib?") +import hgLib3 +hgLib3.forceUnicode = True + import pymysql.cursors # not using feedback button for now. Fan would have liked it, but not sure how we can write # to any form of database. # the list of allowed chars in cgi args: digits, letters and dashes legalChars = set(string.digits) legalChars.update(set(string.ascii_letters)) legalChars.update("_-./: ") # number of genes to show on graph by default DEFGENECOUNT="25" # ignore all text mining data with less than X abstracts MINSUPP=2 @@ -124,44 +127,33 @@ "json" : "application/json", "sif" : "application/octet-stream" } # gbdb file names and descriptions geneAnnotFiles = [ ("none", None, "No Annotation", "Do not annotate color genes on graph by external information"), ("gnf2", "gnf2Avg.tab", "GNF2 Expression", "Gene Expression Atlas 2 average across tissues"), ("drugbank", "drugbank.tab", "DrugBank", "DrugBank. Black = gene is targetable with a drug. Mouse-over shows drug names"), ("cosmic" , "cosmicCensus.tab", "Cancer Gene Census", "COSMIC Cancer Gene Census Tumor Types. Black = gene is in cancer gene census. Mouse-over shows cancer type"), ("tcgaMut" , "tcgaMut.tab", "Pan-Cancer Mutations", "TCGA PanCan12 samples with non-silent mutations - Gene mouse-over shows count") ] # ==== FUNCTIONS === -#<script src="//code.jquery.com/jquery-1.10.2.js"></script> -#<script src="//code.jquery.com/ui/1.11.0/jquery-ui.js"></script> -#<script src="../js/readmore.min.js"></script> - -#<link rel="stylesheet" href="//code.jquery.com/ui/1.11.0/themes/smoothness/jquery-ui.css"> -#<link rel="stylesheet" href="../style/HGStyle.css" type="text/css" /> -#<link rel='stylesheet' href='../style/nice_menu.css' type='text/css' /> - def printInlineAndStyles(): - #print('<script src="//code.jquery.com/ui/1.11.0/jquery-ui.js"></script>') print('<script src="//cdn.rawgit.com/jedfoster/Readmore.js/master/readmore.min.js"></script>') - - print((""" <script type="text/javascript" nonce='%s'> $(function() { //$( document ).uitooltip(); //$( document ).uitooltip(); $('[data-toggle="tooltip"]').bsTooltip(); // bootstrap does not really allow HTML in the title attribute // use jquery ui tooltips for the graph var opt = { items: "area", track : true, content: function() {return $(this).prop('title')} }; $("area").uitooltip(opt); @@ -1220,31 +1212,35 @@ # text above graph print("Mouse over or click genes or lines for details. Dashed lines indicate interactions without text mining support. ") print("Click any gene to make it the new center. Click any line to show details about the interaction. ") print(("Only %s-interacting genes and only the most-mentioned/most-curated interactions are shown in the graph. " % (targetGene))) print("See the <a href='../../goldenPath/help/hgGeneGraph.html'>Help Page</a> for details.<br>") # menu above graph # background #fffef5 would be an alternive print('<div style="display:inline-block"> <!-- graph -->') printGraphMenu(conn, targetGene, addNeighbors) print("<p>") # graph itself print('<img src="%s" usemap="#test">' % picName) mapData = open(mapName, "rb").read() - print(mapData.decode("latin1")) # graphviz seems to use latin1 encoding for its output file? + # the data in the graphviz map file seems to be sometimes utf8 and sometimes other codepages (not latin1). So passing it through as a byte string + # to the byte-buffer underneath sys.stdout. I have not found another way to make this work. Not sure why web browsers can show it just fine. + sys.stdout.flush() + sys.stdout.buffer.write(mapData) + sys.stdout.flush() print('</div> <!-- graph -->') print("<p>") def printPmidSearchForm(): " print a little form that allows to search for a PMID " print("<hr>") print("Search for a PMID: ") print('<form action="hgGeneGraph" method="get">') print(' <input type="text" name="search">') print(' <input type="submit" name="Search">') print('</form><p>') def printDisclaimer(): print(''' @@ -1351,31 +1347,31 @@ # convert list of pairs to mysql quoted string list newList = [] for pairData in pairs: g1, g2 = pairData[:2] pairQuote = "('%s', '%s')" % (g1, g2) newList.append(pairQuote) listStr = "(%s)" % ",".join(newList) query = "SELECT gene1, gene2, snippet FROM ggLink "\ "WHERE (gene1, gene2) in %s" % listStr rows = sqlQuery(conn, query) pairSnips = {} for g1, g2, snip in rows: - pairSnips[(g1,g2)]=snip.decode("utf8") + pairSnips[(g1,g2)]=snip return pairSnips def makeHgGeneLink(sym, db): return "<a title='Link to Gene Details page' href='hgGene?db=%s&hgg_gene=%s'>%s</a>" % (db, sym, sym) def showGraphBrowser(): " run graphviz on gene graph around gene, print html img and html map " gene, alg, addNeighbors, sortByCount, geneCount = parseGraphArgs() conn = sqlConnect(GGDB) graphLinks, lowLinks = buildGraph(conn, gene, geneCount, MINSUPP, addNeighbors) weightedLinks, minAbsCount = flattenLink(graphLinks) humanDb = getCgiVar("db", "hg19") @@ -1463,37 +1459,30 @@ if eType == "family": geneStr = "/".join(eGenes) if eName=="": return "%s" % geneStr else: return "%s (%s)" % (eName, geneStr) def printDbRows(conn, rows, onlyDoc=None): " print a row from the ggEventDb table as html " print('<ul class="more">') # sort by database name for row in sorted(rows, key=operator.itemgetter(9)): eventId, causeType, causeName, causeGenes, themeType, themeName, themeGenes, \ relType, relSubtype, sourceDb, sourceId, sourceDesc, docIds, evidence = row - # in Python3, 'longblob' fields get converted to byte strings - causeGenes = causeGenes.decode("utf8") - themeGenes = themeGenes.decode("utf8") - sourceId = sourceId.decode("utf8") - docIds = docIds.decode("utf8") - evidence = evidence.decode("utf8") - if eventId.startswith("ppi_"): isPpi = True else: isPpi = False docSet = set(docIds.split("|")) if eventId.startswith("ppi_iref"): dbName, dbUrlPat = dbData["iref"] dbName = "IRef %s" % sourceDb.capitalize() else: dbName, dbUrlPat = dbData[sourceDb] url = dbUrlPat % sourceId @@ -1554,104 +1543,100 @@ ppiRows.append(r) else: pwyRows.append(r) if len(pwyRows)!=0: print("<h3>Pathways - manually collected, often from reviews:</h3>") printDbRows(conn, pwyRows) if len(ppiRows)!=0: print("<h3>Protein-Protein interactions - manually collected from original source literature:</h3>") print('<p>Studies that report less than %d interactions are marked with *</p>' % LTCUTOFF) printDbRows(conn, ppiRows) def markupSentence(row): " given a MSR-textmining row, print a sentence with the various detected words marked up in bold " - sent = row.sentence.decode("utf8") + sent = row.sentence tStart, tEnd = int(row.themeTokenStart), int(row.themeTokenEnd) cStart, cEnd = int(row.causeTokenStart), int(row.causeTokenEnd) trigToken = int(row.triggerTokenId) sentId = int(row.sentenceId) # put into temporary var to be able to replace " , " later parts = [] for i, word in enumerate(sent.split()): - themeGenes = row.themeGenes.decode("utf8") # longblob = byte string - causeGenes = row.causeGenes.decode("utf8") + themeGenes = row.themeGenes + causeGenes = row.causeGenes if i==tStart or i==cStart: if i==cStart: genes = themeGenes # XX bug in MsrNlp tab file: Inversed? else: genes = causeGenes geneDisp = genes.replace("|", ", ") gene1 = genes.split("|")[0] parts.append( '<b><i><a href="http://www.genecards.org/cgi-bin/carddisp.pl?gene=%s" data-toggle="tooltip" data-placement="top" title="recognized genes: %s, click to go to Gene Cards" target=_blank>' % \ (gene1, geneDisp)) if i==trigToken: parts.append( '<i class="trigger">') if i==tEnd or i==cEnd: parts.append( "</a></i></b>") if i==trigToken+1: parts.append('</i class="trigger">') parts.append(word) line = " ".join(parts) line = line.replace(" , ", ", ").replace(" . ", ". ").rstrip(". ") line = line.replace("-LRB-", "(").replace("-RRB-", ")") return line def iterUniqueInteractions(rows): " iterate over msrNlp rows, but remove interactions we already had (=ignore direction) " doneDocs = set() for row in rows: # make sure to skip duplicated info (=same genes + same document + same sentence) - causeGenes = row.causeGenes.decode("utf8") # longblob = byte string - themeGenes = row.themeGenes.decode("utf8") + causeGenes = row.causeGenes + themeGenes = row.themeGenes docInfo = tuple(sorted([causeGenes, themeGenes, row.docId, row.sentenceId])) if docInfo in doneDocs: continue doneDocs.add(docInfo) yield row def printDocNlpResults(rows): " print text mining results for document-centered view " for row in iterUniqueInteractions(rows): - causeGene, themeGene = row.causeGenes, row.themeGenes - if "Negative" in row.relType: sym = "8867" # unicode right-tack elif "Positive" in row.relType: sym = "8594" # unicode right arrow else: sym = "8212" # unicode long dash print('%s &#%s; %s: "' % (row.causeName, sym, row.themeName)) print(markupSentence(row)) print('"<p>') def printMsrNlpRows(rows): " print msrNlp table rows with marked-up snippets " phrases = [] for row in iterUniqueInteractions(rows): - causeGene, themeGene = row.causeGenes, row.themeGenes - #if addGenes: - #print "%s → %s :" % (causeGene, themeGene) line = markupSentence(row) phrases.append(line) - print(" ... ".join(phrases)) + lines = " ... ".join(phrases) + print(lines) # the blacklist ist not used right now, as we cannot write to the database. # pending further discussions with senior engineers #def readBlackList(conn): #""" return flagged interactions as a set of (cause, theme, pmid (as str)) """ #centralDb = cfgOption("central.db") #blackList = set() #rows = sqlQuery(conn, "SELECT causeGene, themeGene, pmid from %s.ggFeedback" % centralDb) #for row in rows: #blackList.add( (row.causeGene, row.themeGene, str(row.pmid)) ) #return blackList def queryEventText(conn, gene1, gene2): " return rows from the ggEventText table " @@ -1672,68 +1657,68 @@ def prettyDocLinks(conn, pmids): " given a list of pmids, return a list of nice links to articles on our own site " quoteList = ['"%s"' % pmid for pmid in pmids] idStr = "(%s)" % (",".join(quoteList)) # XX remove distinct if not needed anymore query = "SELECT authors, title, journal, year, docId, resCount FROM ggDoc WHERE docId IN %s" % idStr rows = sqlQuery(conn, query) links = [] for row in rows: links.append(prettyDocLink(row)) return ", ".join(links) def prettyDocLink(row, showStar=True): " given a row that includes author/title/year/docId fields, return a nice link to our doc view " - authors = row.authors.decode("utf8").split("; ") # mysql longblob => byte string + authors = row.authors if len(authors)>0: - fAu = authors[0].split(",")[0] + fAu = authors.split(",")[0] else: fAu = "" suffix = "" if len(authors) > 1: suffix = " et al." note = "" if showStar and int(row.resCount)<=LTCUTOFF: note = '<a href="#" title="Low-throughput publication">*</a>' #note = "*" text = "%s %s, %s %s" % (fAu, suffix, row.journal, row.year) mouseOver = None - title = row.title.decode("utf8") # longBlob = byte string + title = row.title if title!="": mouseOver = title.replace('"', "") return docLink(row.docId, text=text, mouseOver=mouseOver)+note def showSnipsLink(conn, gene1, gene2): " show snippets for a gene pair " rows = queryEventText(conn, gene1, gene2) if len(rows)!=0: print('<h3>Text-mined interactions from <A HREF="http://literome.azurewebsites.net/Network?gene1=%s&gene2=%s">Literome</A></h3>' % (gene1, gene2)) byDoc = defaultdict(list) for row in rows: byDoc[row.docId].append(row) for docId, rows in byDoc.items(): print(prettyDocLink(rows[0], showStar=False)) disSuffix = "" - context = rows[0].context.decode("utf8") # longblob = byte string + context = rows[0].context if context!="": contexts = context.split("|") if len(contexts)>1: suffix = "..." else: suffix = "" disSuffix = "(%s%s)" % (contexts[0], suffix) print("%s :" % disSuffix) printMsrNlpRows(rows) print("<br>") def showLink(link): " print details page with db info and snippets for a gene interaction " @@ -1978,30 +1963,31 @@ pair = tuple(sorted([g1, g2])) pairToPmids[pair].add(row.docId) pairs["literome"].add(pair) for pair, pmids in pairToPmids.items(): if len(pmids)>1: pairs["literome (>= 2 PMIDs)"].add(pair) return pairs def showStats(): " " print("<h3>Databases - number of unique pairs</h3>") print("<ul>") conn = sqlConnect(GGDB) + pairs = uniquePairs(conn) for db, pairSet in sorted(pairs.items()): print("<li>%s: %d" % (db, len(pairSet))) print("</ul>") def htmlMiddle(): " print html middle part " sys.stdout.flush() flag = getCgiVar("flag") if flag!=None: flagInteraction(flag) exit(0) remove = getCgiVar("remove") @@ -2028,31 +2014,33 @@ showGraphBrowser() def main(): cgiSetup() format = getCgiVar("format") if format in ["pdf", "svg", "sif", "json"]: conn = sqlConnect(GGDB) gene, alg, addNeighbors, sortByCount, geneCount = parseGraphArgs() graphLinks, lowLinks = buildGraph(conn, gene, geneCount, MINSUPP, addNeighbors) weightedLinks, minAbsCount = flattenLink(graphLinks) printGraph(conn, weightedLinks, alg, addNeighbors, gene, format) sys.exit(0) + # Apache doesn't set LANG, so the default encoding of stdout is ASCII: Change it to utf8 + import codecs sys.stdout = codecs.getwriter('utf8')(sys.stdout.buffer) printContentType() if cgiString("debug") is not None: global DEBUG DEBUG = True htmlHeader() printInlineAndStyles() htmlMiddle() jsInlineFinish() htmlPageEnd() main()