ea7173bd838a0f4795a1222eb5b49c85defbc4cf
max
  Mon Jul 3 05:13:11 2023 -0700
the proper fix for hgGeneGraph unicode problems, taking back some of the earlier changes, refs #31563

diff --git src/hg/pyLib/hgLib3.py src/hg/pyLib/hgLib3.py
index 3280587..6c28fda 100644
--- src/hg/pyLib/hgLib3.py
+++ src/hg/pyLib/hgLib3.py
@@ -45,30 +45,32 @@
 # like in the kent tree, we keep track of whether we have already output the content-type line
 contentLineDone = False
 
 # show the bot delay warning message before other printing is done?
 doWarnBot = False
 # current bot delay in milliseconds
 botDelayMsecs = 0
 
 # two global variables: the first is the botDelay limit after which the page is slowed down and a warning is shown
 # the second is the limit after which the page is not shown anymore
 botDelayWarn = 1000
 botDelayBlock = 5000
 
 jksqlTrace = False
 
+forceUnicode = False
+
 def warn(format, *args):
     print (format % args)
 
 def errAbort(msg, status=None, headers = None):
     " show msg and abort. Like errAbort.c "
     printContentType(status=status, headers=headers)
     print(msg)
     exit(0)
 
 def debug(level, msg):
     " output debug message with a given verbosity level "
     if verboseLevel >= level:
         printContentType()
         print(msg+"<br>")
         sys.stdout.flush()
@@ -160,30 +162,43 @@
     cfg = parseHgConf()
     if "slow-db.host" not in cfg:
         return
 
     host, user, passwd = cfg["slow-db.host"], cfg["slow-db.user"], cfg["slow-db.password"]
     sys.stderr.write("SQL_CONNECT 0 %s %s %s\n" % (host, user, passwd))
     db = conn.db
     failoverConn = pymysql.connect(host=host, user=user, passwd=passwd, db=db)
     conn.failoverConn = failoverConn
 
 def _timeDeltaSeconds(time1, time2):
     " convert time difference to total seconds for python 2.6 "
     td = time1 - time2
     return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6
 
+def byteToUnicode(rows):
+    " makes sure that all fields are of type 'str' and no single field is a byte string "
+    newRows = []
+    for row in rows:
+        newRow = []
+        for field in row:
+            if isinstance(field, bytes):
+                field = field.decode("utf8")
+            newRow.append(field)
+        newRows.append(newRow)
+
+    return newRows
+
 def sqlQuery(conn, query, args=None):
     """ Return all rows for query, placeholders can be used, args is a list to
     replace placeholders, to prevent Mysql injection.  Never do replacement
     with %s yourself, unless the value is coming from inside the program. This
     is called a "parameterized query". There is only %s, %d does not work.
 
     example:
     query = "SELECT contents FROM table WHERE id=%(id)s and name=%(name)s;"
     rows = sqlQuery(conn, query, {"id":1234, "name":"hiram"})
     """
     cursor = conn.cursor()
 
     if jksqlTrace:
         from datetime import datetime
         sys.stderr.write("SQL_QUERY 0 %s %s %s %s\n" % (conn.host, conn.db, query, args))
@@ -215,31 +230,36 @@
         cursor = conn.failoverConn.cursor()
         rows = cursor.execute(query, args)
 
     if jksqlTrace:
         startTime = datetime.now()
 
     data = cursor.fetchall()
     cursor.close()
 
     if jksqlTrace:
         timeDiff = _timeDeltaSeconds(datetime.now(), startTime)
         sys.stderr.write("SQL_FETCH 0 %s %s %.3f\n" % (conn.host, conn.db, timeDiff))
 
     colNames = [desc[0] for desc in cursor.description]
     Rec = namedtuple("MysqlRow", colNames)
+
+    if forceUnicode:
+        data = byteToUnicode(data)
+
     recs = [Rec(*row) for row in data]
+
     return recs
 
 def htmlPageEnd(oldJquery=False):
     " close html body/page "
     print("</body>")
     print("</html>")
 
 def printMenuBar(oldJquery=False):
     baseDir = "../"
     " print the menubar. Mostly copied from src/hg/hgMenuBar.c "
 
     print ("<noscript><div class='noscript'><div class='noscript-inner'><p><b>JavaScript is disabled in your web browser</b></p>")
     print ("<p>You must have JavaScript enabled in your web browser to use the Genome Browser</p></div></div></noscript>\n")
 
     menuPath = "../htdocs/inc/globalNavBar.inc"