225c0d55992aefae478461bba278644bdfdda3c5
max
  Wed Jan 15 08:33:57 2014 -0800
library changes for the browser box: This changes mostly hdb and jksql,plus - to a smaller extent - various other places in the code that deal
with /gbdb/ files.  The overall aim is to make it possible to have the
data remote at UCSC while having the CGIs on a machine far away. At up to
180msecs distance from UCSC (Europe,Japan), each query can get slow. So
I tried to reduce the number of queries sent to UCSC while allowing to
keep some mysql tables on localhost.

I changed four things:
- extend larry's table cache to include field names. The code uses
"describe" very often, which is slow from remote. With a table name
cache these queries can be handled locally. This is configured in
hg.conf
- mysql "failover" connections: a mysql connection can have a 2nd
connection that is used if a query fails, configured in hg.conf
(I didn't call it "remote" connections, because we use that term already
in the code)
- mysql lazy connects: don't connect a sqlConnection right away, but
only when needed. a mysql connect takes >500msecs from across the
atlantic.
- move gbdb: patch various places that use absolute "/gbdb/" pathnames
to go through a central function that can change the filename of
gbdb files to something else, as configured in hg.conf

Plus patch 1 or 2 lines for more speed + update the hgMirror script

diff --git src/hg/lib/hgFind.c src/hg/lib/hgFind.c
index 32e18e2..f2d2f0a 100644
--- src/hg/lib/hgFind.c
+++ src/hg/lib/hgFind.c
@@ -535,42 +535,58 @@
 
 static struct hgPosTable *addKnownGeneTable(char *db, struct hgPositions *hgp)
 /* Create new table for known genes matches, add it to hgp, and return it. */
 {
 struct hgPosTable *table;
 AllocVar(table);
 if (hTableExists(db, "kgProtMap2"))
     table->description = cloneString("UCSC Genes");
 else
     table->description = cloneString("Known Genes");
 table->name = cloneString("knownGene");
 slAddHead(&hgp->tableList, table);
 return table;
 }
 
+char *makeIndexPath(char *db)
+{
+/* create the pathname with the knowngene index for a db, result needs to be freed */
+char *path = needMem(PATH_LEN);
+char *gbdbLoc = cfgOptionDefault("gbdb.loc", "/gbdb/");
+safef(path, PATH_LEN, "%s%s/knownGene.ix", gbdbLoc, db);
+return path;
+}
+
 static boolean gotFullText(char *db)
 /* Return TRUE if we have full text index. */
 {
-char path[PATH_LEN];
-safef(path, sizeof(path), "/gbdb/%s/knownGene.ix", db);
-if (fileExists(path))
-    return TRUE;
+char *indexPath = makeIndexPath(db);
+boolean result = FALSE;
+
+if (udcIsLocal(indexPath))
+    if (fileExists(indexPath))
+        result = TRUE;
     else
         {
-    warn("%s doesn't exist", path);
-    return FALSE;
+        warn("%s doesn't exist", indexPath);
+        result = FALSE;
         }
+else
+    result = TRUE;
+
+freez(&indexPath);
+return result;
 }
 
 struct tsrPos
 /* Little helper structure tying together search result
  * and pos, used by addKnownGeneItems */
     {
     struct tsrPos *next;	/* Next in list. */
     struct trixSearchResult *tsr;	/* Basically a gene symbol */
     struct hgPos *posList;		/* Associated list of positions. */
     };
 
 static void addKnownGeneItems(struct hgPosTable *table,
 	struct trixSearchResult *tsrList, struct sqlConnection *conn)
 /* Convert tsrList to posList, and hang posList off of table. */
 {
@@ -667,39 +683,38 @@
     struct hgPos *next;
     for (pos = tp->posList; pos != NULL; pos = next)
         {
 	next = pos->next;
 	slAddHead(&posList, pos);
 	}
     }
 table->posList = posList;
 hashFree(&hash);
 dyStringFree(&dy);
 }
 
 boolean findKnownGeneFullText(char *db, char *term,struct hgPositions *hgp)
 /* Look for position in full text. */
 {
-char path[PATH_LEN];
 boolean gotIt = FALSE;
 struct trix *trix;
 struct trixSearchResult *tsrList;
 char *lowered = cloneString(term);
 char *keyWords[HGFIND_MAX_KEYWORDS];
 int keyCount;
 
-safef(path, sizeof(path), "/gbdb/%s/knownGene.ix", db);
+char *path = makeIndexPath(db);
 trix = trixOpen(path);
 tolowers(lowered);
 keyCount = chopLine(lowered, keyWords);
 tsrList = trixSearch(trix, keyCount, keyWords, TRUE);
 if (tsrList != NULL)
     {
     struct hgPosTable *table = addKnownGeneTable(db, hgp);
     struct sqlConnection *conn = hAllocConn(db);
     addKnownGeneItems(table, tsrList, conn);
     hFreeConn(&conn);
     gotIt = TRUE;
     }
 freez(&lowered);
 trixSearchResultFreeList(&tsrList);
 trixClose(&trix);