225c0d55992aefae478461bba278644bdfdda3c5
max
  Wed Jan 15 08:33:57 2014 -0800
library changes for the browser box: This changes mostly hdb and jksql,plus - to a smaller extent - various other places in the code that deal
with /gbdb/ files.  The overall aim is to make it possible to have the
data remote at UCSC while having the CGIs on a machine far away. At up to
180msecs distance from UCSC (Europe,Japan), each query can get slow. So
I tried to reduce the number of queries sent to UCSC while allowing to
keep some mysql tables on localhost.

I changed four things:
- extend larry's table cache to include field names. The code uses
"describe" very often, which is slow from remote. With a table name
cache these queries can be handled locally. This is configured in
hg.conf
- mysql "failover" connections: a mysql connection can have a 2nd
connection that is used if a query fails, configured in hg.conf
(I didn't call it "remote" connections, because we use that term already
in the code)
- mysql lazy connects: don't connect a sqlConnection right away, but
only when needed. a mysql connect takes >500msecs from across the
atlantic.
- move gbdb: patch various places that use absolute "/gbdb/" pathnames
to go through a central function that can change the filename of
gbdb files to something else, as configured in hg.conf

Plus patch 1 or 2 lines for more speed + update the hgMirror script

diff --git src/hg/lib/wigDataStream.c src/hg/lib/wigDataStream.c
index 4e1801f..81d4008 100644
--- src/hg/lib/wigDataStream.c
+++ src/hg/lib/wigDataStream.c
@@ -1,25 +1,26 @@
 /*	wigDataStream - an object to access wiggle data values from
  *	either a DB access or from a .wig text file (==custom track)
  */
 #include "common.h"
 #include "memalloc.h"
 #include "wiggle.h"
 #include "portable.h"
 #include "hgColors.h"
 #include "obscure.h"
 #include "customTrack.h"
+#include "udc.h"
 
 /*	size of memory to work with during wiggle data fetch operations,
  *	~4 Gb is good enough for about 750,000,000 bases	*/
 /* raising this number has a big effect and slows down some tracks ?
  * namely, psl/bed custom tracks, this doesn't make any sense */
 #define	HUGE_LIMIT	2100000000
 
 
 /*	Routines that are not strictly part of the wigDataStream object,
 	but they are used to do things with the object.
 **************************************************************************/
 
 void wigStatsTableHeading(FILE * fh, boolean htmlOut)
 /*	Print the single html (or text) table row for statistics
 	column headings */
@@ -203,50 +204,50 @@
     sqlRow = sqlNextRow(wds->sr);
     if (sqlRow == NULL)
 	return FALSE;
     /*	skip the bin column sqlRow[0]	*/
     for (i=1; i <= maxRow; ++i)
 	{
 	row[i-1] = sqlRow[i];
 	}
     }
 return TRUE;
 }
 
 static void findWibFile(struct wiggleDataStream *wds, char *file)
 /* look for file in full pathname given, or in same directory */
 {
-wds->wibFile = cloneString(file);
-wds->wibFH = open(wds->wibFile, O_RDONLY);
-if (wds->wibFH == -1)
+wds->wibFile = hCloneRewriteFileName(file);
+wds->wibFH = udcFileMayOpen(wds->wibFile, NULL);
+if (wds->wibFH == NULL)
     {
     char *baseName = strrchr(wds->wibFile, '/');
     if (baseName)
-	wds->wibFH = open(baseName+1, O_RDONLY);
-    if ((NULL == baseName) || (wds->wibFH == -1))
-	errAbort("openWibFile: failed to open %s", wds->wibFile);
+	wds->wibFH = udcFileMayOpen(baseName+1, NULL);
+    if ((NULL == baseName) || (wds->wibFH == NULL))
+	errAbort("findWibFile: failed to open %s", wds->wibFile);
     }
 }
 
 static void openWibFile(struct wiggleDataStream *wds, char *file)
 {
 if (wds->wibFile)
     {		/*	close and open only if different */
     if (differentString(wds->wibFile,file))
 	{
 	if (wds->wibFH > 0)
-	    close(wds->wibFH);
+	    udcFileClose(&wds->wibFH);
 	freeMem(wds->wibFile);
 	findWibFile(wds, file);
 	}
     }
 else
     findWibFile(wds, file);
 
 verbose(VERBOSE_HIGHEST, "#\topened wib file: %s\n", wds->wibFile);
 }
 
 static void setCompareByte(struct wiggleDataStream *wds,
 	double lower, double range)
 {
 if (wds->limit_0 < lower)
     wds->ucLowerLimit = 0;
@@ -319,31 +320,31 @@
 
 AllocVar(bed);
 bed->chrom = cloneString(chrom);
 bed->chromStart = start;
 bed->chromEnd = end;
 safef(name, sizeof(name), "%s.%u",
     chrom, lineCount);
 bed->name = cloneString(name);
 return bed;
 }
 
 static void closeWibFile(struct wiggleDataStream *wds)
 /*	if there is a Wib file open, close it	*/
 {
 if (wds->wibFH > 0)
-    close(wds->wibFH);
+    udcFileClose(&wds->wibFH);
 wds->wibFH = -1;
 if (wds->wibFile)
     freez(&wds->wibFile);
 }
 
 static void closeWigConn(struct wiggleDataStream *wds)
 {
 lineFileClose(&wds->lf);
 closeWibFile(wds);	/*	closes only if it is open	*/
 if (wds->conn)
     {
     sqlFreeResult(&wds->sr);
     hFreeConn(&wds->conn);
     }
 if (wds->sqlConstraint)
@@ -1162,34 +1163,34 @@
 	bytesSkipped += wiggle->count;
 	continue;	/*	next SQL row	*/
 	}
     if (!skipDataRead)
 	{
 	int j;	/*	loop counter through readData	*/
 	unsigned char *datum;    /* to walk through readData bytes */
 	unsigned char *readData;    /* the bytes read in from the file */
 	size_t bytesToRead;	/* to check read validity */
 	size_t bytesRead;	/* to check read validity */
 
 	openWibFile(wds, wiggle->file);
 		    /* possibly open a new wib file */
 	readData = (unsigned char *) needMem((size_t) (wiggle->count + 1));
 	wds->bytesRead += wiggle->count;
-	lseek(wds->wibFH, wiggle->offset, SEEK_SET);
+	udcSeek(wds->wibFH, wiggle->offset);
 	bytesToRead = (size_t) wiggle->count * (size_t) sizeof(unsigned char);
 
-	bytesRead = read(wds->wibFH, readData, bytesToRead);
+	bytesRead = udcRead(wds->wibFH, readData, bytesToRead);
 
 	if (bytesToRead != bytesRead)
 	    {
 	    errAbort("wig_getData: failed to read %llu bytes from %s\n",
 		(unsigned long long)bytesToRead, wiggle->file);
 	    }
 
 	verbose(VERBOSE_PER_VALUE_LEVEL,
 		"#\trow: %llu, reading: %u bytes\n", rowCount, wiggle->count);
 
 	/*	The third element of the for() statement takes care of the end
 	 *	of loop operations for the case of the 'continue;'
 	 *	statement as well as the normal end of loop business
 	 */
 	datum = readData;