225c0d55992aefae478461bba278644bdfdda3c5
max
  Wed Jan 15 08:33:57 2014 -0800
library changes for the browser box: This changes mostly hdb and jksql,plus - to a smaller extent - various other places in the code that deal
with /gbdb/ files.  The overall aim is to make it possible to have the
data remote at UCSC while having the CGIs on a machine far away. At up to
180msecs distance from UCSC (Europe,Japan), each query can get slow. So
I tried to reduce the number of queries sent to UCSC while allowing to
keep some mysql tables on localhost.

I changed four things:
- extend larry's table cache to include field names. The code uses
"describe" very often, which is slow from remote. With a table name
cache these queries can be handled locally. This is configured in
hg.conf
- mysql "failover" connections: a mysql connection can have a 2nd
connection that is used if a query fails, configured in hg.conf
(I didn't call it "remote" connections, because we use that term already
in the code)
- mysql lazy connects: don't connect a sqlConnection right away, but
only when needed. a mysql connect takes >500msecs from across the
atlantic.
- move gbdb: patch various places that use absolute "/gbdb/" pathnames
to go through a central function that can change the filename of
gbdb files to something else, as configured in hg.conf

Plus patch 1 or 2 lines for more speed + update the hgMirror script

diff --git src/inc/linefile.h src/inc/linefile.h
index 1f28171..7de2132 100644
--- src/inc/linefile.h
+++ src/inc/linefile.h
@@ -1,25 +1,26 @@
 /* lineFile - stuff to rapidly read text files and parse them into
  * lines.
  *
  * This file is copyright 2002 Jim Kent, but license is hereby
  * granted for all use - public, private or commercial. */
 
 #ifndef LINEFILE_H
 #define LINEFILE_H
 
 #include "dystring.h"
+#include "udc.h"
 
 #ifdef USE_TABIX
 #include "tabix.h"
 #endif
 
 enum nlType {
  nlt_undet, /* undetermined */
  nlt_unix,  /* lf   */
  nlt_dos,   /* crlf */
  nlt_mac    /* cr   */
 };
 
 struct metaOutput
 /* struct to store list of file handles to output meta data to
  * meta data is text after # */
@@ -42,54 +43,58 @@
     int lineIx;			/* Current line. */
     int lineStart;		/* Offset of line in buffer. */
     int lineEnd;		/* End of line in buffer. */
     bool zTerm;			/* Replace '\n' with zero? */
     enum nlType nlType;         /* type of line endings: dos, unix, mac or undet */
     bool reuse;			/* Set if reusing input. */
     char *buf;			/* Buffer. */
     struct pipeline *pl;        /* pipeline if reading compressed */
     struct metaOutput *metaOutput;   /* list of FILE handles to write metaData to */
     bool isMetaUnique;          /* if set, do not repeat comments in output */
     struct hash *metaLines;     /* save lines to suppress repetition */
 #ifdef USE_TABIX
     tabix_t *tabix;		/* A tabix-compressed file and its binary index file (.tbi) */
     ti_iter_t tabixIter;	/* An iterator to get decompressed indexed lines of text */
 #endif
+    struct udcFile *udcFile;    /* udc file if using caching */
     struct dyString *fullLine;  // Filled with full line when a lineFileNextFull is called
     struct dyString *rawLines;  // Filled with raw lines used to create the full line
     boolean fullLineReuse;      // If TRUE, next call to lineFileNextFull will get
                                 // already built fullLine
     void *dataForCallBack;                                 // ptr to data needed for callbacks
     void(*checkSupport)(struct lineFile *lf, char *where); // check if operation supported 
     boolean(*nextCallBack)(struct lineFile *lf, char **retStart, int *retSize); // next line callback
     void(*closeCallBack)(struct lineFile *lf);             // close callback
     };
 
 char *getFileNameFromHdrSig(char *m);
 /* Check if header has signature of supported compression stream,
    and return a phoney filename for it, or NULL if no sig found. */
 
 struct lineFile *lineFileDecompressFd(char *name, bool zTerm, int fd);
 /* open a linefile with decompression from a file or socket descriptor */
 
 struct lineFile *lineFileDecompressMem(bool zTerm, char *mem, long size);
 /* open a linefile with decompression from a memory stream */
 
 struct lineFile *lineFileMayOpen(char *fileName, bool zTerm);
 /* Try and open up a lineFile. If fileName ends in .gz, .Z, or .bz2,
  * it will be read from a decompress pipeline. */
 
+struct lineFile *lineFileUdcMayOpen(char *fileName, bool zTerm);
+/* Open a lineFile through the UDC */
+
 struct lineFile *lineFileOpen(char *fileName, bool zTerm);
 /* Open up a lineFile or die trying If fileName ends in .gz, .Z, or .bz2,
  * it will be read from a decompress pipeline.. */
 
 struct lineFile *lineFileAttach(char *fileName, bool zTerm, int fd);
 /* Wrap a line file around an open'd file. */
 
 struct lineFile *lineFileStdin(bool zTerm);
 /* Wrap a line file around stdin. */
 
 struct lineFile *lineFileOnString(char *name, bool zTerm, char *s);
 /* Wrap a line file object around string in memory. This buffer
  * have zeroes written into it if zTerm is non-zero.  It will
  * be freed when the line file is closed. */