225c0d55992aefae478461bba278644bdfdda3c5 max Wed Jan 15 08:33:57 2014 -0800 library changes for the browser box: This changes mostly hdb and jksql,plus - to a smaller extent - various other places in the code that deal with /gbdb/ files. The overall aim is to make it possible to have the data remote at UCSC while having the CGIs on a machine far away. At up to 180msecs distance from UCSC (Europe,Japan), each query can get slow. So I tried to reduce the number of queries sent to UCSC while allowing to keep some mysql tables on localhost. I changed four things: - extend larry's table cache to include field names. The code uses "describe" very often, which is slow from remote. With a table name cache these queries can be handled locally. This is configured in hg.conf - mysql "failover" connections: a mysql connection can have a 2nd connection that is used if a query fails, configured in hg.conf (I didn't call it "remote" connections, because we use that term already in the code) - mysql lazy connects: don't connect a sqlConnection right away, but only when needed. a mysql connect takes >500msecs from across the atlantic. - move gbdb: patch various places that use absolute "/gbdb/" pathnames to go through a central function that can change the filename of gbdb files to something else, as configured in hg.conf Plus patch 1 or 2 lines for more speed + update the hgMirror script diff --git src/hg/hgTracks/wigTrack.c src/hg/hgTracks/wigTrack.c index ff5dfaa..01d57f9 100644 --- src/hg/hgTracks/wigTrack.c +++ src/hg/hgTracks/wigTrack.c @@ -6,30 +6,31 @@ #include "obscure.h" #include "hash.h" #include "linefile.h" #include "jksql.h" #include "hdb.h" #include "hgTracks.h" #include "wiggle.h" #include "hmmstats.h" #include "scoredRef.h" #ifndef GBROWSE #include "customTrack.h" #endif /* GBROWSE */ #include "wigCommon.h" #include "imageV2.h" #include "memgfx.h" +#include "udc.h" struct wigItem /* A wig track item. */ { struct wigItem *next; int start, end; /* Start/end in chrom (aka browser) coordinates. */ char *db; /* Database */ int ix; /* Position in list. */ int height; /* Pixel height of item. */ unsigned span; /* each value spans this many bases */ unsigned count; /* number of values to use */ unsigned offset; /* offset in File to fetch data */ char *file; /* path name to data file, one byte per value */ double lowerLimit; /* lowest data value in this block */ @@ -273,33 +274,30 @@ static char *previousFileName = (char *)NULL; char spanName[SMALLBUF]; struct hashEl *el; char *trackName = tg->track; /* Allocate trackSpans one time only, for all tracks */ if (! trackSpans) trackSpans = newHash(4); wi->start = wiggle->chromStart; wi->end = wiggle->chromEnd; if ((previousFileName == (char *)NULL) || differentString(previousFileName,wiggle->file)) { - if (! fileExists(wiggle->file)) - errAbort("wigSetItemData: can't open file '%s' (%s)", - wiggle->file, strerror(errno)); freez(&previousFileName); previousFileName = cloneString(wiggle->file); } wi->file = cloneString(wiggle->file); wi->span = wiggle->span; wi->count = wiggle->count; wi->offset = wiggle->offset; wi->lowerLimit = wiggle->lowerLimit; wi->dataRange = wiggle->dataRange; wi->validCount = wiggle->validCount; wi->sumData = wiggle->sumData; wi->sumSquares = wiggle->sumSquares; /* see if we have a spans hash for this track already */ @@ -1275,42 +1273,40 @@ if (retGraphLowerLimit != NULL) *retGraphLowerLimit = graphLowerLimit; } struct preDrawContainer *wigLoadPreDraw(struct track *tg, int seqStart, int seqEnd, int width) /* Do bits that load the predraw buffer tg->preDrawContainer. */ { /* Just need to do this once... */ if (tg->preDrawContainer) return tg->preDrawContainer; struct wigItem *wi; double pixelsPerBase = scaleForPixels(width); double basesPerPixel = 1.0; int itemCount = 0; -char currentFile[PATH_LEN]; -int wibFH = 0; /* file handle to binary file */ +char *currentFile = NULL; +struct udcFile *wibFH = NULL; /* file handle to binary file */ int i; /* an integer loop counter */ int x1 = 0; /* screen coordinates */ int x2 = 0; /* screen coordinates */ int usingDataSpan = 1; /* will become larger if possible */ if (tg->items == NULL) return NULL; -currentFile[0] = '\0'; - if (pixelsPerBase > 0.0) basesPerPixel = 1.0 / pixelsPerBase; /* width - width of drawing window in pixels * pixelsPerBase - pixels per base * basesPerPixel - calculated as 1.0/pixelsPerBase */ itemCount = 0; /* Allocate predraw and save it and related info in the track. */ struct preDrawContainer *pre = tg->preDrawContainer = initPreDrawContainer(width); struct preDrawElement *preDraw = pre->preDraw; /* to accumulate everything in prep for draw */ int preDrawZero = pre->preDrawZero; /* location in preDraw where screen starts */ int preDrawSize = pre->preDrawSize; /* size of preDraw array */ @@ -1319,53 +1315,51 @@ /* walk through all the data and prepare the preDraw array */ for (wi = tg->items; wi != NULL; wi = wi->next) { size_t bytesRead; /* to check fread being OK */ int dataOffset = 0; /* within data block during drawing */ ++itemCount; /* Now that we know what Span to draw, see if this item should be * drawn at all. */ if (usingDataSpan == wi->span) { /* Check our data file, see if we need to open a new one */ - if (differentString(currentFile,"")) + if (differentStringNullOk(currentFile,"")) { - if (differentString(currentFile,wi->file)) + if (differentStringNullOk(currentFile,wi->file)) { if (wibFH > 0) { - close(wibFH); + udcFileClose(&wibFH); freeMem(currentFile); } - strncpy(currentFile, wi->file, PATH_LEN); - currentFile[PATH_LEN-1] = '\0'; - wibFH = open(currentFile, O_RDONLY); - if (-1 == wibFH) - errAbort("openWibFile: failed to open %s", currentFile); + currentFile = hCloneRewriteFileName(wi->file); + wibFH = udcFileMayOpen(currentFile, NULL); + if ((struct udcFile*)-1 == wibFH) + errAbort("hgTracks/wigLoadPreDraw: failed to open wiggle %s", currentFile); } } else { - strncpy(currentFile, wi->file, PATH_LEN-1); - currentFile[PATH_LEN-1] = '\0'; - wibFH = open(currentFile, O_RDONLY); - if (-1 == wibFH) - errAbort("openWibFile: failed to open %s", currentFile); + currentFile = hCloneRewriteFileName(wi->file); + wibFH = udcFileMayOpen(currentFile, NULL); + if ((struct udcFile*)-1 == wibFH) + errAbort("hgTracks/wigLoadPreDraw: failed to open wiggle %s", currentFile); } /* Ready to draw, what do we know: * the feature being processed: * chrom coords: [wi->start : wi-end) * * The data to be drawn: to be read from file f at offset wi->Offset * data points available: wi->Count, representing wi->Span bases * for each data point * * The drawing window, in pixels: * xOff = left margin, yOff = top margin, h = height of drawing window * drawing window in chrom coords: seqStart, seqEnd * 'basesPerPixel' is known, 'pixelsPerBase' is known */ /* let's check end point screen coordinates. If they are @@ -1377,33 +1371,33 @@ */ double x1d = (double)(wi->start - seqStart) * pixelsPerBase; x1 = round(x1d); double x2d = (double)((wi->start+(wi->count * usingDataSpan))-seqStart) * pixelsPerBase; x2 = round(x2d); /* this used to be if (x2 > x1) which often caused reading of blocks * when they were merely x2 = x1 + 1 due to rounding errors as * they became integers. This double comparison for something over * 0.5 will account for rounding errors that are really small, but * still handle a slipping window size as it walks across the screen */ if ((x2d - x1d) > 0.5) { unsigned char *readData; /* the bytes read in from the file */ - lseek(wibFH, wi->offset, SEEK_SET); + udcSeek(wibFH, wi->offset); readData = (unsigned char *) needMem((size_t) (wi->count + 1)); - bytesRead = read(wibFH, readData, + bytesRead = udcRead(wibFH, readData, (size_t) wi->count * (size_t) sizeof(unsigned char)); /* walk through all the data in this block */ for (dataOffset = 0; dataOffset < wi->count; ++dataOffset) { unsigned char datum = readData[dataOffset]; if (datum != WIG_NO_DATA) { x1 = ((wi->start-seqStart) + (dataOffset * usingDataSpan)) * pixelsPerBase; x2 = x1 + (usingDataSpan * pixelsPerBase); for (i = x1; i <= x2; ++i) { int xCoord = preDrawZero + i; if ((xCoord >= 0) && (xCoord < preDrawSize)) { double dataValue = @@ -1435,32 +1429,33 @@ double upperLimit; preDraw[xCoord].count += wi->validCount; upperLimit = wi->lowerLimit + wi->dataRange; if (upperLimit > preDraw[xCoord].max) preDraw[xCoord].max = upperLimit; if (wi->lowerLimit < preDraw[xCoord].min) preDraw[xCoord].min = wi->lowerLimit; preDraw[xCoord].sumData += wi->sumData; preDraw[xCoord].sumSquares += wi->sumSquares; } } } /* Draw if span is correct */ } /* for (wi = tg->items; wi != NULL; wi = wi->next) */ if (wibFH > 0) { - close(wibFH); + udcFileClose(&wibFH); wibFH = 0; + freeMem(currentFile); } return pre; } static void wigDrawItems(struct track *tg, int seqStart, int seqEnd, struct hvGfx *hvg, int xOff, int yOff, int width, MgFont *font, Color color, enum trackVisibility vis) /* Draw wiggle items that resolve to doing a box for each pixel. */ { struct preDrawContainer *pre = wigLoadPreDraw(tg, seqStart, seqEnd, width); if (pre != NULL) { wigDrawPredraw(tg, seqStart, seqEnd, hvg, xOff, yOff, width, font, color, vis, pre, pre->preDrawZero, pre->preDrawSize, &tg->graphUpperLimit, &tg->graphLowerLimit);