be4311c07e14feb728abc6425ee606ffaa611a58 markd Fri Jan 22 06:46:58 2021 -0800 merge with master diff --git src/hg/lib/trackDbCache.c src/hg/lib/trackDbCache.c index 5811310..32ef75e 100644 --- src/hg/lib/trackDbCache.c +++ src/hg/lib/trackDbCache.c @@ -1,23 +1,24 @@ #include #include #include "common.h" #include "trackDb.h" #include "hex.h" #include "portable.h" #include "localmem.h" #include "hgConfig.h" +#include static char *trackDbCacheDir; // the trackDb cache directory root #ifndef CACHELOG #define cacheLog(a,...) #else /* CACHELOG */ void vaCacheLog(char *format, va_list args) /* Call top of warning stack to issue warning. */ { fputs("cacheLog: ", stderr); vfprintf(stderr, format, args); fprintf(stderr, "\n"); fflush(stderr); } @@ -151,49 +152,87 @@ return newList; } static char *cacheDirForDbAndTable(char *string, char *tdbPathString) /* Determine the directory name for the trackDb cache files */ { char dirName[4096]; if (tdbPathString == NULL) safef(dirName, sizeof dirName, "%s/%s", trackDbCacheDir, string); else safef(dirName, sizeof dirName, "%s/%s.%s", trackDbCacheDir, string, tdbPathString); return cloneString(dirName); } +static time_t checkIncFiles(char *dirName, time_t hubTime) +/* Check the incFiles.txt file (if present) to see if it has files with newer dates than in time. */ +{ +char fileName[4096]; +safef(fileName, sizeof fileName, "%s/%s", dirName, "incFiles.txt"); +struct lineFile *lf = lineFileMayOpen(fileName, TRUE); + +if (lf == NULL) + return hubTime; + +char *line; +while (lineFileNextReal(lf, &line)) + { + struct udcFile *checkCache = udcFileMayOpen(line, NULL); + if (checkCache == NULL) // if the file disappeared, let's expire the cache! + { + hubTime = time(NULL); + break; + } + + time_t incTime = udcUpdateTime(checkCache); + if (incTime > hubTime) + hubTime = incTime; + udcFileClose(&checkCache); + } + +lineFileClose(&lf); + +return hubTime; +} + static struct trackDb *checkCache(char *string, char *tdbPathString, time_t time) /* Check to see if this db or hub has a cached trackDb. string is either a db * or a SHA1 calculated from a hubUrl. Use time to see if cache should be flushed. */ { char *dirName = cacheDirForDbAndTable(string, tdbPathString); if (!isDirectory(dirName)) { cacheLog("abandoning cache search for %s, no directory", string); return NULL; } +// check the incFiles file and see if any of the included files are newer +// than the time we've been given +time = checkIncFiles(dirName, time); + // look for files named by the address they use struct slName *files = listDir(dirName, "*"); char fileName[4096]; for(; files; files = files->next) { if (sameString(files->name, "name.txt")) continue; + if (sameString(files->name, "incFiles.txt")) + continue; + safef(fileName, sizeof fileName, "%s/%s", dirName, files->name); cacheLog("checking cache file %s", fileName); struct stat statBuf; if (stat(fileName, &statBuf) < 0) { // if we can't stat the shared memory, let's just toss it cacheLog("can't stat file %s, unlinking", fileName); mustRemove(fileName); continue; } if (statBuf.st_mtime < time) { // if the cache is older than the data, toss it @@ -259,32 +298,32 @@ return checkCache(db, tdbPathString, time); } struct trackDb *trackDbHubCache(char *trackDbUrl, time_t time) { cacheLog("checking for cache for hub %s at time %ld", trackDbUrl, time); unsigned char hash[SHA_DIGEST_LENGTH]; SHA1((const unsigned char *)trackDbUrl, strlen(trackDbUrl), hash); char newName[(SHA_DIGEST_LENGTH + 1) * 2]; hexBinaryString(hash, SHA_DIGEST_LENGTH, newName, (SHA_DIGEST_LENGTH + 1) * 2); return checkCache(newName, NULL, time); } -static void cloneTdbListToSharedMem(char *string, char *tdbPathString, struct trackDb *list, unsigned long size, char *name) -/* Allocate shared memory and clone trackDb list into it. */ +static void cloneTdbListToSharedMem(char *string, char *tdbPathString, struct trackDb *list, unsigned long size, char *name, char *incFiles) +/* Allocate shared memory and clone trackDb list into it. Record the names of include files in incFiles if not empty. */ { static int inited = 0; if (inited == 0) { srandom(time(NULL)); inited = 1; } int oflags=O_RDWR | O_CREAT; char *dirName = cacheDirForDbAndTable(string, tdbPathString); if (!isDirectory(dirName)) { @@ -357,52 +396,61 @@ char fileName[4096]; safef(fileName, sizeof fileName, "%s/%ld.%d", dirName, paddress, TRACKDB_VERSION); cacheLog("renaming %s to %s", tempFileName, fileName); mustRename(tempFileName, fileName); // write out the name of the trackDb being cached. safef(fileName, sizeof fileName, "%s/name.txt", dirName); FILE *stream = mustOpen(fileName, "w"); if (tdbPathString == NULL) fprintf(stream, "%s\n", name); else fprintf(stream, "%s.%s\n", name,tdbPathString); carefulClose(&stream); + +// write out included files if present +if (!isEmpty(incFiles)) + { + safef(fileName, sizeof fileName, "%s/incFiles.txt", dirName); + stream = mustOpen(fileName, "w"); + fputs(incFiles, stream); + carefulClose(&stream); + } } void trackDbCloneTdbListToSharedMem(char *db, char *tdbPathString, struct trackDb *list, unsigned long size) /* For this native db, allocate shared memory and clone trackDb list into it. */ { cacheLog("cloning memory for db %s %ld", db, size); -cloneTdbListToSharedMem(db, tdbPathString, list, size, db); +cloneTdbListToSharedMem(db, tdbPathString, list, size, db, NULL); } -void trackDbHubCloneTdbListToSharedMem(char *trackDbUrl, struct trackDb *list, unsigned long size) -/* For this hub, Allocate shared memory and clone trackDb list into it. */ +void trackDbHubCloneTdbListToSharedMem(char *trackDbUrl, struct trackDb *list, unsigned long size, char *incFiles) +/* For this hub, Allocate shared memory and clone trackDb list into it. incFiles has a list of include files that may be null. */ { if ((*trackDbUrl == '.') || (list == NULL)) // don't cache empty lists or collections return; cacheLog("cloning memory for hub %s %ld", trackDbUrl, size); unsigned char hash[SHA_DIGEST_LENGTH]; SHA1((const unsigned char *)trackDbUrl, strlen(trackDbUrl), hash); char newName[(SHA_DIGEST_LENGTH + 1) * 2]; hexBinaryString(hash, SHA_DIGEST_LENGTH, newName, (SHA_DIGEST_LENGTH + 1) * 2); -cloneTdbListToSharedMem(newName, NULL, list, size, trackDbUrl); +cloneTdbListToSharedMem(newName, NULL, list, size, trackDbUrl, incFiles); } boolean trackDbCacheOn() /* Check to see if we're caching trackDb contents. */ { static boolean checkedCache = FALSE; static boolean doCache = FALSE; if (!checkedCache) { trackDbCacheDir = cfgOption("cacheTrackDbDir"); if (isNotEmpty(trackDbCacheDir)) { makeDirsOnPath(trackDbCacheDir); chmod(trackDbCacheDir, 0777);