be4311c07e14feb728abc6425ee606ffaa611a58
markd
  Fri Jan 22 06:46:58 2021 -0800
merge with master

diff --git src/hg/lib/trackDbCache.c src/hg/lib/trackDbCache.c
index 5811310..32ef75e 100644
--- src/hg/lib/trackDbCache.c
+++ src/hg/lib/trackDbCache.c
@@ -1,23 +1,24 @@
 #include <sys/mman.h>
 #include <openssl/sha.h>
 #include "common.h"
 #include "trackDb.h"
 #include "hex.h"
 #include "portable.h"
 #include "localmem.h"
 #include "hgConfig.h"
+#include <time.h>
 
 static char *trackDbCacheDir;     // the trackDb cache directory root   
 
 #ifndef CACHELOG
 #define cacheLog(a,...)
 #else /* CACHELOG */
 void vaCacheLog(char *format, va_list args)
 /* Call top of warning stack to issue warning. */
 {
 fputs("cacheLog: ", stderr);
 vfprintf(stderr, format, args);
 fprintf(stderr, "\n");
 fflush(stderr);
 }
 
@@ -151,49 +152,87 @@
 return newList;
 }
 
 static char *cacheDirForDbAndTable(char *string, char *tdbPathString)
 /* Determine the directory name for the trackDb cache files */
 {
 char dirName[4096];
 if (tdbPathString == NULL)
     safef(dirName, sizeof dirName, "%s/%s", trackDbCacheDir, string);
 else
     safef(dirName, sizeof dirName, "%s/%s.%s", trackDbCacheDir, string, tdbPathString);
 
 return cloneString(dirName);
 }
 
+static time_t checkIncFiles(char *dirName, time_t hubTime)
+/* Check the incFiles.txt file (if present) to see if it has files with newer dates than in time. */
+{
+char fileName[4096];
+safef(fileName, sizeof fileName, "%s/%s", dirName, "incFiles.txt");
+struct lineFile *lf = lineFileMayOpen(fileName, TRUE);
+
+if (lf == NULL)
+    return hubTime;
+
+char *line;
+while (lineFileNextReal(lf, &line))
+    {
+    struct udcFile *checkCache = udcFileMayOpen(line, NULL);
+    if (checkCache == NULL)  // if the file disappeared, let's expire the cache!
+        {
+        hubTime = time(NULL);
+        break;
+        }
+
+    time_t incTime = udcUpdateTime(checkCache);
+    if (incTime > hubTime)
+        hubTime = incTime;
+    udcFileClose(&checkCache);
+    }
+
+lineFileClose(&lf);
+
+return hubTime;
+}
+
 static struct trackDb *checkCache(char *string, char *tdbPathString, time_t time)
 /* Check to see if this db or hub has a cached trackDb. string is either a db 
  * or a SHA1 calculated from a hubUrl. Use time to see if cache should be flushed. */
 {
 char *dirName = cacheDirForDbAndTable(string, tdbPathString);
 if (!isDirectory(dirName))
     {
     cacheLog("abandoning cache search for %s, no directory", string);
     return NULL;
     }
 
+// check the incFiles file and see if any of the included files are newer 
+// than the time we've been given
+time = checkIncFiles(dirName, time);
+
 // look for files named by the address they use
 struct slName *files = listDir(dirName, "*");
 char fileName[4096];
 for(; files; files = files->next)
     {
     if (sameString(files->name, "name.txt"))
         continue;
 
+    if (sameString(files->name, "incFiles.txt"))
+        continue;
+
     safef(fileName, sizeof fileName, "%s/%s", dirName, files->name);
     cacheLog("checking cache file %s", fileName);
     
     struct stat statBuf;
     if (stat(fileName, &statBuf) < 0)
         {
         // if we can't stat the shared memory, let's just toss it
         cacheLog("can't stat file %s, unlinking", fileName);
         mustRemove(fileName);
         continue;
         }
 
     if (statBuf.st_mtime < time)
         {
         // if the cache is older than the data, toss it
@@ -259,32 +298,32 @@
 return checkCache(db, tdbPathString, time);
 }
 
 struct trackDb *trackDbHubCache(char *trackDbUrl, time_t time)
 {
 cacheLog("checking for cache for hub %s at time  %ld", trackDbUrl, time);
 unsigned char hash[SHA_DIGEST_LENGTH];
 SHA1((const unsigned char *)trackDbUrl, strlen(trackDbUrl), hash);
 
 char newName[(SHA_DIGEST_LENGTH + 1) * 2];
 hexBinaryString(hash,  SHA_DIGEST_LENGTH, newName, (SHA_DIGEST_LENGTH + 1) * 2);
 
 return checkCache(newName, NULL, time);
 }
 
-static void cloneTdbListToSharedMem(char *string, char *tdbPathString, struct trackDb *list, unsigned long size, char *name)
-/* Allocate shared memory and clone trackDb list into it. */
+static void cloneTdbListToSharedMem(char *string, char *tdbPathString, struct trackDb *list, unsigned long size, char *name, char *incFiles)
+/* Allocate shared memory and clone trackDb list into it. Record the names of include files in incFiles if not empty. */
 {
 static int inited = 0;
 
 if (inited == 0)
     {
     srandom(time(NULL));
     inited = 1;
     }
     
 int oflags=O_RDWR | O_CREAT;
 
 char *dirName = cacheDirForDbAndTable(string, tdbPathString);
 
 if (!isDirectory(dirName))
     {
@@ -357,52 +396,61 @@
 
 char fileName[4096];
 safef(fileName, sizeof fileName, "%s/%ld.%d", dirName, paddress, TRACKDB_VERSION);
 
 cacheLog("renaming %s to %s", tempFileName, fileName);
 mustRename(tempFileName, fileName);
 
 // write out the name of the trackDb being cached.
 safef(fileName, sizeof fileName, "%s/name.txt", dirName);
 FILE *stream = mustOpen(fileName, "w");
 if (tdbPathString == NULL)
     fprintf(stream, "%s\n", name);
 else
     fprintf(stream, "%s.%s\n", name,tdbPathString);
 carefulClose(&stream);
+
+// write out included files if present
+if (!isEmpty(incFiles))
+    {
+    safef(fileName, sizeof fileName, "%s/incFiles.txt", dirName);
+    stream = mustOpen(fileName, "w");
+    fputs(incFiles, stream);
+    carefulClose(&stream);
+    }
 }
 
 void trackDbCloneTdbListToSharedMem(char *db, char *tdbPathString, struct trackDb *list, unsigned long size)
 /* For this native db, allocate shared memory and clone trackDb list into it. */
 {
 cacheLog("cloning memory for db %s %ld", db, size);
-cloneTdbListToSharedMem(db, tdbPathString, list, size, db);
+cloneTdbListToSharedMem(db, tdbPathString, list, size, db, NULL);
 }
 
-void trackDbHubCloneTdbListToSharedMem(char *trackDbUrl, struct trackDb *list, unsigned long size)
-/* For this hub, Allocate shared memory and clone trackDb list into it. */
+void trackDbHubCloneTdbListToSharedMem(char *trackDbUrl, struct trackDb *list, unsigned long size, char *incFiles)
+/* For this hub, Allocate shared memory and clone trackDb list into it. incFiles has a list of include files that may be null. */
 {
 if ((*trackDbUrl == '.') || (list == NULL)) // don't cache empty lists or collections
     return;
 cacheLog("cloning memory for hub %s %ld", trackDbUrl, size);
 unsigned char hash[SHA_DIGEST_LENGTH];
 SHA1((const unsigned char *)trackDbUrl, strlen(trackDbUrl), hash);
 
 char newName[(SHA_DIGEST_LENGTH + 1) * 2];
 hexBinaryString(hash,  SHA_DIGEST_LENGTH, newName, (SHA_DIGEST_LENGTH + 1) * 2);
 
-cloneTdbListToSharedMem(newName, NULL, list, size, trackDbUrl);
+cloneTdbListToSharedMem(newName, NULL, list, size, trackDbUrl, incFiles);
 }
 
 boolean trackDbCacheOn()
 /* Check to see if we're caching trackDb contents. */
 {
 static boolean checkedCache = FALSE;
 static boolean doCache = FALSE;
 
 if (!checkedCache)
     {
     trackDbCacheDir = cfgOption("cacheTrackDbDir");
     if (isNotEmpty(trackDbCacheDir))
         {
         makeDirsOnPath(trackDbCacheDir);
         chmod(trackDbCacheDir, 0777);