bf0f2a64192b5c80db6ff314a3fa66266976ea8f
braney
  Tue Jun 11 12:57:07 2019 -0700
ongoing work on trackDb caching

diff --git src/hg/lib/trackDbCustom.c src/hg/lib/trackDbCustom.c
index 03574c5..f109c2e 100644
--- src/hg/lib/trackDbCustom.c
+++ src/hg/lib/trackDbCustom.c
@@ -10,30 +10,32 @@
 #include "linefile.h"
 #include "jksql.h"
 #include "trackDb.h"
 #include "hdb.h"
 #include "hui.h"
 #include "ra.h"
 #include "hash.h"
 #include "net.h"
 #include "sqlNum.h"
 #include "obscure.h"
 #include "hgMaf.h"
 #include "customTrack.h"
 #include "regexHelper.h"
 #include "fieldedTable.h"
 #include "tagRepo.h"
+#include "hex.h"
+#include <openssl/sha.h>
 
 
 /* ----------- End of AutoSQL generated code --------------------- */
 
 struct trackDb *trackDbNew()
 /* Allocate a new trackDb with just very minimal stuff filled in. */
 {
 struct trackDb *tdb;
 AllocVar(tdb);
 tdb->canPack = 2;	/* Unknown value. */
 return tdb;
 }
 
 int trackDbCmp(const void *va, const void *vb)
 /* Compare to sort based on priority; use shortLabel as secondary sort key.
@@ -1572,34 +1574,32 @@
 return NULL;
 }
 
 struct trackDb *lmCloneSuper(struct lm *lm, struct trackDb *tdb, struct hash *superHash)
 /* clone a super track tdb structure. */
 {
 if (superHash == NULL)
     errAbort("parsing supertrack without superHash");
 
 struct trackDb *super = (struct trackDb *)hashFindVal(superHash, tdb->parent->track);
 
 if (super == NULL)
     {
     super = lmCloneTdb(lm, tdb->parent, NULL, NULL);
     hashAdd(superHash, super->track, super);
-
-    tdb->parent->shortLabel = NULL;
     }
-refAdd(&super->children, tdb);
+lmRefAdd(lm, &super->children, tdb);
 
 return super;
 }
 
 struct trackDb *lmCloneTdb(struct lm *lm, struct trackDb *tdb, struct trackDb *parent,  struct hash *superHash)
 /* clone a single tdb structure.  Will clone its children if it has any */
 {
 struct trackDb *newTdb = lmAlloc(lm, sizeof(struct trackDb));
 
 *newTdb = *tdb;
 
 if (tdb->subtracks)
     newTdb->subtracks = lmCloneTdbList(lm, tdb->subtracks, newTdb, NULL);
 
 if ((tdb->parent != NULL) && (superHash != NULL))
@@ -1612,145 +1612,205 @@
 newTdb->track = lmCloneString(lm, tdb->track);
 newTdb->table = lmCloneString(lm, tdb->table);
 newTdb->shortLabel = lmCloneString(lm, tdb->shortLabel);
 newTdb->longLabel = lmCloneString(lm, "trere");// tdb->longLabel);
 newTdb->type = lmCloneString(lm, tdb->type);
 if ( newTdb->restrictCount )
     {
     lmAllocArray(lm, newTdb->restrictList, newTdb->restrictCount);
     int ii;
     for(ii=0; ii < newTdb->restrictCount; ii++)
         newTdb->restrictList[ii] = lmCloneString(lm, tdb->restrictList[ii]);
     }
 newTdb->url = lmCloneString(lm, tdb->url);
 newTdb->html = lmCloneString(lm, tdb->html);
 newTdb->grp = lmCloneString(lm, tdb->grp);
-newTdb->settings = lmCloneString(lm, tdb->settings);
 newTdb->parentName = lmCloneString(lm, tdb->parentName);
 
 newTdb->viewHash =  NULL;
 newTdb->children = NULL;
 newTdb->overrides = NULL;
-
-// TODO: these two need a fixin
-//tdbExtrasMembership(newTdb);
-//newTdb->settingsHash = trackDbSettingsFromString(newTdb, newTdb->settings);
+newTdb->tdbExtras = NULL;
+
+struct dyString *dy = newDyString(1000);
+struct hashEl *hel = hashElListHash(tdb->settingsHash);
+dyStringPrintf(dy,"%s\n", tdb->settings);
+for(; hel; hel = hel->next)
+    dyStringPrintf(dy,"%s %s\n", hel->name, (char *)hel->val);
+newTdb->settings = lmCloneString(lm, dy->string);
 newTdb->settingsHash = NULL;
 return newTdb;
 }
 
 struct trackDb *lmCloneTdbList(struct lm *lm, struct trackDb *list, struct trackDb *parent, struct hash *superHash)
 /* clone a list of tdb structures. */
 {
 struct trackDb *tdb, *prevTdb = NULL, *newList = NULL;
 
 for(tdb = list; tdb; tdb = tdb->next)
     {
     struct trackDb *newTdb = lmCloneTdb(lm, tdb, parent, superHash); 
 
     if (prevTdb)
         prevTdb->next = newTdb;
     else
         newList = newTdb;
 
     prevTdb = newTdb;
     }
 
 return newList;
 }
 
-struct trackDb *trackDbCache(char *db)
-/* Check to see if this db has a cached trackDb. */
+
+static struct trackDb *checkCache(char *string)
+/* Check to see if this db or hub has a cached trackDb. string is either a db 
+ * or a SHA1 calculated from a hubUrl. */
 {
 char dirName[4096];
 
-safef(dirName, sizeof dirName, "/dev/shm/trackDbCache/%s", db);
+safef(dirName, sizeof dirName, "/dev/shm/trackDbCache/%s", string);
 if (!isDirectory(dirName))
     return NULL;
 
 // look for files named by the address they use
 struct slName *files = listDir(dirName, "*");
 for(; files; files = files->next)
     {
     char sharedMemoryName[4096];
-    safef(sharedMemoryName, sizeof sharedMemoryName, "trackDbCache/%s/%s", db, files->name);
+    safef(sharedMemoryName, sizeof sharedMemoryName, "trackDbCache/%s/%s", string, files->name);
     
     int oflags = O_RDWR;
     int fd = shm_open(sharedMemoryName, oflags, 0666 );
     if (fd < 0)
         continue;
 
     unsigned long address = atoi(files->name); // the name of the file is the address it uses
     char fileName[4096];
-    safef(fileName, sizeof fileName, "/dev/shm/trackDbCache/%s/%s",  db, files->name);
+    safef(fileName, sizeof fileName, "/dev/shm/trackDbCache/%s/%s",  string, files->name);
     unsigned long size = fileSize(fileName);
 
     u_char *mem = (u_char *) mmap((void *)address, size, PROT_READ|PROT_WRITE, MAP_PRIVATE, fd, 0);
 
     if ((unsigned long)mem == address)  // make sure we can get this address
         {
         u_char *ret = mem + lmBlockHeaderSize();
         return (struct trackDb *)ret;
         }
     munmap((void *)address, size);
     close(fd);
     }
 
 return NULL;
 }
 
-void trackDbCloneTdbListToSharedMem(char *db, struct trackDb *list, unsigned long size)
+struct trackDb *trackDbCache(char *db)
+/* Check to see if this db has a cached trackDb. */
+{
+return checkCache(db);
+}
+
+struct trackDb *trackDbHubCache(char *hubUrl, char *genome)
+{
+char buffer[10 * 1024];
+
+safef(buffer, sizeof buffer, "%s-%s", hubUrl, genome);
+
+unsigned char hash[SHA_DIGEST_LENGTH];
+SHA1((const unsigned char *)buffer, strlen(buffer), hash);
+
+char newName[(SHA_DIGEST_LENGTH + 1) * 2];
+hexBinaryString(hash,  SHA_DIGEST_LENGTH, newName, (SHA_DIGEST_LENGTH + 1) * 2);
+
+return checkCache(newName);
+}
+
+static void cloneTdbListToSharedMem(char *string, struct trackDb *list, unsigned long size)
 /* Allocate shared memory and clone trackDb list into it. */
 {
 int oflags=O_RDWR | O_CREAT;
 // should use a unique name
 char dirName[4096];
 
-safef(dirName, sizeof dirName, "/dev/shm/trackDbCache/%s", db);
+safef(dirName, sizeof dirName, "/dev/shm/trackDbCache/%s", string);
 if (!isDirectory(dirName))
     {
     makeDir(dirName);
     chmod(dirName, 0777);
     }
 
 char sharedMemoryName[4096];
-safef(sharedMemoryName, sizeof sharedMemoryName, "trackDbCache/%s",  rTempName(db, "temp", ""));
+safef(sharedMemoryName, sizeof sharedMemoryName, "trackDbCache/%s",  rTempName(string, "temp", ""));
 
 char tempFileName[4096];
 safef(tempFileName, sizeof tempFileName, "/dev/shm/%s",  sharedMemoryName);
 
 int fd = shm_open(sharedMemoryName, oflags, 0666 );
 if (fd < 0)
     {
     unlink(tempFileName);
     return;
     }
+ftruncate(fd, 0);
+ftruncate(fd, size);
+
 size_t psize = getpagesize();
 unsigned long pageMask = psize - 1;
-// we should choose an address semi-randomly and make sure we can grab it rather than assume we can
-unsigned long address = 0x7000000;
-address = (address + psize - 1) & ~pageMask;
-u_char *mem;
 
-ftruncate(fd, 0);
-ftruncate(fd, size);
+// we should choose an address semi-randomly and make sure we can grab it rather than assume we can
+unsigned long address;
+unsigned long paddress;
+unsigned char *mem;
+for(address = 0x7000000; address < 0xf000000; address += 0x500000)
+    {
+    paddress = (address + psize - 1) & ~pageMask;
 
     mem = (u_char *) mmap((void *)address, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+    if ((unsigned long)mem == address)
+        break;
+    }
+
+if (address >= 0xf000000)
+    errAbort("can't get address");
 
 struct lm *lm = lmInitWMem(mem, size);
 struct hash *superHash = newHash(8);
 
 lmCloneTdbList(lm, list, NULL, superHash);
 
 unsigned long memUsed = lmUsed(lm);
 
-msync((void *)address, memUsed, MS_SYNC);
+msync((void *)paddress, memUsed, MS_SYNC);
 ftruncate(fd, memUsed);
 
-munmap((void *)address, size);
-close(fd);
+// for the moment we're not unmapping these so multiple attached hubs will get
+// different addresses
+//munmap((void *)paddress, size);
+//close(fd);
 
 char fileName[4096];
-safef(fileName, sizeof fileName, "/dev/shm/trackDbCache/%s/%ld",  db, address);
+safef(fileName, sizeof fileName, "/dev/shm/trackDbCache/%s/%ld",  string, paddress);
 
 mustRename(tempFileName, fileName);
 }
+
+void trackDbCloneTdbListToSharedMem(char *db, struct trackDb *list, unsigned long size)
+/* Allocate shared memory and clone trackDb list into it. */
+{
+cloneTdbListToSharedMem(db, list, size);
+}
+
+void trackDbHubCloneTdbListToSharedMem(char *hubUrl, char *genome, struct trackDb *list, unsigned long size)
+{
+char buffer[10 * 1024];
+
+safef(buffer, sizeof buffer, "%s-%s", hubUrl, genome);
+
+unsigned char hash[SHA_DIGEST_LENGTH];
+SHA1((const unsigned char *)buffer, strlen(buffer), hash);
+
+char newName[(SHA_DIGEST_LENGTH + 1) * 2];
+hexBinaryString(hash,  SHA_DIGEST_LENGTH, newName, (SHA_DIGEST_LENGTH + 1) * 2);
+
+cloneTdbListToSharedMem(newName, list, size);
+}
+