fa7a11afb5f19dd5a2fa3382fc38662f34b6f147
braney
  Wed May 29 13:55:44 2019 -0700
first cut at caching trackDb in a shared memory location.   No changes
in running code if "cacheTrackDb" hg.conf variable doesn't exist

diff --git src/hg/lib/trackDbCustom.c src/hg/lib/trackDbCustom.c
index dedd2a6..062dd64 100644
--- src/hg/lib/trackDbCustom.c
+++ src/hg/lib/trackDbCustom.c
@@ -1,22 +1,23 @@
 /* trackDbCustom - custom (not autoSQL generated) code for working
  * with trackDb.  This code is concerned with making the trackDb
  * MySQL table out of the trackDb.ra files. */
 
 /* Copyright (C) 2014 The Regents of the University of California 
  * See README in this or parent directory for licensing information. */
 
+#include <sys/mman.h>
 #include "common.h"
 #include "linefile.h"
 #include "jksql.h"
 #include "trackDb.h"
 #include "hdb.h"
 #include "hui.h"
 #include "ra.h"
 #include "hash.h"
 #include "net.h"
 #include "sqlNum.h"
 #include "obscure.h"
 #include "hgMaf.h"
 #include "customTrack.h"
 #include "regexHelper.h"
 #include "fieldedTable.h"
@@ -1550,15 +1551,144 @@
     char *tabSepMeta = trackDbSetting(tdb, "metaTab");
     if (tabSepMeta)
         return tabSepMetaPairs(tabSepMeta, tdb, metaTag);
 
     char *tagStormFile = trackDbSetting(tdb, "metaDb");
     if (tagStormFile)
         return tagRepoPairs(tagStormFile, "meta", metaTag);
     }
 
 char *metadataInTdb = trackDbSetting(tdb, "metadata");
 if (metadataInTdb)
     return convertNameValueString(metadataInTdb);
 
 return NULL;
 }
+
+struct trackDb *lmCloneSuper(struct lm *lm, struct trackDb *tdb, struct hash *superHash)
+/* clone a super track tdb structure. */
+{
+if (superHash == NULL)
+    errAbort("parsing supertrack without superHash");
+
+struct trackDb *super = (struct trackDb *)hashFindVal(superHash, tdb->parent->track);
+
+if (super == NULL)
+    {
+    super = lmCloneTdb(lm, tdb->parent, NULL, NULL);
+    hashAdd(superHash, super->track, super);
+
+    tdb->parent->shortLabel = NULL;
+    }
+refAdd(&super->children, tdb);
+
+return super;
+}
+
+struct trackDb *lmCloneTdb(struct lm *lm, struct trackDb *tdb, struct trackDb *parent,  struct hash *superHash)
+/* clone a single tdb structure.  Will clone its children if it has any */
+{
+struct trackDb *newTdb = lmAlloc(lm, sizeof(struct trackDb));
+
+*newTdb = *tdb;
+
+if (tdb->subtracks)
+    newTdb->subtracks = lmCloneTdbList(lm, tdb->subtracks, newTdb, NULL);
+
+if ((tdb->parent != NULL) && (superHash != NULL))
+    {
+    newTdb->parent = lmCloneSuper(lm, newTdb, superHash);
+    }
+else
+    newTdb->parent = parent;
+
+newTdb->track = lmCloneString(lm, tdb->track);
+newTdb->table = lmCloneString(lm, tdb->table);
+newTdb->shortLabel = lmCloneString(lm, tdb->shortLabel);
+newTdb->longLabel = lmCloneString(lm, "trere");// tdb->longLabel);
+newTdb->type = lmCloneString(lm, tdb->type);
+if ( newTdb->restrictCount )
+    {
+    lmAllocArray(lm, newTdb->restrictList, newTdb->restrictCount);
+    int ii;
+    for(ii=0; ii < newTdb->restrictCount; ii++)
+        newTdb->restrictList[ii] = lmCloneString(lm, tdb->restrictList[ii]);
+    }
+newTdb->url = lmCloneString(lm, tdb->url);
+newTdb->html = lmCloneString(lm, tdb->html);
+newTdb->grp = lmCloneString(lm, tdb->grp);
+newTdb->settings = lmCloneString(lm, tdb->settings);
+newTdb->parentName = lmCloneString(lm, tdb->parentName);
+
+newTdb->viewHash =  NULL;
+newTdb->children = NULL;
+newTdb->overrides = NULL;
+
+// TODO: these two need a fixin
+//tdbExtrasMembership(newTdb);
+//newTdb->settingsHash = trackDbSettingsFromString(newTdb, newTdb->settings);
+newTdb->settingsHash = NULL;
+return newTdb;
+}
+
+struct trackDb *lmCloneTdbList(struct lm *lm, struct trackDb *list, struct trackDb *parent, struct hash *superHash)
+/* clone a list of tdb structures. */
+{
+struct trackDb *tdb, *prevTdb = NULL, *newList = NULL;
+
+for(tdb = list; tdb; tdb = tdb->next)
+    {
+    struct trackDb *newTdb = lmCloneTdb(lm, tdb, parent, superHash); 
+
+    if (prevTdb)
+        prevTdb->next = newTdb;
+    else
+        newList = newTdb;
+
+    prevTdb = newTdb;
+    }
+
+return newList;
+}
+
+struct trackDb *mapSharedMemTrackDb(char *file, unsigned long address, unsigned long size)
+/* Use a hunk of shared memory as our trackDb list. */
+{
+int oflags=O_RDWR | O_CREAT;
+int     fd = shm_open(file, oflags, 0666 );
+u_char *mem = (u_char *) mmap((void *)address, size, PROT_READ|PROT_WRITE, MAP_PRIVATE, fd, 0);
+u_char *ret = mem + 32;  // lm header is 32 bytes long
+
+return (struct trackDb *)ret;
+}
+
+struct trackDb *cloneTdbListToSharedMem(struct trackDb *list, unsigned long size)
+/* Allocate shared memory and clone trackDb list into it. */
+{
+int oflags=O_RDWR | O_CREAT;
+// should use a unique name
+char *file ="brtest/flart";
+int     fd = shm_open(file, oflags, 0666 );
+size_t psize = getpagesize();
+unsigned long pageMask = psize - 1;
+// we should choose an address semi-randomly and make sure we can grab it rather than assume we can
+unsigned long address = 0x7000000;
+address = (address + psize - 1) & ~pageMask;
+u_char *mem;
+struct trackDb *newList = NULL;
+mem = (u_char *)newList;
+
+ftruncate(fd, 0);
+ftruncate(fd, size);
+
+mem = (u_char *) mmap((void *)address, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+
+struct lm *lm = lmInitWMem(mem, size);
+struct hash *superHash = newHash(8);
+
+newList = lmCloneTdbList(lm, list, NULL, superHash);
+msync((void *)address, size, MS_SYNC);
+munmap((void *)address, size);
+close(fd);
+
+return mapSharedMemTrackDb(file, address, size);
+}