a83d09e8ccd1c4c9c97b251d4117014010588ac8
galt
  Fri Feb 21 12:56:26 2014 -0800
Fixes ##12648. This fixes a problem caused by a new glibc malloc policy in which a new heap arena of 64MB is created per thread instead of per cpu.  A track hub with many visible will use up to 100 threads, CGI can waste 15GB of RAM and rlimits on RR cause out-of-memory failures. The fix is to explicitly set the maximum number of arenas to 8 using a new malloc option before launching threads.
diff --git src/hg/hgTracks/hgTracks.c src/hg/hgTracks/hgTracks.c
index 5eba5f7..e1020b5 100644
--- src/hg/hgTracks/hgTracks.c
+++ src/hg/hgTracks/hgTracks.c
@@ -1,22 +1,23 @@
 /* hgTracks - the original, and still the largest module for the UCSC Human Genome
  * Browser main cgi script.  Currently contains most of the track framework, though
  * there's quite a bit of other framework type code in simpleTracks.c.  The main
  * routine got moved to create a new entry point to the bulk of the code for the
  * hgRenderTracks web service.  See mainMain.c for the main used by the hgTracks CGI. */
 
 #include <pthread.h>
+#include <malloc.h>
 #include "common.h"
 #include "hCommon.h"
 #include "linefile.h"
 #include "portable.h"
 #include "memalloc.h"
 #include "localmem.h"
 #include "obscure.h"
 #include "dystring.h"
 #include "hash.h"
 #include "jksql.h"
 #include "gfxPoly.h"
 #include "memgfx.h"
 #include "hvGfx.h"
 #include "psGfx.h"
 #include "cheapcgi.h"
@@ -4400,79 +4401,102 @@
 /* adjust visibility */
 for (track = trackList; track != NULL; track = track->next)
     {
     /* adjust track visibility based on supertrack just before load loop */
     if (tdbIsSuperTrackChild(track->tdb))
         limitSuperTrackVis(track);
 
     /* remove cart priority variables if they are set
        to the default values in the trackDb */
     if (!hTrackOnChrom(track->tdb, chromName))
         {
         track->limitedVis = tvHide;
         track->limitedVisSet = TRUE;
 	}
     }
+
 /* pre-load remote tracks in parallel */
 int ptMax = atoi(cfgOptionDefault("parallelFetch.threads", "20"));  // default number of threads for parallel fetch.
+int pfdListCount = 0;
 pthread_t *threads = NULL;
+pthread_attr_t attr;
+int stackSize = 2*1024*1024;  // 2 MB per thread? 10MB is default on 64-bit
 if (ptMax > 0)     // parallelFetch.threads=0 to disable parallel fetch
     {
     findLeavesForParallelLoad(trackList, &pfdList);
+    pfdListCount = slCount(pfdList);
     /* launch parallel threads */
-    ptMax = min(ptMax, slCount(pfdList));
+    ptMax = min(ptMax, pfdListCount);
     if (ptMax > 0)
 	{
 	AllocArray(threads, ptMax);
+
+	#if defined(M_ARENA_MAX) 
+	    mallopt(M_ARENA_MAX, 8);   // Otherwise new glibc allocates 64MB arena per thread
+	#endif /* glibc malloc tuning */ 
+
+	/* Initialize thread creation attributes */
+	int rc = pthread_attr_init(&attr);
+	if (rc) errAbort("Unexpected error %d from pthread_attr_init(): %s",rc,strerror(rc));
+	/* Set thread stack size */
+	rc = pthread_attr_setstacksize(&attr, stackSize);
+	if (rc) errAbort("Unexpected error %d from pthread_attr_setstacksize(): %s",rc,strerror(rc));
+	/* Create threads */
 	int pt;
 	for (pt = 0; pt < ptMax; ++pt)
 	    {
-	    int rc = pthread_create(&threads[pt], NULL, remoteParallelLoad, &threads[pt]);
+	    rc = pthread_create(&threads[pt], &attr, remoteParallelLoad, &threads[pt]);
 	    if (rc)
 		{
 		errAbort("Unexpected error %d from pthread_create(): %s",rc,strerror(rc));
 		}
 	    }
+	/* Thread attr no longer needed */
+	rc = pthread_attr_destroy(&attr);
+	if (rc) errAbort("Unexpected error %d from pthread_attr_destroy(): %s",rc,strerror(rc));
+
 	}
     }
+
 /* load regular tracks */
 for (track = trackList; track != NULL; track = track->next)
     {
     if (track->visibility != tvHide)
 	{
 	if (!track->parallelLoading)
 	    {
 	    if (measureTiming)
 		lastTime = clock1000();
 
 	    checkMaxWindowToDraw(track);
 	    track->loadItems(track);
 
 	    if (measureTiming)
 		{
 		thisTime = clock1000();
 		track->loadTime = thisTime - lastTime;
 		}
 	    }
 	}
     }
+
 if (ptMax > 0)
     {
     /* wait for remote parallel load to finish */
     remoteParallelLoadWait(atoi(cfgOptionDefault("parallelFetch.timeout", "90")));  // wait up to default 90 seconds.
     if (measureTiming)
-	measureTime("Waiting for parallel (%d thread) remote data fetch", ptMax);
+	measureTime("Waiting for parallel (%d threads for %d tracks) remote data fetch", ptMax, pfdListCount);
     }
 
 printTrackInitJavascript(trackList);
 
 /* Generate two lists of hidden variables for track group visibility.  Kludgy,
    but required b/c we have two different navigation forms on this page, but
    we want open/close changes in the bottom form to be submitted even if the user
    submits via the top form. */
 struct dyString *trackGroupsHidden1 = newDyString(1000);
 struct dyString *trackGroupsHidden2 = newDyString(1000);
 for (group = groupList; group != NULL; group = group->next)
     {
     if (group->trackList != NULL)
         {
         int looper;