d2a2afcd6c5c19257f8ed2f746c3f94474f4ab78 braney Tue Sep 1 13:37:34 2026 -0700 hgTracks: let hg.conf set how big a step malloc takes, refs #38225 hgTracks loads tracks in parallel threads, and each thread grows its own memory pool in the 128 kB steps glibc uses by default. A heavy render asks the kernel to enlarge a pool about 133,000 times, work that does nothing for the reader. The new hg.conf setting mallocTopPad is a number of bytes. When it is set, cfgSetMallocTopPad() passes it to mallopt(M_TOP_PAD) before anything else in main() has a chance to allocate, so the heap grows in that size step instead. Absent or zero, glibc is left alone and nothing changes. Measured with the #38094 harness over eight sessions and two positions, at 32 MB: renders take 0.926 of the time they did, against a noise floor of 1.006. With the setting absent the same build measures 1.004, inside that noise floor. Through apache on a sandbox it measures 0.906. The cost is about 65 MB more resident memory on a heavy session and 8 to 20 MB on an ordinary view, where it also saves no measurable time; the saving and the cost are the same effect and appear together. The rendered image is byte identical either way. Off by default, so a server opts in. diff --git src/hg/hgTracks/renderMain.c src/hg/hgTracks/renderMain.c index 71ef4e7b212..362089b3054 100644 --- src/hg/hgTracks/renderMain.c +++ src/hg/hgTracks/renderMain.c @@ -1,190 +1,195 @@ /* renderMain - execute enough of browser to render list of tracks without touching cart or * writing form. . */ /* Copyright (C) 2013 The Regents of the University of California * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */ #include "common.h" #include "linefile.h" #include "hash.h" #include "htmshell.h" #include "options.h" #include "errAbort.h" #include "portable.h" #include "cheapcgi.h" #include "ra.h" #include "hdb.h" #include "net.h" #include "hgTracks.h" #include "imageV2.h" #include "botDelay.h" +#include "hgConfig.h" static void usage() /* Print out usage and exit - just temporary. */ { errAbort( "hgRenderTracks - execute enough of browser to render a list of tracks\n" "usage:\n" " hgTrackRender track.ra cart.ra output.ps\n" "where track.ra is a trackDb.ra entry flattened out to include stuff inherited from parent\n" "and cart.ra is a file with the cart settings. Both .ra files should just have a single stanza\n" ); } static struct optionSpec options[] = { {NULL, 0}, }; long enteredMainTime = 0; struct trackDb *hTrackDbForTrackAndAncestors(char *db, char *track); /* Load trackDb object for a track. If need be grab its ancestors too. * This does not load children. hTrackDbForTrack will handle children, and * is actually faster if being called on lots of tracks. This function * though is faster on one or two tracks. */ static void hashIntoHash(struct hash *newStuff, struct hash *hash) /* Add newStuff into hash. */ { struct hashCookie cookie = hashFirst(newStuff); struct hashEl *hel; while ((hel = hashNext(&cookie)) != NULL) hashAdd(hash, hel->name, hel->val); } struct track *trackFromSettingsHash(struct hash *settings) /* Wrap a trackDb, and then a track around settings, and return track. */ { char *trackName = hashMustFindVal(settings, "track"); struct trackDb *tdb = hTrackDbForTrackAndAncestors(database, trackName); hashIntoHash(settings, tdb->settingsHash); trackDbFieldsFromSettings(tdb); trackDbAddTableField(tdb); return trackFromTrackDb(tdb); } static void hgTrackRenderFromCommandLine(char *trackFile, char *cartFile, char *outFile) /* Generate tracks from trackFile and cart from cartFile, both in .ra format. Then * call makeActiveImage, the heart of the genome browser.*/ { struct track *track, *trackList = NULL; /* Load in cart file into a single hash and then wrap a cart around it. * We need to set cart before we can make tracks. */ struct hash *cartHash = raReadSingle(cartFile); cart = cartFromHash(cartHash); database = hashMustFindVal(cartHash, "db"); position = cloneString(hashMustFindVal(cartHash, "position")); if (!hgParseChromRange(NULL, position, &chromName, &winStart, &winEnd)) errAbort("position not in chrom:start-end format"); /* Initialize layout. */ initTl(); setLayoutGlobals(); /* Make list of tracks out of track file. */ struct lineFile *lf = lineFileOpen(trackFile, TRUE); struct hash *trackRa; while ((trackRa = raNextRecord(lf)) != NULL) { track = trackFromSettingsHash(trackRa); slAddHead(&trackList, track); } slReverse(&trackList); lineFileClose(&lf); verboseTime(2, "Before load %d tracks", slCount(trackList)); /* Prepare track list for drawing. */ for (track = trackList; track != NULL; track = track->next) { track->loadItems(track); } verboseTime(2, "After load"); /* Initialize global image box. */ int sideSliceWidth = 0; // Just being explicit if (withLeftLabels) sideSliceWidth = (insideX - gfxBorder*3) + 2; theImgBox = imgBoxStart(database,chromName,winStart,winEnd,(!revCmplDisp),sideSliceWidth,tl.picWidth); makeActiveImage(trackList, NULL); verboseTime(2,"After makeActiveImage"); } boolean issueBotWarning; int main(int argc, char *argv[]) { +// First, before anything has a chance to allocate: read hg.conf and set how +// big a step the C library takes when it grows the heap. refs #38225 +cfgSetMallocTopPad(); + cgiSpoof(&argc, argv); if(argc == 1) { enteredMainTime = clock1000(); issueBotWarning = earlyBotCheck(enteredMainTime, "hgRenderTracks", delayFraction, 0, 0, "html"); // CGI call // htmlPushEarlyHandlers(); XXXX do I need to do this? hPrintDisable(); oldVars = hashNew(10); struct cart *cart = cartForSession(hUserCookie(), excludeVars, oldVars); // setup approriate CGI variables which tell hgTracks code what to do. cartSetBoolean(cart, "hgt.trackImgOnly", TRUE); if(cartVarExists(cart, "jsonp")) { // experimental code to support remote rendering via a jsonp callback // e.g.: /cgi-bin/hgRenderTracks?track=bamMMS9MbutiPygmy&track=...&jsonp=remoteTrackCallback&postion=...&pix=800 cartSetString(cart, "hgt.contentType", "jsonp"); cartSetString(cart, "hgt.trackNameFilter", cartString(cart, "track")); cartSetString(cart, cartString(cart, "track"), cartUsualString(cart, "vis", "pack")); } else { // remote rendering of hgTracks PNG image based on contents of a session; caller may pass in a subset of // hgTracks parameters: e.g. db, hgsid, pix, position and tracks with explicit visibilities (e.g. knownGene=pack). // hgt.psOutput=on asks for PDF instead of PNG. Same variable the hgTracks // View->PDF menu item uses, so callers do not have to learn a second name. if (cartUsualBoolean(cart, "hgt.psOutput", FALSE)) cartSetString(cart, "hgt.contentType", "pdf"); else cartSetString(cart, "hgt.contentType", "png"); cartSetBoolean(cart, "hgt.imageV1", TRUE); if(!cartVarExists(cart, "hgt.internal")) { if(!cartVarExists(cart, "hgt.baseShowAsm")) cartSetBoolean(cart, "hgt.baseShowAsm", TRUE); if(!cartVarExists(cart, "hgt.baseShowPos")) cartSetBoolean(cart, "hgt.baseShowPos", TRUE); } // XXXX support track filtering? - if(cartVarExists(cart, "hgt.trackNameFilter")) } doMiddle(cart); cgiExitTime("hgRenderTracks", enteredMainTime); } else { // XXXX remove this code ... well, maybe not - this still might be useful for a stand-alone remote renderer. // command line call /* Set up some timing since we're trying to optimize things very often. */ long enteredMainTime = clock1000(); verboseTimeInit(); /* Push very early error handling - this is just * for the benefit of the cgiVarExists, which * somehow can't be moved effectively into doMiddle. */ // htmlPushEarlyHandlers(); /* Set up cgi vars from command line. */ // cgiSpoof(&argc, argv); optionInit(&argc, argv, options); if (argc != 4) usage(); hgTrackRenderFromCommandLine(argv[1], argv[2], argv[3]); verbose(2, "Overall total time: %ld millis
\n", clock1000() - enteredMainTime); } return 0; }