faf17a6ad5e1cf7379a375b106f4900990d35c2d kate Tue Apr 23 16:10:51 2019 -0700 Support for summary file to improve performance of collapseEmptySubtracks setting. This implementation is first cut for the feature (w/o UI). refs #23365 diff --git src/hg/hgTracks/hgTracks.c src/hg/hgTracks/hgTracks.c index 9147990..acf4fb8 100644 --- src/hg/hgTracks/hgTracks.c +++ src/hg/hgTracks/hgTracks.c @@ -60,30 +60,31 @@ #include "search.h" #include "errCatch.h" #include "iupac.h" #include "botDelay.h" #include "chromInfo.h" #include "extTools.h" #include "basicBed.h" #include "customFactory.h" #include "genbank.h" #include "bigWarn.h" #include "wigCommon.h" #include "knetUdc.h" #include "hex.h" #include <openssl/sha.h> #include "customComposite.h" +#include "bed3Sources.h" /* Other than submit and Submit all these vars should start with hgt. * to avoid weeding things out of other program's namespaces. * Because the browser is a central program, most of its cart * variables are not hgt. qualified. It's a good idea if other * program's unique variables be qualified with a prefix though. */ char *excludeVars[] = { "submit", "Submit", "dirty", "hgt.reset", "hgt.in1", "hgt.in2", "hgt.in3", "hgt.inBase", "hgt.out1", "hgt.out2", "hgt.out3", "hgt.out4", "hgt.left1", "hgt.left2", "hgt.left3", "hgt.right1", "hgt.right2", "hgt.right3", "hgt.dinkLL", "hgt.dinkLR", "hgt.dinkRL", "hgt.dinkRR", "hgt.tui", "hgt.hideAll", "hgt.visAllFromCt", "hgt.psOutput", "hideControls", "hgt.toggleRevCmplDisp", "hgt.collapseGroups", "hgt.expandGroups", "hgt.suggest", @@ -4584,35 +4585,137 @@ struct window *window; for (window=windows, winTrack=track; window; window=window->next, winTrack=winTrack->nextWindow) { setGlobalsFromWindow(window); int trackHeight = trackPlusLabelHeight(winTrack, fontHeight); if (trackHeight > maxHeight) maxHeight = trackHeight; } setGlobalsFromWindow(windows); // first window flatTrack->maxHeight = maxHeight; } -boolean doCollapseEmptySubtracks(struct track *track) -/* Suppress display of empty subtracks. Initial support only for bed's. */ +char *collapseEmptySubtracksSetting(struct track *track) +/* Setting to suppress display of empty subtracks */ { -char *collapseEmptySubtracks = trackDbSetting(track->tdb, "collapseEmptySubtracks"); -return (collapseEmptySubtracks && sameWord(collapseEmptySubtracks, "on")); +if (!tdbIsComposite(track->tdb)) + return FALSE; +return trackDbSetting(track->tdb, "collapseEmptySubtracks"); +} + +boolean doCollapseEmptySubtracksNoMultiBed(struct track *track) +/* TRUE if collapseEmptySubtracks setting is 'on' (no file specs for multiBed */ +{ +char *setting = collapseEmptySubtracksSetting(track); +if (setting && sameString(setting, "on")) + return TRUE; +return FALSE; +} + +boolean doCollapseEmptySubtracks(struct track *track, char **multiBedFile, char **subtrackIdFile) +{ +/* Support setting to suppress display of empty subtracks. + * (Initial support only for bed's). + * + * Format: collapseEmptySubtracks on + * or + * collapseEmptySubtracks multiBed.bed subtrackIds.tab + * where multiBed.bed is a bed3Sources bigBed, generated with bedtools multiinter + * post-processed by UCSC multiBed.pl tool + * subtrackIds.tab is a tab-sep file: id subtrackName + */ +char *collapse = collapseEmptySubtracksSetting(track); +if (!collapse) + return FALSE; +if (doCollapseEmptySubtracksNoMultiBed(track)) + return TRUE; +char *orig = cloneString(collapse); +char *words[2]; +int wordCount = chopByWhite(collapse, words, ArraySize(words)); +if (wordCount == 2) + { + if (multiBedFile) + *multiBedFile = cloneString(hReplaceGbdb(words[0])); + if (subtrackIdFile) + *subtrackIdFile = cloneString(words[1]); + return TRUE; + } +warn("Track %s collapseEmptySubtracks setting invalid: %s", + track->track, orig); +return FALSE; +} + +struct hash *getNonEmptySubtracks(struct track *track) +{ +/* Support setting to suppress display of empty subtracks. + * (Initial support only for bed's). + * + * Format: collapseEmptySubtracks on + * or + * collapseEmptySubtracks multiBed.bed subtrackIds.tab + * where multiBed.bed is a bed3Sources bigBed, generated with bedtools multiinter + * post-processed by UCSC multiBed.pl tool + * subtrackIds.tab is a tab-sep file: id subtrackName + * + * Return hash with subtrack names as keys + */ + +char *multiBedFile = NULL; +char *subtracksIdFile = NULL; +if (!doCollapseEmptySubtracks(track, &multiBedFile, &subtracksIdFile)) + return NULL; +if (!multiBedFile) + return NULL; + +// load multiBed items in window +// TODO: filters here ? +// TODO: protect against temporary network error ? */ +struct lm *lm = lmInit(0); +struct bbiFile *bbi = bigBedFileOpen(multiBedFile); +struct bigBedInterval *bb, *bbList = bigBedIntervalQuery(bbi, chromName, winStart, winEnd, 0, lm); +char *row[bbi->fieldCount]; +char startBuf[16], endBuf[16]; +struct hash *nonEmptySubtracksHash = hashNew(0); +for (bb = bbList; bb != NULL; bb = bb->next) + { + bigBedIntervalToRow(bb, chromName, startBuf, endBuf, row, ArraySize(row)); + // TODO: do this in bed3Sources.c + char *idList = row[4]; + struct slName *ids = slNameListFromComma(idList); + struct slName *id = NULL; + for (id = ids; id != NULL; id = id->next) + hashStore(nonEmptySubtracksHash, id->name); + // TODO: free some stuff ? + } + +// read file containing ids of subtracks +struct lineFile *lf = udcWrapShortLineFile(subtracksIdFile, NULL, 0); +char *words[2]; +while (lineFileChopNext(lf, words, sizeof words)) + { + char *id = words[0]; + char *name = words[1]; + if (hashLookup(nonEmptySubtracksHash, id)) + { + hashStore(nonEmptySubtracksHash, cloneString(name)); + } + } +lineFileClose(&lf); +return nonEmptySubtracksHash; } void makeActiveImage(struct track *trackList, char *psOutput) /* Make image and image map. */ { struct track *track; MgFont *font = tl.font; struct hvGfx *hvg; struct tempName pngTn; char *mapName = "map"; int fontHeight = mgFontLineHeight(font); int trackPastTabX = (withLeftLabels ? trackTabWidth : 0); int trackTabX = gfxBorder; int trackPastTabWidth = tl.picWidth - trackPastTabX; int pixWidth, pixHeight; @@ -4817,38 +4920,41 @@ safef(buffer, sizeof buffer, "%s_imgOrd", name->name); cartRemove(cart, buffer); } } // Construct flatTracks for (track = trackList; track != NULL; track = track->next) { if (tdbIsComposite(track->tdb)) { struct track *subtrack; if (isCompositeInAggregate(track)) flatTracksAdd(&flatTracks,track,cart, orderedWiggles); else { - boolean doCollapse = doCollapseEmptySubtracks(track); + boolean doCollapse = doCollapseEmptySubtracksNoMultiBed(track); + // If multibed was found, it has been used to suppress loading, + // and subtracks lacking items in window are already set hidden for (subtrack = track->subtracks; subtrack != NULL; subtrack = subtrack->next) { if (!isSubtrackVisible(subtrack)) continue; if (!isLimitedVisHiddenForAllWindows(subtrack) && !(doCollapse && slCount(subtrack->items) == 0)) + // Ignore subtracks with no items in window { flatTracksAdd(&flatTracks,subtrack,cart, orderedWiggles); } } } } else { if (!isLimitedVisHiddenForAllWindows(track)) { flatTracksAdd(&flatTracks,track,cart, orderedWiggles); } } } flatTracksSort(&flatTracks); // Now we should have a perfectly good flat track list! @@ -7138,30 +7244,52 @@ subtrack->drawItems = drawMaxWindowWarning; subtrack->limitedVis = tvDense; subtrack->limitedVisSet = TRUE; } } } else if (maxWinToDraw > 1 && (winEnd - winStart) > maxWinToDraw) { tg->loadItems = dontLoadItems; tg->drawItems = drawMaxWindowWarning; tg->limitedVis = tvDense; tg->limitedVisSet = TRUE; } } +static void checkCollapseEmptySubtracks(struct track *tg) +/* Suppress queries on subtracks w/o data in window (identified from multiIntersect file) */ +{ +if (!tdbIsComposite(tg->tdb)) + return; +struct hash *nonEmptySubtracksHash = getNonEmptySubtracks(tg); +if (!nonEmptySubtracksHash) + return; +struct track *subtrack; +for (subtrack = tg->subtracks; subtrack != NULL; subtrack = subtrack->next) + { + if (!isSubtrackVisible(subtrack)) + continue; + if (!hashLookup(nonEmptySubtracksHash, subtrack->track)) + { + subtrack->loadItems = dontLoadItems; + subtrack->limitedVis = tvHide; + subtrack->limitedVisSet = TRUE; + } + } +} + void printTrackInitJavascript(struct track *trackList) { hPrintf("<input type='hidden' id='%s' name='%s' value=''>\n", hgtJsCommand, hgtJsCommand); } void jsCommandDispatch(char *command, struct track *trackList) /* Dispatch a command sent to us from some javaScript event. * This gets executed after the track list is built, but before * the track->loadItems methods are called. */ { if (startsWithWord("makeItems", command)) makeItemsJsCommand(command, trackList, trackHash); else warn("Unrecognized jsCommand %s", command); } @@ -7301,30 +7429,31 @@ pthread_mutex_unlock( &pfdMutex ); if (allDone) return NULL; long thisTime = 0, lastTime = 0; if (measureTiming) lastTime = clock1000(); /* protect against errAbort */ struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) { pfd->done = FALSE; checkMaxWindowToDraw(pfd->track); + checkCollapseEmptySubtracks(pfd->track); pfd->track->loadItems(pfd->track); pfd->done = TRUE; } errCatchEnd(errCatch); if (errCatch->gotError) { pfd->track->networkErrMsg = cloneString(errCatch->message->string); pfd->done = TRUE; } errCatchFree(&errCatch); if (measureTiming) { thisTime = clock1000(); pfd->track->loadTime = thisTime - lastTime; @@ -7946,30 +8075,32 @@ } // TODO GALT /* load regular tracks */ for (track = trackList; track != NULL; track = track->next) { if (track->visibility != tvHide) { if (!track->parallelLoading) { if (measureTiming) lastTime = clock1000(); checkMaxWindowToDraw(track); + checkCollapseEmptySubtracks(track); // TODO: Test with multi-window feature + checkIfWiggling(cart, track); if (!loadHack) { track->loadItems(track); } else { // TEMP HACK GALT REMOVE if (currentWindow == windows) // first window { track->loadItems(track); } else {