a137caa4731599f89daf08875f24bcb19b11e597 galt Thu Sep 11 14:32:31 2025 -0700 Deals with multi-region and bigBeds so limitWiggle optimization works properly with all windows or regions. Fixes error test cases in rm35580 as well as full chromosome and recount3 and lrg and unusual tracks etc. Also works with quickLift and bigLollly now. Tested all tracks on hg38 and some public trackhubs. fixes #35580 diff --git src/hg/hgTracks/hgTracks.c src/hg/hgTracks/hgTracks.c index a724f2aee1d..9cf4b5e695e 100644 --- src/hg/hgTracks/hgTracks.c +++ src/hg/hgTracks/hgTracks.c @@ -5743,31 +5743,31 @@ // center label slice of tracks Must always make, even if the centerLabel is empty sliceHeight = fontHeight; sliceOffsetY = y; curImgTrack = imgBoxTrackFind(theImgBox,track->tdb,NULL); curSlice = imgTrackSliceUpdateOrAdd(curImgTrack,stCenter,theOneImg,NULL, sliceWidth[stData],sliceHeight, sliceOffsetX[stData],sliceOffsetY); (void) sliceMapFindOrStart(curSlice,track->tdb->track,NULL); // No common linkRoot if (isCenterLabelConditional(track)) // sometimes calls track height, especially when no data there { imgTrackUpdateCenterLabelSeen(curImgTrack,isCenterLabelConditionallySeen(track) ? clNowSeen : clNotSeen); } } - int savey = y; // GALT + int savey = y; y = doCenterLabels(track, track, hvg, font, y, fullInsideWidth); // calls track height y = savey + flatTrack->maxHeight; } hvGfxUnclip(hvg); setGlobalsFromWindow(windows); // first window } /* Draw tracks. */ { // brace allows local vars long lastTime = 0; @@ -7759,69 +7759,93 @@ if (cartVis) child->visibility = hTvFromString(cartVis); } } } } lmCleanup(&lm); } struct paraFetchData { struct paraFetchData *next; struct track *track; boolean done; + boolean doLoadSummary; }; static boolean isTrackForParallelLoad(struct track *track) /* Is this a track that should be loaded in parallel ? */ { char *bdu = trackDbSetting(track->tdb, "bigDataUrl"); return customFactoryParallelLoad(bdu, track->tdb->type) && (track->subtracks == NULL); } -static void findLeavesForParallelLoad(struct track *trackList, struct paraFetchData **ppfdList) +static void findLeavesForParallelLoad(struct track *trackList, struct paraFetchData **ppfdList, boolean doLoadSummary) /* Find leaves of track tree that are remote network resources for parallel-fetch loading */ { struct track *track; if (!trackList) return; for (track = trackList; track != NULL; track = track->next) { + char *quickLiftFile = trackDbSetting(track->tdb, "quickLiftUrl"); + if (doLoadSummary && quickLiftFile) + continue; + + if (doLoadSummary && !track->loadSummary) + continue; + + if (doLoadSummary && startsWith("bigLolly", track->tdb->type)) + continue; + if (track->visibility != tvHide) { if (isTrackForParallelLoad(track)) { struct paraFetchData *pfd; AllocVar(pfd); pfd->track = track; // need pointer to be stable + pfd->doLoadSummary = doLoadSummary; slAddHead(ppfdList, pfd); track->parallelLoading = TRUE; } struct track *subtrack; for (subtrack=track->subtracks; subtrack; subtrack=subtrack->next) { + + char *quickLiftFile = cloneString(trackDbSetting(subtrack->tdb, "quickLiftUrl")); + if (doLoadSummary && quickLiftFile) + continue; + + if (doLoadSummary && !subtrack->loadSummary) + continue; + + if (doLoadSummary && startsWith("bigLolly", subtrack->tdb->type)) + continue; + if (isTrackForParallelLoad(subtrack)) { if (tdbVisLimitedByAncestors(cart,subtrack->tdb,TRUE,TRUE) != tvHide) { struct paraFetchData *pfd; AllocVar(pfd); pfd->track = subtrack; // need pointer to be stable + pfd->doLoadSummary = doLoadSummary; slAddHead(ppfdList, pfd); subtrack->parallelLoading = TRUE; } } } } } } static pthread_mutex_t pfdMutex = PTHREAD_MUTEX_INITIALIZER; static struct paraFetchData *pfdList = NULL, *pfdRunning = NULL, *pfdDone = NULL, *pfdNeverStarted = NULL; static void checkHideEmptySubtracks(struct track *tg) /* Suppress queries on subtracks w/o data in window (identified from multiIntersect file) */ { @@ -7940,38 +7964,47 @@ } pthread_mutex_unlock( &pfdMutex ); if (allDone) return NULL; long thisTime = 0, lastTime = 0; if (measureTiming) lastTime = clock1000(); /* protect against errAbort */ struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) { pfd->done = FALSE; + + if (pfd->doLoadSummary) + { + pfd->track->loadSummary(pfd->track); + } + else + { checkMaxWindowToDraw(pfd->track); checkHideEmptySubtracks(pfd->track); pfd->track->loadItems(pfd->track); if (tdbHasDecorators(pfd->track)) { loadDecorators(pfd->track); decoratorMethods(pfd->track); } + } + pfd->done = TRUE; } errCatchEnd(errCatch); if (errCatch->gotWarning) { // do something intelligent to permit reporting of warnings // Can't pass it to warn yet - the fancy warnhandlers aren't ready } if (errCatch->gotError) { pfd->track->networkErrMsg = cloneString(errCatch->message->string); pfd->done = TRUE; } errCatchFree(&errCatch); @@ -8848,74 +8881,82 @@ subcopy->nextWindow = NULL; subcopy->prevWindow = subtrack; slAddHead(©->subtracks, subcopy); subtrack->nextWindow = subcopy; } } slReverse(©->subtracks); } } slReverse(&newTrackList); trackList = newTrackList; window->next->trackList = trackList; // save new track list in window } trackList = windows->trackList; // restore original track list + // Loop over each window loading all tracks trackLoadingInProgress = TRUE; // LOAD OPTIMIZATION HACK GALT // This is an attempt to try to optimize loading by having multiple regions // treated as a single span. The hack just grabs the dimensions of the first and last windows // and uses the loader in the first window to load them, then copies the results to all tracks. // This basically has only been tried on BED-like tracks, and only for exon/gene-mostly vmodes. // I am not re-partitioning the results after the load, so this means all windows see all items. // The reason that tends to work is that by luck most BED handlers have code to check if the item // overlaps the current window and to skip it if it does not. // I do not expect something so simple would work with wigs and other track types. // Even if we do want to optimize the BED-like tracks (which are already the fastest loading type), // to handle all of the virtmodes properly, this would have be be done differently. // Instead of just lumping them all into a single range, you would have to cluster together // ranges that are close together and on the same chromosome. // Clearly this was just to test an idea for optimizing. // NOT FINISHED. bool loadHack = FALSE; //TRUE; // probably should only be tried on non-wiggle tracks //warn ("loadHack = %d", loadHack); // TODO int lastWinEnd = 0; for (window=windows; window; window=window->next) lastWinEnd = window->winEnd; +int doLoadLoop; +boolean doLoadSummary = FALSE; + +for (doLoadLoop=0; doLoadLoop < 2; ++doLoadLoop) + { + for (window=windows; window; window=window->next) { trackList = window->trackList; // set track list setGlobalsFromWindow(window); // TEMP HACK GALT REMOVE if (loadHack) { if (currentWindow == windows) // first window winEnd = lastWinEnd; // so now we load the entire span inside the first window. } /* pre-load remote tracks in parallel */ int ptMax = atoi(cfgOptionDefault("parallelFetch.threads", "20")); // default number of threads for parallel fetch. int pfdListCount = 0; pthread_t *threads = NULL; if (ptMax > 0) // parallelFetch.threads=0 to disable parallel fetch { - findLeavesForParallelLoad(trackList, &pfdList); + findLeavesForParallelLoad(trackList, &pfdList, doLoadSummary); pfdListCount = slCount(pfdList); + /* launch parallel threads */ ptMax = min(ptMax, pfdListCount); if (ptMax > 0) { AllocArray(threads, ptMax); /* Create threads */ int pt; for (pt = 0; pt < ptMax; ++pt) { int rc = pthread_create(&threads[pt], NULL, remoteParallelLoad, NULL); if (rc) { errAbort("Unexpected error %d from pthread_create(): %s",rc,strerror(rc)); } pthread_detach(threads[pt]); // this thread will never join back with it's progenitor @@ -8931,37 +8972,78 @@ if (track->visibility != tvHide) { if (!track->parallelLoading) { if (measureTiming) lastTime = clock1000(); checkMaxWindowToDraw(track); checkHideEmptySubtracks(track); checkIfWiggling(cart, track); if (!loadHack) { + + if (doLoadSummary) + { + if (track->loadSummary) + { + if (!track->subtracks) + { + char *quickLiftFile = trackDbSetting(track->tdb, "quickLiftUrl"); + if (!quickLiftFile) + { + if (!startsWith("bigLolly", track->tdb->type)) + { + track->loadSummary(track); + } + } + } + } + struct track *subtrack; + for (subtrack=track->subtracks; subtrack; subtrack=subtrack->next) + { + if (tdbVisLimitedByAncestors(cart,subtrack->tdb,TRUE,TRUE) != tvHide) + { + if (!subtrack->parallelLoading) + { + char *quickLiftFile = trackDbSetting(subtrack->tdb, "quickLiftUrl"); + if (!quickLiftFile) + { + if (!startsWith("bigLolly", subtrack->tdb->type)) + { + if (subtrack->loadSummary) + subtrack->loadSummary(subtrack); + } + } + } + } + } + } + else + { track->loadItems(track); if (tdbHasDecorators(track)) { loadDecorators(track); decoratorMethods(track); } } + + } else { // TEMP HACK GALT REMOVE if (currentWindow == windows) // first window { track->loadItems(track); } else { track->items = track->prevWindow->items; // just point to the previous windows items (faster than loading) // apparently loadItems is setting some other fields that we want, but which ones? track->visibility = track->prevWindow->visibility; track->limitedVis = track->prevWindow->limitedVis; track->limitedVisSet = track->prevWindow->limitedVisSet; track->height = track->prevWindow->height; @@ -8976,30 +9058,34 @@ thisTime = clock1000(); track->loadTime = thisTime - lastTime; } } } } if (ptMax > 0) { /* wait for remote parallel load to finish */ remoteParallelLoadWait(getParaLoadTimeout()); // wait up to default 90 seconds. if (measureTiming) measureTime("Waiting for parallel (%d threads for %d tracks) remote data fetch", ptMax, pfdListCount); } } + + doLoadSummary = TRUE; + } + trackLoadingInProgress = FALSE; setGlobalsFromWindow(windows); // first window // restore globals trackList = windows->trackList; // restore track list // Some loadItems() calls will have already set limitedVis. // Look for lowest limitedVis across all windows // if found, set all windows to same lowest limitedVis for (track = trackList; track != NULL; track = track->next) { setSharedLimitedVisAcrossWindows(track); struct track *sub; for (sub=track->subtracks; sub; sub=sub->next) { setSharedLimitedVisAcrossWindows(sub);