4833b6c8e4583d27b99b06a6bdf0357fe786215d galt Mon Sep 8 01:45:07 2025 -0700 Deals with multi-region and bigBeds so limitWiggle optimization works properly with all windows or regions. Fixes error test cases in rm35580 as well as full chromosome and recount3 and lrg and unusual tracks etc. fixes 35580 diff --git src/hg/hgTracks/hgTracks.c src/hg/hgTracks/hgTracks.c index a724f2aee1d..3c5774f1179 100644 --- src/hg/hgTracks/hgTracks.c +++ src/hg/hgTracks/hgTracks.c @@ -7759,69 +7759,79 @@ if (cartVis) child->visibility = hTvFromString(cartVis); } } } } lmCleanup(&lm); } struct paraFetchData { struct paraFetchData *next; struct track *track; boolean done; + boolean doLoadSummary; }; static boolean isTrackForParallelLoad(struct track *track) /* Is this a track that should be loaded in parallel ? */ { char *bdu = trackDbSetting(track->tdb, "bigDataUrl"); return customFactoryParallelLoad(bdu, track->tdb->type) && (track->subtracks == NULL); } -static void findLeavesForParallelLoad(struct track *trackList, struct paraFetchData **ppfdList) +static void findLeavesForParallelLoad(struct track *trackList, struct paraFetchData **ppfdList, boolean doLoadSummary) /* Find leaves of track tree that are remote network resources for parallel-fetch loading */ { struct track *track; if (!trackList) return; for (track = trackList; track != NULL; track = track->next) { + if (doLoadSummary && !track->loadSummary) + continue; + if (track->visibility != tvHide) { if (isTrackForParallelLoad(track)) { struct paraFetchData *pfd; AllocVar(pfd); pfd->track = track; // need pointer to be stable + pfd->doLoadSummary = doLoadSummary; slAddHead(ppfdList, pfd); track->parallelLoading = TRUE; } struct track *subtrack; for (subtrack=track->subtracks; subtrack; subtrack=subtrack->next) { + + if (doLoadSummary && !subtrack->loadSummary) + continue; + if (isTrackForParallelLoad(subtrack)) { if (tdbVisLimitedByAncestors(cart,subtrack->tdb,TRUE,TRUE) != tvHide) { struct paraFetchData *pfd; AllocVar(pfd); pfd->track = subtrack; // need pointer to be stable + pfd->doLoadSummary = doLoadSummary; slAddHead(ppfdList, pfd); subtrack->parallelLoading = TRUE; } } } } } } static pthread_mutex_t pfdMutex = PTHREAD_MUTEX_INITIALIZER; static struct paraFetchData *pfdList = NULL, *pfdRunning = NULL, *pfdDone = NULL, *pfdNeverStarted = NULL; static void checkHideEmptySubtracks(struct track *tg) /* Suppress queries on subtracks w/o data in window (identified from multiIntersect file) */ { @@ -7940,38 +7950,45 @@ } pthread_mutex_unlock( &pfdMutex ); if (allDone) return NULL; long thisTime = 0, lastTime = 0; if (measureTiming) lastTime = clock1000(); /* protect against errAbort */ struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) { pfd->done = FALSE; + + if (pfd->doLoadSummary) + pfd->track->loadSummary(pfd->track); + else + { checkMaxWindowToDraw(pfd->track); checkHideEmptySubtracks(pfd->track); pfd->track->loadItems(pfd->track); if (tdbHasDecorators(pfd->track)) { loadDecorators(pfd->track); decoratorMethods(pfd->track); } + } + pfd->done = TRUE; } errCatchEnd(errCatch); if (errCatch->gotWarning) { // do something intelligent to permit reporting of warnings // Can't pass it to warn yet - the fancy warnhandlers aren't ready } if (errCatch->gotError) { pfd->track->networkErrMsg = cloneString(errCatch->message->string); pfd->done = TRUE; } errCatchFree(&errCatch); @@ -8872,49 +8889,55 @@ // The reason that tends to work is that by luck most BED handlers have code to check if the item // overlaps the current window and to skip it if it does not. // I do not expect something so simple would work with wigs and other track types. // Even if we do want to optimize the BED-like tracks (which are already the fastest loading type), // to handle all of the virtmodes properly, this would have be be done differently. // Instead of just lumping them all into a single range, you would have to cluster together // ranges that are close together and on the same chromosome. // Clearly this was just to test an idea for optimizing. // NOT FINISHED. bool loadHack = FALSE; //TRUE; // probably should only be tried on non-wiggle tracks //warn ("loadHack = %d", loadHack); // TODO int lastWinEnd = 0; for (window=windows; window; window=window->next) lastWinEnd = window->winEnd; +int doLoadLoop; +boolean doLoadSummary = FALSE; + +for (doLoadLoop=0; doLoadLoop < 2; ++doLoadLoop) + { + for (window=windows; window; window=window->next) { trackList = window->trackList; // set track list setGlobalsFromWindow(window); // TEMP HACK GALT REMOVE if (loadHack) { if (currentWindow == windows) // first window winEnd = lastWinEnd; // so now we load the entire span inside the first window. } /* pre-load remote tracks in parallel */ int ptMax = atoi(cfgOptionDefault("parallelFetch.threads", "20")); // default number of threads for parallel fetch. int pfdListCount = 0; pthread_t *threads = NULL; if (ptMax > 0) // parallelFetch.threads=0 to disable parallel fetch { - findLeavesForParallelLoad(trackList, &pfdList); + findLeavesForParallelLoad(trackList, &pfdList, doLoadSummary); pfdListCount = slCount(pfdList); /* launch parallel threads */ ptMax = min(ptMax, pfdListCount); if (ptMax > 0) { AllocArray(threads, ptMax); /* Create threads */ int pt; for (pt = 0; pt < ptMax; ++pt) { int rc = pthread_create(&threads[pt], NULL, remoteParallelLoad, NULL); if (rc) { errAbort("Unexpected error %d from pthread_create(): %s",rc,strerror(rc)); } @@ -8931,37 +8954,47 @@ if (track->visibility != tvHide) { if (!track->parallelLoading) { if (measureTiming) lastTime = clock1000(); checkMaxWindowToDraw(track); checkHideEmptySubtracks(track); checkIfWiggling(cart, track); if (!loadHack) { + + if (doLoadSummary) + { + if (track->loadSummary) + track->loadSummary(track); + } + else + { track->loadItems(track); if (tdbHasDecorators(track)) { loadDecorators(track); decoratorMethods(track); } } + + } else { // TEMP HACK GALT REMOVE if (currentWindow == windows) // first window { track->loadItems(track); } else { track->items = track->prevWindow->items; // just point to the previous windows items (faster than loading) // apparently loadItems is setting some other fields that we want, but which ones? track->visibility = track->prevWindow->visibility; track->limitedVis = track->prevWindow->limitedVis; track->limitedVisSet = track->prevWindow->limitedVisSet; track->height = track->prevWindow->height; @@ -8976,30 +9009,34 @@ thisTime = clock1000(); track->loadTime = thisTime - lastTime; } } } } if (ptMax > 0) { /* wait for remote parallel load to finish */ remoteParallelLoadWait(getParaLoadTimeout()); // wait up to default 90 seconds. if (measureTiming) measureTime("Waiting for parallel (%d threads for %d tracks) remote data fetch", ptMax, pfdListCount); } } + + doLoadSummary = TRUE; + } + trackLoadingInProgress = FALSE; setGlobalsFromWindow(windows); // first window // restore globals trackList = windows->trackList; // restore track list // Some loadItems() calls will have already set limitedVis. // Look for lowest limitedVis across all windows // if found, set all windows to same lowest limitedVis for (track = trackList; track != NULL; track = track->next) { setSharedLimitedVisAcrossWindows(track); struct track *sub; for (sub=track->subtracks; sub; sub=sub->next) { setSharedLimitedVisAcrossWindows(sub);