e72cf0100e06d6fadb7282d4e7eb2c592f320951 galt Mon Jul 4 13:08:35 2011 -0700 Adding parallel-fetch loading of remote bigDataUrl tracks using pthreads diff --git src/hg/hgTracks/bedTrack.c src/hg/hgTracks/bedTrack.c index d25e693..e92e817 100644 --- src/hg/hgTracks/bedTrack.c +++ src/hg/hgTracks/bedTrack.c @@ -22,31 +22,30 @@ { char buf[1024]; struct bed *ret = bedLoad6(row); safef(buf, sizeof(buf), "%s%c%s%c%s", ret->name, SEQ_DELIM, row[6], SEQ_DELIM, row[7]); freez(&(ret->name)); ret->name = cloneString(buf); return ret; } void loadSimpleBed(struct track *tg) /* Load the items in one track - just move beds in * window... */ { struct bed *(*loader)(char **row); struct bed *bed, *list = NULL; -struct sqlConnection *conn = hAllocConnTrack(database, tg->tdb); char **row; int rowOffset; char *words[3]; int wordCt; char query[128]; char *setting = NULL; bool doScoreCtFilter = FALSE; int scoreFilterCt = 0; char *topTable = NULL; if (tg->bedSize <= 3) loader = bedLoad3; else if (tg->bedSize == 4) loader = bedLoad; else if (tg->bedSize == 5) @@ -70,89 +69,87 @@ { doScoreCtFilter = cartUsualBooleanClosestToHome(cart, tg->tdb, FALSE, "filterTopScorersOn",sameString(words[0], "on")); scoreFilterCt = cartUsualIntClosestToHome(cart, tg->tdb, FALSE, "filterTopScorersCt", atoi(words[1])); topTable = words[2]; /* if there are not too many rows in the table then can define */ /* top table as the track or subtrack table */ if (sameWord(topTable, "self")) topTable = cloneString(tg->table); } } /* Get list of items */ if (tg->isBigBed) - { + { // avoid opening an unneeded db connection for bigBed; required not to use mysql for parallel fetch tracks char *scoreFilter = cartOrTdbString(cart, tg->tdb, "scoreFilter", NULL); - if (scoreFilter != NULL || tg->visibility != tvDense) - { struct lm *lm = lmInit(0); struct bigBedInterval *bb, *bbList = bigBedSelectRange(tg, chromName, winStart, winEnd, lm); char *bedRow[32]; char startBuf[16], endBuf[16]; int minScore = 0; if (scoreFilter) minScore = atoi(scoreFilter); for (bb = bbList; bb != NULL; bb = bb->next) { bigBedIntervalToRow(bb, chromName, startBuf, endBuf, bedRow, ArraySize(bedRow)); bed = loader(bedRow); if (scoreFilter == NULL || bed->score >= minScore) slAddHead(&list, bed); } lmCleanup(&lm); } - } else { + struct sqlConnection *conn = hAllocConnTrack(database, tg->tdb); struct sqlResult *sr = NULL; /* limit to items above a specified score */ char *scoreFilterClause = getScoreFilterClause(cart, tg->tdb,NULL); if (doScoreCtFilter && (topTable != NULL) && hTableExists(database, topTable)) { safef(query, sizeof(query),"select * from %s order by score desc limit %d", topTable, scoreFilterCt); sr = sqlGetResult(conn, query); rowOffset = hOffsetPastBin(database, hDefaultChrom(database), topTable); } else if(scoreFilterClause != NULL && tg->bedSize >= 5) { sr = hRangeQuery(conn, tg->table, chromName, winStart, winEnd, scoreFilterClause, &rowOffset); } else { sr = hRangeQuery(conn, tg->table, chromName, winStart, winEnd, NULL, &rowOffset); } freeMem(scoreFilterClause); while ((row = sqlNextRow(sr)) != NULL) { bed = loader(row+rowOffset); slAddHead(&list, bed); } sqlFreeResult(&sr); + hFreeConn(&conn); } if (doScoreCtFilter) { /* filter out items not in this window */ struct bed *newList = bedFilterListInRange(list, NULL, chromName, winStart, winEnd); list = newList; } slReverse(&list); -hFreeConn(&conn); tg->items = list; } void bed8To12(struct bed *bed) /* Turn a bed 8 into a bed 12 by defining one block. */ { // Make up a block: the whole thing. bed->blockCount = 1; bed->blockSizes = needMem(bed->blockCount * sizeof(int)); bed->chromStarts = needMem(bed->blockCount * sizeof(int)); bed->blockSizes[0] = bed->chromEnd - bed->chromStart; bed->chromStarts[0] = 0; // Some tracks overload thickStart and thickEnd -- catch garbage here. if ((bed->thickStart != 0) && ((bed->thickStart < bed->chromStart) || @@ -174,120 +171,120 @@ adjustBedScoreGrayLevel(tdb, bed, scoreMin, scoreMax); struct linkedFeatures *lf = lfFromBedExtra(bed, scoreMin, scoreMax); if (useItemRgb) { lf->extra = (void *)USE_ITEM_RGB; /* signal for coloring */ lf->filterColor=bed->itemRgb; } bedFree(pBed); return lf; } void loadBed9(struct track *tg) /* Convert bed 9 info in window to linked feature. (to handle itemRgb)*/ { struct trackDb *tdb = tg->tdb; -struct sqlConnection *conn = hAllocConnTrack(database, tdb); struct bed *bed; struct linkedFeatures *lfList = NULL, *lf; int scoreMin = atoi(trackDbSettingClosestToHomeOrDefault(tdb, "scoreMin", "0")); int scoreMax = atoi(trackDbSettingClosestToHomeOrDefault(tdb, "scoreMax", "1000")); boolean useItemRgb = FALSE; useItemRgb = bedItemRgb(tdb); if (tg->isBigBed) - { + { // avoid opening an unneeded db connection for bigBed; required not to use mysql for parallel fetch tracks bigBedAddLinkedFeaturesFrom(tg, chromName, winStart, winEnd, scoreMin, scoreMax, useItemRgb, 9, &lfList); } else { + struct sqlConnection *conn = hAllocConnTrack(database, tdb); struct sqlResult *sr; char **row; int rowOffset; char *scoreFilterClause = getScoreFilterClause(cart, tg->tdb,NULL); if (scoreFilterClause != NULL) { sr = hRangeQuery(conn, tg->table, chromName, winStart, winEnd,scoreFilterClause, &rowOffset); freeMem(scoreFilterClause); } else { sr = hRangeQuery(conn, tg->table, chromName, winStart, winEnd, NULL, &rowOffset); } while ((row = sqlNextRow(sr)) != NULL) { bed = bedLoadN(row+rowOffset, 9); lf = bedMungToLinkedFeatures(&bed, tdb, 9, scoreMin, scoreMax, useItemRgb); slAddHead(&lfList, lf); } sqlFreeResult(&sr); - } hFreeConn(&conn); + } slReverse(&lfList); slSort(&lfList, linkedFeaturesCmp); tg->items = lfList; } void loadBed8(struct track *tg) /* Convert bed 8 info in window to linked feature. */ { -struct sqlConnection *conn = hAllocConn(database); struct bed *bed; struct linkedFeatures *lfList = NULL, *lf; struct trackDb *tdb = tg->tdb; int scoreMin = atoi(trackDbSettingClosestToHomeOrDefault(tdb, "scoreMin", "0")); int scoreMax = atoi(trackDbSettingClosestToHomeOrDefault(tdb, "scoreMax", "1000")); boolean useItemRgb = FALSE; useItemRgb = bedItemRgb(tdb); if (tg->isBigBed) - { + { // avoid opening an unneeded db connection for bigBed; required not to use mysql for parallel fetch tracks bigBedAddLinkedFeaturesFrom(tg, chromName, winStart, winEnd, scoreMin, scoreMax, useItemRgb, 8, &lfList); } else { + struct sqlConnection *conn = hAllocConn(database); struct sqlResult *sr; char **row; int rowOffset; char *scoreFilterClause = getScoreFilterClause(cart, tg->tdb,NULL); if (scoreFilterClause != NULL) { sr = hRangeQuery(conn, tg->table, chromName, winStart, winEnd,scoreFilterClause, &rowOffset); freeMem(scoreFilterClause); } else { sr = hRangeQuery(conn, tg->table, chromName, winStart, winEnd, NULL, &rowOffset); } while ((row = sqlNextRow(sr)) != NULL) { bed = bedLoadN(row+rowOffset, 8); lf = bedMungToLinkedFeatures(&bed, tdb, 8, scoreMin, scoreMax, useItemRgb); slAddHead(&lfList, lf); } sqlFreeResult(&sr); - } hFreeConn(&conn); + } slReverse(&lfList); slSort(&lfList, linkedFeaturesCmp); tg->items = lfList; } static void filterBed(struct track *tg, struct linkedFeatures **pLfList) /* Apply filters if any to mRNA linked features. */ { struct linkedFeatures *lf, *next, *newList = NULL, *oldList = NULL; struct mrnaUiData *mud = tg->extraUiData; struct mrnaFilter *fil; char *type; boolean anyFilter = FALSE; boolean colorIx = 0; boolean isExclude = FALSE; @@ -421,78 +418,78 @@ } } *pLfList = newList; tg->limitedVisSet = FALSE; /* Need to recalculate this after filtering. */ /* Free up hashes, etc. */ for (fil = mud->filterList; fil != NULL; fil = fil->next) { hashFree(&fil->hash); } } void loadGappedBed(struct track *tg) /* Convert bed info in window to linked feature. */ { -struct sqlConnection *conn = hAllocConn(database); struct sqlResult *sr; char **row; int rowOffset; struct bed *bed; struct linkedFeatures *lfList = NULL, *lf; struct trackDb *tdb = tg->tdb; int scoreMin = atoi(trackDbSettingClosestToHomeOrDefault(tdb, "scoreMin", "0")); int scoreMax = atoi(trackDbSettingClosestToHomeOrDefault(tdb, "scoreMax", "1000")); boolean useItemRgb = FALSE; useItemRgb = bedItemRgb(tdb); if (tg->isBigBed) - { + { // avoid opening an unneeded db connection for bigBed; required not to use mysql for parallel fetch tracks bigBedAddLinkedFeaturesFrom(tg, chromName, winStart, winEnd, scoreMin, scoreMax, useItemRgb, 12, &lfList); } else { /* Use tg->tdb->track because subtracks inherit composite track's tdb * by default, and the variable is named after the composite track. */ + struct sqlConnection *conn = hAllocConn(database); char *scoreFilterClause = getScoreFilterClause(cart, tg->tdb,NULL); if (scoreFilterClause != NULL) { sr = hRangeQuery(conn, tg->table, chromName, winStart, winEnd,scoreFilterClause, &rowOffset); freeMem(scoreFilterClause); } else { sr = hRangeQuery(conn, tg->table, chromName, winStart, winEnd, NULL, &rowOffset); } while ((row = sqlNextRow(sr)) != NULL) { bed = bedLoad12(row+rowOffset); adjustBedScoreGrayLevel(tdb, bed, scoreMin, scoreMax); lf = lfFromBedExtra(bed, scoreMin, scoreMax); if (useItemRgb) { lf->extra = (void *)USE_ITEM_RGB; /* signal for coloring */ lf->filterColor=bed->itemRgb; } slAddHead(&lfList, lf); bedFree(&bed); } sqlFreeResult(&sr); - } hFreeConn(&conn); + } slReverse(&lfList); if(tg->extraUiData) filterBed(tg, &lfList); slSort(&lfList, linkedFeaturesCmp); tg->items = lfList; } void bedDrawSimpleAt(struct track *tg, void *item, struct hvGfx *hvg, int xOff, int y, double scale, MgFont *font, Color color, enum trackVisibility vis) /* Draw a single simple bed item at position. */ { struct bed *bed = item; int heightPer = tg->heightPer; int x1 = round((double)((int)bed->chromStart-winStart)*scale) + xOff;