ffcb793749fe12db176c00c04c0bd0676cf76a4d hiram Tue May 14 16:15:12 2019 -0700 now indicating maxItemsOutput limit reached refs #18869 diff --git src/hg/hubApi/getData.c src/hg/hubApi/getData.c index 80c4b48..43f6735 100644 --- src/hg/hubApi/getData.c +++ src/hg/hubApi/getData.c @@ -32,30 +32,32 @@ jsonWriteNumber(jw, NULL, (long long)e); jsonWriteDouble(jw, NULL, val); jsonWriteListEnd(jw); } else { jsonWriteObjectStart(jw, NULL); jsonWriteNumber(jw, "start", (long long)s); jsonWriteNumber(jw, "end", (long long)e); jsonWriteDouble(jw, "value", val); jsonWriteObjectEnd(jw); } ++itemCount; } } +if ((itemCount + itemsDone) >= maxItemsOutput) + reachedMaxItems = TRUE; return itemCount; } /* static unsigned wigTableDataOutput(struct jsonWrite *jw, ...) */ static void jsonDatumOut(struct jsonWrite *jw, char *name, char *val, int jsonType) /* output a json item, determine type, appropriate output, name can be NULL */ { if (JSON_DOUBLE == jsonType) jsonWriteDouble(jw, name, sqlDouble(val)); else if (JSON_NUMBER == jsonType) jsonWriteNumber(jw, name, sqlLongLong(val)); else jsonWriteString(jw, name, val); } @@ -104,30 +106,32 @@ jsonWriteListStart(jw, NULL); for (i = 0; i < columnCount; ++i) jsonDatumOut(jw, NULL, row[i], jsonTypes[i]); jsonWriteListEnd(jw); } else { jsonWriteObjectStart(jw, NULL); for (i = 0; i < columnCount; ++i) jsonDatumOut(jw, columnNames[i], row[i], jsonTypes[i]); jsonWriteObjectEnd(jw); } ++itemCount; } sqlFreeResult(&sr); +if ((itemCount + itemsDone) >= maxItemsOutput) + reachedMaxItems = TRUE; return itemCount; } static void tableDataOutput(char *db, struct trackDb *tdb, struct sqlConnection *conn, struct jsonWrite *jw, char *track, char *chrom, unsigned start, unsigned end) /* output the SQL table data for given track */ { /* for MySQL select statements, name for 'chrom' 'start' 'end' to use * for a table which has different names than that */ char chromName[256]; char startName[256]; char endName[256]; @@ -332,30 +336,32 @@ if (hti && hti->isSplit) /* when split, make up split chr name */ { safef(fullTableName, sizeof(fullTableName), "%s_%s", ci->chrom, hti->rootName); sqlDyStringPrintf(query, "select * from %s", fullTableName); } else sqlDyStringPrintf(query, "select * from %s", splitSqlTable); if (startsWith("wig", tdb->type)) itemsDone += wigTableDataOutput(jw, db, splitSqlTable, chrom, start, end, itemsDone); else itemsDone += sqlQueryJsonOutput(conn, jw, query->string, columnCount, columnNames, jsonTypes, itemsDone); jsonWriteListEnd(jw); /* chrom data output list end */ } + if (itemsDone >= maxItemsOutput) + reachedMaxItems = TRUE; jsonWriteObjectEnd(jw); /* end track data output */ } else { /* a single chrom has been requested, run it */ jsonWriteListStart(jw, track); /* data output list starting */ itemsDone += sqlQueryJsonOutput(conn, jw, query->string, columnCount, columnNames, jsonTypes, itemsDone); jsonWriteListEnd(jw); /* data output list end */ } freeDyString(&query); } /* static void tableDataOutput(char *db, struct trackDb *tdb, ... ) */ static unsigned bbiDataOutput(struct jsonWrite *jw, struct bbiFile *bbi, char *chrom, unsigned start, unsigned end, struct sqlFieldType *fiList, struct trackDb *tdb, unsigned itemsDone) @@ -396,30 +402,32 @@ { jsonWriteListStart(jw, NULL); for (i = 0; i < bbi->fieldCount; ++i) jsonDatumOut(jw, NULL, row[i], jsonTypes[i]); jsonWriteListEnd(jw); } else { jsonWriteObjectStart(jw, NULL); for (i = 0; i < bbi->fieldCount; ++i, fi = fi->next) jsonDatumOut(jw, fi->name, row[i], jsonTypes[i]); jsonWriteObjectEnd(jw); } ++itemCount; } + if (itemCount >= maxItemsOutput) + reachedMaxItems = TRUE; lmCleanup(&bbLm); return itemCount; } /* static void bbiDataOutput(struct jsonWrite *jw, . . . ) */ static unsigned wigDataOutput(struct jsonWrite *jw, struct bbiFile *bwf, char *chrom, unsigned start, unsigned end) /* output wig data for one chrom in the given bwf file, return itemCount out */ { unsigned itemCount = 0; struct lm *lm = lmInit(0); struct bbiInterval *iv, *ivList = bigWigIntervalQuery(bwf, chrom, start, end, lm); if (NULL == ivList) return itemCount; jsonWriteListStart(jw, chrom); @@ -435,47 +443,51 @@ jsonWriteNumber(jw, NULL, (long long)e); jsonWriteDouble(jw, NULL, val); jsonWriteListEnd(jw); } else { jsonWriteObjectStart(jw, NULL); jsonWriteNumber(jw, "start", (long long)s); jsonWriteNumber(jw, "end", (long long)e); jsonWriteDouble(jw, "value", val); jsonWriteObjectEnd(jw); } ++itemCount; } jsonWriteListEnd(jw); +if (itemCount >= maxItemsOutput) + reachedMaxItems = TRUE; return itemCount; } static void wigData(struct jsonWrite *jw, struct bbiFile *bwf, char *chrom, unsigned start, unsigned end) /* output the data for a bigWig bbi file */ { struct bbiChromInfo *chromList = NULL; if (isEmpty(chrom)) { chromList = bbiChromList(bwf); struct bbiChromInfo *bci; unsigned itemsDone = 0; for (bci = chromList; bci && (itemsDone < maxItemsOutput); bci = bci->next) { itemsDone += wigDataOutput(jw, bwf, bci->name, 0, bci->size); } + if (itemsDone >= maxItemsOutput) + reachedMaxItems = TRUE; } else (void) wigDataOutput(jw, bwf, chrom, start, end); } static void bigColumnTypes(struct jsonWrite *jw, struct sqlFieldType *fiList, struct asObject *as) /* show the column types from a big file autoSql definitions */ { struct asColumn *columnEl = as->columnList; jsonWriteListStart(jw, "columnTypes"); struct sqlFieldType *fi = fiList; for ( ; fi; fi = fi->next, columnEl = columnEl->next) { int jsonType = autoSqlToJsonType(fi->type); @@ -565,30 +577,32 @@ struct asObject *as = bigBedAsOrDefault(bbi); struct sqlFieldType *fiList = sqlFieldTypesFromAs(as); if (jsonOutputArrays || debug) bigColumnTypes(jw, fiList, as); jsonWriteListStart(jw, track); unsigned itemsDone = 0; if (isEmpty(chrom)) { struct bbiChromInfo *bci; for (bci = chromList; bci && (itemsDone < maxItemsOutput); bci = bci->next) { itemsDone += bbiDataOutput(jw, bbi, bci->name, 0, bci->size, fiList, thisTrack, itemsDone); } + if (itemsDone >= maxItemsOutput) + reachedMaxItems = TRUE; } else itemsDone += bbiDataOutput(jw, bbi, chrom, uStart, uEnd, fiList, thisTrack, itemsDone); jsonWriteListEnd(jw); } else if (startsWith("bigWig", thisTrack->type)) { if (jsonOutputArrays || debug) wigColumnTypes(jw); jsonWriteObjectStart(jw, track); wigData(jw, bbi, chrom, uStart, uEnd); jsonWriteObjectEnd(jw); } bbiFileClose(&bbi); @@ -753,30 +767,32 @@ struct asObject *as = bigBedAsOrDefault(bbi); struct sqlFieldType *fiList = sqlFieldTypesFromAs(as); if (jsonOutputArrays || debug) bigColumnTypes(jw, fiList, as); jsonWriteListStart(jw, track); unsigned itemsDone = 0; if (isEmpty(chrom)) { struct bbiChromInfo *bci; for (bci = chromList; bci && (itemsDone < maxItemsOutput); bci = bci->next) { itemsDone += bbiDataOutput(jw, bbi, bci->name, 0, bci->size, fiList, thisTrack, itemsDone); } + if (itemsDone >= maxItemsOutput) + reachedMaxItems = TRUE; } else itemsDone += bbiDataOutput(jw, bbi, chrom, uStart, uEnd, fiList, thisTrack, itemsDone); jsonWriteListEnd(jw); } else if (startsWith("bigWig", thisTrack->type)) { if (jsonOutputArrays || debug) wigColumnTypes(jw); jsonWriteObjectStart(jw, track); wigData(jw, bbi, chrom, uStart, uEnd); jsonWriteObjectEnd(jw); bbiFileClose(&bbi);