0e092ec31d4d48d6a85f4e68d081bd92d88a92b7 hiram Wed Apr 17 14:18:30 2019 -0700 correct array output better timing and column header information refs #18869 diff --git src/hg/hubApi/hubApi.c src/hg/hubApi/hubApi.c index f6db268..6abbcd2 100644 --- src/hg/hubApi/hubApi.c +++ src/hg/hubApi/hubApi.c @@ -21,36 +21,37 @@ int maxItemsOutput = 1000; /* can be set in URL maxItemsOutput=N */ static int maxItemLimit = 1000000; /* maximum of 1,000,000 items returned */ /* for debugging purpose, current bot delay value */ int botDelay = 0; boolean debug = FALSE; /* can be set in URL debug=1, to turn off: debug=0 */ #define delayFraction 0.03 /* default is to list all trackDb entries, composite containers too. * This option will limit to only the actual track entries with data */ boolean trackLeavesOnly = FALSE; /* set by CGI parameter 'trackLeavesOnly' */ /* this selects output type 'arrays', where the default type is: objects */ boolean jsonOutputArrays = FALSE; /* set by CGI parameter 'jsonOutputArrays' */ +boolean measureTiming = FALSE; /* set by CGI parameters */ + /* Global only to this one source file */ static struct cart *cart; /* CGI and other variables */ static struct hash *oldVars = NULL; static struct hash *trackCounter = NULL; static long totalTracks = 0; -static boolean measureTiming = FALSE; /* set by CGI parameters */ static boolean allTrackSettings = FALSE; /* checkbox setting */ static char **shortLabels = NULL; /* public hub short labels in array */ static int publicHubCount = 0; static char *defaultHub = "Plants"; static char *defaultDb = "ce11"; static long enteredMainTime = 0; /* will become = clock1000() on entry */ /* to allow calculation of when to bail out, taking too long */ static long timeOutSeconds = 100; static boolean timedOut = FALSE; static char *urlPrefix = ""; /* initalized to support self references */ /* supportedTypes will be initialized to a known supported set */ static struct slName *supportedTypes = NULL; @@ -226,54 +227,54 @@ if (hub) genome = hub->genomeList->name; struct dyString *extraDyFlags = newDyString(128); if (debug) dyStringAppend(extraDyFlags, ";debug=1"); if (jsonOutputArrays) dyStringAppend(extraDyFlags, ";jsonOutputArrays=1"); char *extraFlags = dyStringCannibalize(&extraDyFlags); if (db) { if (hub) { char urlReference[2048]; - safef(urlReference, sizeof(urlReference), " <a href='%s/getData/track?hubUrl=%s;genome=%s;track=%s;chrom=%s;start=%u;end=%u%s' target=_blank>(sample data)%s</a>\n", urlPrefix, hub->url, genome, tdb->track, chrom, start, end, extraFlags, errorPrint); + safef(urlReference, sizeof(urlReference), " <a href='%s/getData/track?hubUrl=%s;genome=%s;track=%s;chrom=%s;start=%u;end=%u;maxItemsOutput=5%s' target=_blank>(sample data)%s</a>\n", urlPrefix, hub->url, genome, tdb->track, chrom, start, end, extraFlags, errorPrint); if (tdb->parent) hPrintf("<li><b>%s</b>: %s subtrack of parent: %s%s</li>\n", tdb->track, tdb->type, tdb->parent->track, urlReference); else hPrintf("<li><b>%s</b>: %s%s</li>\n", tdb->track, tdb->type, urlReference); } else { char urlReference[2048]; - safef(urlReference, sizeof(urlReference), " <a href='%s/getData/track?db=%s;chrom=%s;track=%s;start=%u;end=%u%s' target=_blank>(sample data)%s</a>\n", urlPrefix, db, chrom, tdb->track, start, end, extraFlags, errorPrint); + safef(urlReference, sizeof(urlReference), " <a href='%s/getData/track?db=%s;chrom=%s;track=%s;start=%u;end=%u;maxItemsOutput=5%s' target=_blank>(sample data)%s</a>\n", urlPrefix, db, chrom, tdb->track, start, end, extraFlags, errorPrint); if (superChild) hPrintf("<li><b>%s</b>: %s superTrack child of parent: %s%s</li>\n", tdb->track, tdb->type, tdb->parent->track, urlReference); else if (tdb->parent) hPrintf("<li><b>%s</b>: %s subtrack of parent: %s%s</li>\n", tdb->track, tdb->type, tdb->parent->track, urlReference); else hPrintf("<li><b>%s</b>: %s%s</li>\n", tdb->track, tdb->type, urlReference ); } } else if (hub) { char urlReference[2048]; - safef(urlReference, sizeof(urlReference), " <a href='%s/getData/track?hubUrl=%s;genome=%s;track=%s;chrom=%s;start=%u;end=%u%s' target=_blank>(sample data)%s</a>\n", urlPrefix, hub->url, genome, tdb->track, chrom, start, end, extraFlags, errorPrint); + safef(urlReference, sizeof(urlReference), " <a href='%s/getData/track?hubUrl=%s;genome=%s;track=%s;chrom=%s;start=%u;end=%u;maxItemsOutput=5%s' target=_blank>(sample data)%s</a>\n", urlPrefix, hub->url, genome, tdb->track, chrom, start, end, extraFlags, errorPrint); if (tdb->parent) hPrintf("<li><b>%s</b>: %s subtrack of parent: %s%s</li>\n", tdb->track, tdb->type, tdb->parent->track, urlReference); else hPrintf("<li><b>%s</b>: %s%s</li>\n", tdb->track, tdb->type, urlReference); } else hPrintf("<li>%s : %s not db hub track ?</li>\n", tdb->track, tdb->type); } static void hubSampleUrl(struct trackHub *hub, struct trackDb *tdb, long chromCount, long itemCount, char *chromName, unsigned chromSize, char *genome, char *errorString) { unsigned start = chromSize / 4; @@ -300,31 +301,31 @@ char countsMessage[512]; countsMessage[0] = 0; if (chromCount > 0 || itemCount > 0) { if (allowedBigBedType(tdb->type)) safef(countsMessage, sizeof(countsMessage), " : %ld chroms : %ld item count ", chromCount, itemCount); else if (startsWithWord("bigWig", tdb->type)) safef(countsMessage, sizeof(countsMessage), " : %ld chroms : %ld bases covered ", chromCount, itemCount); else safef(countsMessage, sizeof(countsMessage), " : %ld chroms : %ld count ", chromCount, itemCount); } if (isSupportedType(tdb->type)) { char urlReference[2048]; - safef(urlReference, sizeof(urlReference), "<a href='%s/getData/track?hubUrl=%s;genome=%s;track=%s;chrom=%s;start=%u;end=%u%s' target=_blank>(sample data)%s</a>\n", urlPrefix, hub->url, genome, tdb->track, chromName, start, end, extraFlags, errorPrint); + safef(urlReference, sizeof(urlReference), "<a href='%s/getData/track?hubUrl=%s;genome=%s;track=%s;chrom=%s;start=%u;end=%u;maxItemsOutput=5%s' target=_blank>(sample data)%s</a>\n", urlPrefix, hub->url, genome, tdb->track, chromName, start, end, extraFlags, errorPrint); if (allowedBigBedType(tdb->type)) hPrintf(" <li><b>%s</b>: %s%s%s</li>\n", tdb->track, tdb->type, countsMessage, urlReference); else if (startsWithWord("bigWig", tdb->type)) hPrintf(" <li><b>%s</b>: %s%s%s</li>\n", tdb->track, tdb->type, countsMessage, urlReference); else hPrintf(" <li><b>%s</b>: %s%s%s</li>\n", tdb->track, tdb->type, countsMessage, urlReference); } else { if (allowedBigBedType(tdb->type)) hPrintf(" <li><b>%s</b>: %s%s</li>\n", tdb->track, tdb->type, countsMessage); else if (startsWithWord("bigWig", tdb->type)) hPrintf(" <li><b>%s</b>: %s%s</li>\n", tdb->track, tdb->type, countsMessage); else @@ -531,53 +532,49 @@ else hPrintf("<li><b>%s</b>: %s : subtrack of parent: %s</li>\n", tdbEl->track, tdbEl->type, tdbEl->parent->track); } hashCountTrack(tdbEl, countTracks); if (tdbEl->subtracks) showSubTracks(hub, db, tdbEl, countTracks, chromName, chromSize, errorString); } } hPrintf(" </ul></li>\n"); } static void trackSettings(struct trackDb *tdb, struct hash *countTracks) /* process the settingsHash for a trackDb, recursive when subtracks */ { hPrintf(" <li><ul>\n"); -// if (tdb->children) haven't yet seen a track with children ? -// hPrintf(" <li>%s: has children</li>\n", tdb->track); -// else -// hPrintf(" <li>%s: NO children</li>\n", tdb->track); struct hashEl *hel; struct hashCookie hc = hashFirst(tdb->settingsHash); while ((hel = hashNext(&hc)) != NULL) { if (sameWord("track", hel->name)) continue; // already output in header if (isEmpty((char *)hel->val)) hPrintf(" <li><b>%s</b>: <empty></li>\n", hel->name); else hPrintf(" <li><b>%s</b>: '%s'</li>\n", hel->name, (char *)hel->val); } if (tdb->subtracks) { struct trackDb *tdbEl = NULL; if (debug) hPrintf(" <li>has %d subtrack(s)</li>\n", slCount(tdb->subtracks)); for (tdbEl = tdb->subtracks; tdbEl; tdbEl = tdbEl->next) { - hPrintf("<li>subtrack: %s of parent: %s : type: '%s'</li>\n", tdbEl->track, tdbEl->parent->track, tdbEl->type); + hPrintf("<li>subtrack: %s of parent: %s : type: '%s' (TBD: sample data)</li>\n", tdbEl->track, tdbEl->parent->track, tdbEl->type); hashCountTrack(tdbEl, countTracks); trackSettings(tdbEl, countTracks); } } hPrintf(" </ul></li>\n"); } static void hubCountOneTdb(struct trackHub *hub, char *db, struct trackDb *tdb, char *bigDataIndex, struct hash *countTracks, char *chromName, unsigned chromSize, char *genome) { char *bigDataUrl = trackDbSetting(tdb, "bigDataUrl"); boolean compositeContainer = tdbIsComposite(tdb); boolean compositeView = tdbIsCompositeView(tdb); boolean superChild = tdbIsSuperTrackChild(tdb); @@ -881,31 +878,31 @@ char sizeString[64]; sprintLongWithCommas(sizeString, chromSize); hPrintf("<li><b>Sequence count</b> %d, <b>largest</b>: %s at %s bases</li>\n", slCount(ci), chromName, sizeString); safef(urlReference, sizeof(urlReference), " <a href='%s/getData/sequence?hubUrl=%s;genome=%s;chrom=%s;start=%u;end=%u' target=_blank>JSON example sequence output: %s:%u-%u</a>", urlPrefix, hubTop->url, genome->name, chromName, chromSize/4, (chromSize/4)+128, chromName, chromSize/4, (chromSize/4)+128); hPrintf("<li>%s</li>\n", urlReference); } safef(urlReference, sizeof(urlReference), " <a href='%s/list/tracks?hubUrl=%s;genome=%s' target=_blank>JSON example list tracks output</a>", urlPrefix, hubTop->url, genome->name); hPrintf("<li>%s</li>\n", urlReference); hubInfo("organism", genome->organism); hubInfo("name", genome->name); hubInfo("description", genome->description); hubInfo("groups", genome->groups); hubInfo("defaultPos", genome->defaultPos); hubInfo("trackDbFile", genome->trackDbFile); hubAssemblySettings(hubTop, genome); - if (measureTiming || debug) + if (measureTiming) { long thisTime = clock1000(); hPrintf("<li><em>processing time %s: %ld millis</em></li>\n", genome->name, thisTime - lastTime); hPrintf("<hr>\n"); } if (timeOutReached()) break; } if (trackCounter->elCount) { hPrintf(" <li>total genome assembly count: %ld</li>\n", totalAssemblyCount); hPrintf(" <li>%ld total tracks counted, %d different track types:</li>\n", totalTracks, trackCounter->elCount); hPrintf(" <li><ul>\n"); struct hashEl *hel, *helList = hashElListHash(trackCounter); slSort(&helList, hashElCmpIntValDesc); @@ -1233,35 +1230,37 @@ hPrintf("<tr><th colspan=3>(example JSON list output: <a href='/list/publicHubs' target=_blank>Public hubs</a>, and <a href='/list/ucscGenomes' target=_blank>UCSC database genomes</a>)</th></tr>\n"); hPrintf("</table>\n"); hPrintf("</td></tr></table>\n"); /* how does debug carry forward ? */ // if (debug) // cgiMakeHiddenVar("debug", "1"); } static void doMiddle(struct cart *theCart) /* Set up globals and make web page */ { cart = theCart; -measureTiming = hPrintStatus() && isNotEmpty(cartOptionalString(cart, "measureTiming")); -measureTiming = TRUE; +measureTiming = isNotEmpty(cartOptionalString(cart, "measureTiming")); char *database = NULL; char *genome = NULL; +if (measureTiming) + startProcessTiming(); + cgiVarSet("ignoreCookie", "1"); getDbAndGenome(cart, &database, &genome, oldVars); initGenbankTableNames(database); initSupportedTypes(); initUrlPrefix(); trackLeavesOnly = cartUsualBoolean(cart, "trackLeavesOnly", trackLeavesOnly); jsonOutputArrays = cartUsualBoolean(cart, "jsonOutputArrays", jsonOutputArrays); /* global variable for all workers to honor this limit */ maxItemsOutput = cartUsualInt(cart, "maxItemsOutput", maxItemsOutput); if (maxItemsOutput < 0) /* can use -1 to indicate as much as allowed */ maxItemsOutput = maxItemLimit; /* maxItemsOutput of 0 might be useful, to be seen, let it go through */ @@ -1365,34 +1364,33 @@ hPrintf("</ul>\n"); } if (isEmpty(otherHubUrl)) otherHubUrl = urlInput; if (sameWord(RADIO_OTHERHUB, selectRadio)) /* requested other hub URL */ urlInput = otherHubUrl; if (commandError) { hPrintf("<h3>ERROR: no such command: '%s/%s' for endpoint '%s'</h3>", words[0], words[1], pathInfo); } long lastTime = clock1000(); struct trackHub *hub = errCatchTrackHubOpen(urlInput); -if (measureTiming || debug) +if (measureTiming) { long thisTime = clock1000(); - if (debug) hPrintf("<em>hub open time: %ld millis</em><br>\n", thisTime - lastTime); } hPrintf("<h3>Documentation: <a href='../../goldenPath/help/api.html'>API definitions/help</a>, and <a href='../../goldenPath/help/trackDb/trackDbHub.html' target=_blank>Track definition document</a> for definitions of track settings.</h3>\n"); if (debug) showCartDump(); hPrintf("<h2>Explore hub or database assemblies and tracks</h2>\n"); selectionForm(); /* these style mentions need to go into custom css file */ hPrintf("<div style='height:500px;overflow:scroll'>\n"); @@ -1411,54 +1409,54 @@ hubInfo("default db", hub->defaultDb); hubInfo("description url", hub->descriptionUrl); hubInfo("email", hub->email); if (debug) { hubInfo("version", hub->version); /* UCSC internal info */ hubInfo("level", hub->level); /* UCSC internal info */ } hPrintf("</ul>\n"); genomeList(hub); } if (timedOut) hPrintf("<h1>Reached time out %ld seconds</h1>", timeOutSeconds); -if (measureTiming || debug) +if (measureTiming) hPrintf("<em>Overall total time: %ld millis</em><br>\n", clock1000() - enteredMainTime); hPrintf("</div> <!-- end of text analysis output -->\n"); hPrintf("</div> <!-- end of surrounding border-->\n"); hPrintf("</div> <!-- end this page contents -->\n"); webIncludeFile("inc/jWestFooter.html"); webEndJWest(); // cartWebEnd(); } /* void doMiddle(struct cart *theCart) */ /* Null terminated list of CGI Variables we don't want to save * permanently. */ static char *excludeVars[] = {"Submit", "submit", "sourceSelected", "selectRadio", "ucscGenome", "publicHubs", "clade", NULL,}; int main(int argc, char *argv[]) /* Process command line. */ { enteredMainTime = clock1000(); cgiSpoof(&argc, argv); -measureTiming = TRUE; verboseTimeInit(); /* similar delay system as in DAS server */ botDelay = hgBotDelayTimeFrac(delayFraction); if (botDelay > 0) { if (botDelay > 2000) { hogExit(); return 0; } sleep1000(botDelay); } trackCounter = hashNew(0); cartEmptyShellNoContent(doMiddle, hUserCookie(), excludeVars, oldVars); +cgiExitTime("hubApi", enteredMainTime); return 0; }