0e092ec31d4d48d6a85f4e68d081bd92d88a92b7
hiram
  Wed Apr 17 14:18:30 2019 -0700
correct array output better timing and column header information refs #18869

diff --git src/hg/hubApi/hubApi.c src/hg/hubApi/hubApi.c
index f6db268..6abbcd2 100644
--- src/hg/hubApi/hubApi.c
+++ src/hg/hubApi/hubApi.c
@@ -21,36 +21,37 @@
 
 int maxItemsOutput = 1000;	/* can be set in URL maxItemsOutput=N */
 static int maxItemLimit = 1000000;   /* maximum of 1,000,000 items returned */
 /* for debugging purpose, current bot delay value */
 int botDelay = 0;
 boolean debug = FALSE;	/* can be set in URL debug=1, to turn off: debug=0 */
 #define delayFraction	0.03
 
 /* default is to list all trackDb entries, composite containers too.
  * This option will limit to only the actual track entries with data
  */
 boolean trackLeavesOnly = FALSE;  /* set by CGI parameter 'trackLeavesOnly' */
 /* this selects output type 'arrays', where the default type is: objects */
 boolean jsonOutputArrays = FALSE; /* set by CGI parameter 'jsonOutputArrays' */
 
+boolean measureTiming = FALSE;	/* set by CGI parameters */
+
 /* Global only to this one source file */
 static struct cart *cart;             /* CGI and other variables */
 static struct hash *oldVars = NULL;
 static struct hash *trackCounter = NULL;
 static long totalTracks = 0;
-static boolean measureTiming = FALSE;	/* set by CGI parameters */
 static boolean allTrackSettings = FALSE;	/* checkbox setting */
 static char **shortLabels = NULL;	/* public hub short labels in array */
 static int publicHubCount = 0;
 static char *defaultHub = "Plants";
 static char *defaultDb = "ce11";
 static long enteredMainTime = 0;	/* will become = clock1000() on entry */
 		/* to allow calculation of when to bail out, taking too long */
 static long timeOutSeconds = 100;
 static boolean timedOut = FALSE;
 static char *urlPrefix = "";	/* initalized to support self references */
 
 	/* supportedTypes will be initialized to a known supported set */
 static struct slName *supportedTypes = NULL;
 
 
@@ -226,54 +227,54 @@
 if (hub)
     genome = hub->genomeList->name;
 
 struct dyString *extraDyFlags = newDyString(128);
 if (debug)
     dyStringAppend(extraDyFlags, ";debug=1");
 if (jsonOutputArrays)
     dyStringAppend(extraDyFlags, ";jsonOutputArrays=1");
 char *extraFlags = dyStringCannibalize(&extraDyFlags);
 
 if (db)
     {
     if (hub)
 	{
 	char urlReference[2048];
-	safef(urlReference,	sizeof(urlReference), " <a href='%s/getData/track?hubUrl=%s;genome=%s;track=%s;chrom=%s;start=%u;end=%u%s' target=_blank>(sample data)%s</a>\n", urlPrefix, hub->url, genome, tdb->track, chrom, start, end, extraFlags, errorPrint);
+	safef(urlReference,	sizeof(urlReference), " <a href='%s/getData/track?hubUrl=%s;genome=%s;track=%s;chrom=%s;start=%u;end=%u;maxItemsOutput=5%s' target=_blank>(sample data)%s</a>\n", urlPrefix, hub->url, genome, tdb->track, chrom, start, end, extraFlags, errorPrint);
 
 	if (tdb->parent)
 	    hPrintf("<li><b>%s</b>: %s subtrack of parent: %s%s</li>\n", tdb->track, tdb->type, tdb->parent->track, urlReference);
 	else
 	    hPrintf("<li><b>%s</b>: %s%s</li>\n", tdb->track, tdb->type, urlReference);
 	}
     else
 	{
 	char urlReference[2048];
-	safef(urlReference, sizeof(urlReference), " <a href='%s/getData/track?db=%s;chrom=%s;track=%s;start=%u;end=%u%s' target=_blank>(sample data)%s</a>\n", urlPrefix, db, chrom, tdb->track, start, end, extraFlags, errorPrint);
+	safef(urlReference, sizeof(urlReference), " <a href='%s/getData/track?db=%s;chrom=%s;track=%s;start=%u;end=%u;maxItemsOutput=5%s' target=_blank>(sample data)%s</a>\n", urlPrefix, db, chrom, tdb->track, start, end, extraFlags, errorPrint);
 
 	if (superChild)
 	    hPrintf("<li><b>%s</b>: %s superTrack child of parent: %s%s</li>\n", tdb->track, tdb->type, tdb->parent->track, urlReference);
 	else if (tdb->parent)
 	    hPrintf("<li><b>%s</b>: %s subtrack of parent: %s%s</li>\n", tdb->track, tdb->type, tdb->parent->track, urlReference);
 	else
 	    hPrintf("<li><b>%s</b>: %s%s</li>\n", tdb->track, tdb->type, urlReference );
 	}
     }
 else if (hub)
     {
     char urlReference[2048];
-    safef(urlReference, sizeof(urlReference), " <a href='%s/getData/track?hubUrl=%s;genome=%s;track=%s;chrom=%s;start=%u;end=%u%s' target=_blank>(sample data)%s</a>\n", urlPrefix, hub->url, genome, tdb->track, chrom, start, end, extraFlags, errorPrint);
+    safef(urlReference, sizeof(urlReference), " <a href='%s/getData/track?hubUrl=%s;genome=%s;track=%s;chrom=%s;start=%u;end=%u;maxItemsOutput=5%s' target=_blank>(sample data)%s</a>\n", urlPrefix, hub->url, genome, tdb->track, chrom, start, end, extraFlags, errorPrint);
 
     if (tdb->parent)
 	hPrintf("<li><b>%s</b>: %s subtrack of parent: %s%s</li>\n", tdb->track, tdb->type, tdb->parent->track, urlReference);
     else
 	hPrintf("<li><b>%s</b>: %s%s</li>\n", tdb->track, tdb->type, urlReference);
     }
 else
     hPrintf("<li>%s : %s not db hub track ?</li>\n", tdb->track, tdb->type);
 }
 
 static void hubSampleUrl(struct trackHub *hub, struct trackDb *tdb,
     long chromCount, long itemCount, char *chromName, unsigned chromSize,
       char *genome, char *errorString)
 {
 unsigned start = chromSize / 4;
@@ -300,31 +301,31 @@
 char countsMessage[512];
 countsMessage[0] = 0;
 if (chromCount > 0 || itemCount > 0)
     {
     if (allowedBigBedType(tdb->type))
         safef(countsMessage, sizeof(countsMessage), " : %ld chroms : %ld item count ", chromCount, itemCount);
     else if (startsWithWord("bigWig", tdb->type))
         safef(countsMessage, sizeof(countsMessage), " : %ld chroms : %ld bases covered ", chromCount, itemCount);
     else
         safef(countsMessage, sizeof(countsMessage), " : %ld chroms : %ld count ", chromCount, itemCount);
     }
 
 if (isSupportedType(tdb->type))
     {
 	char urlReference[2048];
-	safef(urlReference, sizeof(urlReference), "<a href='%s/getData/track?hubUrl=%s;genome=%s;track=%s;chrom=%s;start=%u;end=%u%s' target=_blank>(sample data)%s</a>\n", urlPrefix, hub->url, genome, tdb->track, chromName, start, end, extraFlags, errorPrint);
+	safef(urlReference, sizeof(urlReference), "<a href='%s/getData/track?hubUrl=%s;genome=%s;track=%s;chrom=%s;start=%u;end=%u;maxItemsOutput=5%s' target=_blank>(sample data)%s</a>\n", urlPrefix, hub->url, genome, tdb->track, chromName, start, end, extraFlags, errorPrint);
 
 	if (allowedBigBedType(tdb->type))
             hPrintf("    <li><b>%s</b>: %s%s%s</li>\n", tdb->track, tdb->type, countsMessage, urlReference);
         else if (startsWithWord("bigWig", tdb->type))
             hPrintf("    <li><b>%s</b>: %s%s%s</li>\n", tdb->track, tdb->type, countsMessage, urlReference);
         else
             hPrintf("    <li><b>%s</b>: %s%s%s</li>\n", tdb->track, tdb->type, countsMessage, urlReference);
     }
 else
     {
         if (allowedBigBedType(tdb->type))
             hPrintf("    <li><b>%s</b>: %s%s</li>\n", tdb->track, tdb->type, countsMessage);
         else if (startsWithWord("bigWig", tdb->type))
             hPrintf("    <li><b>%s</b>: %s%s</li>\n", tdb->track, tdb->type, countsMessage);
         else
@@ -531,53 +532,49 @@
 	    else
 		hPrintf("<li><b>%s</b>: %s : subtrack of parent: %s</li>\n", tdbEl->track, tdbEl->type, tdbEl->parent->track);
 	    }
 	hashCountTrack(tdbEl, countTracks);
         if (tdbEl->subtracks)
 	    showSubTracks(hub, db, tdbEl, countTracks, chromName, chromSize, errorString);
 	}
     }
 hPrintf("    </ul></li>\n");
 }
 
 static void trackSettings(struct trackDb *tdb, struct hash *countTracks)
 /* process the settingsHash for a trackDb, recursive when subtracks */
 {
 hPrintf("    <li><ul>\n");
-// if (tdb->children)  haven't yet seen a track with children ?
-//   hPrintf("    <li>%s: has children</li>\n", tdb->track);
-// else
-//   hPrintf("    <li>%s: NO children</li>\n", tdb->track);
 struct hashEl *hel;
 struct hashCookie hc = hashFirst(tdb->settingsHash);
 while ((hel = hashNext(&hc)) != NULL)
     {
     if (sameWord("track", hel->name))
 	continue;	// already output in header
     if (isEmpty((char *)hel->val))
 	hPrintf("    <li><b>%s</b>: &lt;empty&gt;</li>\n", hel->name);
     else
 	hPrintf("    <li><b>%s</b>: '%s'</li>\n", hel->name, (char *)hel->val);
     }
 if (tdb->subtracks)
     {
     struct trackDb *tdbEl = NULL;
     if (debug)
 	hPrintf("   <li>has %d subtrack(s)</li>\n", slCount(tdb->subtracks));
     for (tdbEl = tdb->subtracks; tdbEl; tdbEl = tdbEl->next)
 	{
-        hPrintf("<li>subtrack: %s of parent: %s : type: '%s'</li>\n", tdbEl->track, tdbEl->parent->track, tdbEl->type);
+        hPrintf("<li>subtrack: %s of parent: %s : type: '%s' (TBD: sample data)</li>\n", tdbEl->track, tdbEl->parent->track, tdbEl->type);
 	hashCountTrack(tdbEl, countTracks);
 	trackSettings(tdbEl, countTracks);
 	}
     }
 hPrintf("    </ul></li>\n");
 }
 
 static void hubCountOneTdb(struct trackHub *hub, char *db, struct trackDb *tdb,
     char *bigDataIndex, struct hash *countTracks, char *chromName,
     unsigned chromSize, char *genome)
 {
 char *bigDataUrl = trackDbSetting(tdb, "bigDataUrl");
 boolean compositeContainer = tdbIsComposite(tdb);
 boolean compositeView = tdbIsCompositeView(tdb);
 boolean superChild = tdbIsSuperTrackChild(tdb);
@@ -881,31 +878,31 @@
 	char sizeString[64];
 	sprintLongWithCommas(sizeString, chromSize);
 	hPrintf("<li><b>Sequence count</b> %d, <b>largest</b>: %s at %s bases</li>\n", slCount(ci), chromName, sizeString);
        safef(urlReference, sizeof(urlReference), " <a href='%s/getData/sequence?hubUrl=%s;genome=%s;chrom=%s;start=%u;end=%u' target=_blank>JSON example sequence output: %s:%u-%u</a>", urlPrefix, hubTop->url, genome->name, chromName, chromSize/4, (chromSize/4)+128, chromName, chromSize/4, (chromSize/4)+128);
         hPrintf("<li>%s</li>\n", urlReference);
 	}
     safef(urlReference, sizeof(urlReference), " <a href='%s/list/tracks?hubUrl=%s;genome=%s' target=_blank>JSON example list tracks output</a>", urlPrefix, hubTop->url, genome->name);
     hPrintf("<li>%s</li>\n", urlReference);
     hubInfo("organism", genome->organism);
     hubInfo("name", genome->name);
     hubInfo("description", genome->description);
     hubInfo("groups", genome->groups);
     hubInfo("defaultPos", genome->defaultPos);
     hubInfo("trackDbFile", genome->trackDbFile);
     hubAssemblySettings(hubTop, genome);
-    if (measureTiming || debug)
+    if (measureTiming)
 	{
 	long thisTime = clock1000();
 	hPrintf("<li><em>processing time %s: %ld millis</em></li>\n", genome->name, thisTime - lastTime);
 	hPrintf("<hr>\n");
         }
     if (timeOutReached())
 	break;
     }
 if (trackCounter->elCount)
     {
     hPrintf("    <li>total genome assembly count: %ld</li>\n", totalAssemblyCount);
     hPrintf("    <li>%ld total tracks counted, %d different track types:</li>\n", totalTracks, trackCounter->elCount);
     hPrintf("    <li><ul>\n");
     struct hashEl *hel, *helList = hashElListHash(trackCounter);
     slSort(&helList, hashElCmpIntValDesc);
@@ -1233,35 +1230,37 @@
 
 hPrintf("<tr><th colspan=3>(example JSON list output: <a href='/list/publicHubs' target=_blank>Public hubs</a>, and <a href='/list/ucscGenomes' target=_blank>UCSC database genomes</a>)</th></tr>\n");
 
 hPrintf("</table>\n");
 hPrintf("</td></tr></table>\n");
 
 /* how does debug carry forward ? */
 // if (debug)
 //    cgiMakeHiddenVar("debug", "1");
 }
 
 static void doMiddle(struct cart *theCart)
 /* Set up globals and make web page */
 {
 cart = theCart;
-measureTiming = hPrintStatus() && isNotEmpty(cartOptionalString(cart, "measureTiming"));
-measureTiming = TRUE;
+measureTiming = isNotEmpty(cartOptionalString(cart, "measureTiming"));
 char *database = NULL;
 char *genome = NULL;
 
+if (measureTiming)
+    startProcessTiming();
+
 cgiVarSet("ignoreCookie", "1");
 
 getDbAndGenome(cart, &database, &genome, oldVars);
 initGenbankTableNames(database);
 initSupportedTypes();
 initUrlPrefix();
 
 trackLeavesOnly = cartUsualBoolean(cart, "trackLeavesOnly", trackLeavesOnly);
 jsonOutputArrays = cartUsualBoolean(cart, "jsonOutputArrays", jsonOutputArrays);
 
 /* global variable for all workers to honor this limit */
 maxItemsOutput = cartUsualInt(cart, "maxItemsOutput", maxItemsOutput);
 if (maxItemsOutput < 0)	/* can use -1 to indicate as much as allowed */
     maxItemsOutput = maxItemLimit;
 /* maxItemsOutput of 0 might be useful, to be seen, let it go through */
@@ -1365,34 +1364,33 @@
     hPrintf("</ul>\n");
     }
 if (isEmpty(otherHubUrl))
     otherHubUrl = urlInput;
 
 if (sameWord(RADIO_OTHERHUB, selectRadio))	/* requested other hub URL */
     urlInput = otherHubUrl;
 
 if (commandError)
   {
   hPrintf("<h3>ERROR: no such command: '%s/%s' for endpoint '%s'</h3>", words[0], words[1], pathInfo);
   }
 
 long lastTime = clock1000();
 struct trackHub *hub = errCatchTrackHubOpen(urlInput);
-if (measureTiming || debug)
+if (measureTiming)
     {
     long thisTime = clock1000();
-    if (debug)
     hPrintf("<em>hub open time: %ld millis</em><br>\n", thisTime - lastTime);
     }
 
 hPrintf("<h3>Documentation: <a href='../../goldenPath/help/api.html'>API definitions/help</a>, and <a href='../../goldenPath/help/trackDb/trackDbHub.html' target=_blank>Track definition document</a> for definitions of track settings.</h3>\n");
 
 if (debug)
     showCartDump();
 
 hPrintf("<h2>Explore hub or database assemblies and tracks</h2>\n");
 
 selectionForm();
 
 /* these style mentions need to go into custom css file */
 hPrintf("<div style='height:500px;overflow:scroll'>\n");
 
@@ -1411,54 +1409,54 @@
     hubInfo("default db", hub->defaultDb);
     hubInfo("description url", hub->descriptionUrl);
     hubInfo("email", hub->email);
     if (debug)
 	{
 	hubInfo("version", hub->version);	/* UCSC internal info */
 	hubInfo("level", hub->level);		/* UCSC internal info */
 	}
     hPrintf("</ul>\n");
 
     genomeList(hub);
     }
 
 if (timedOut)
     hPrintf("<h1>Reached time out %ld seconds</h1>", timeOutSeconds);
-if (measureTiming || debug)
+if (measureTiming)
     hPrintf("<em>Overall total time: %ld millis</em><br>\n", clock1000() - enteredMainTime);
 
 hPrintf("</div> <!-- end of text analysis output -->\n");
 hPrintf("</div> <!-- end of surrounding border-->\n");
 hPrintf("</div> <!-- end this page contents -->\n");
 
 webIncludeFile("inc/jWestFooter.html");
 webEndJWest();
 // cartWebEnd();
 }	/*	void doMiddle(struct cart *theCart)	*/
 
 /* Null terminated list of CGI Variables we don't want to save
  * permanently. */
 static char *excludeVars[] = {"Submit", "submit", "sourceSelected", "selectRadio", "ucscGenome", "publicHubs", "clade", NULL,};
 
 int main(int argc, char *argv[])
 /* Process command line. */
 {
 enteredMainTime = clock1000();
 cgiSpoof(&argc, argv);
-measureTiming = TRUE;
 verboseTimeInit();
 /* similar delay system as in DAS server */
 botDelay = hgBotDelayTimeFrac(delayFraction);
 if (botDelay > 0)
     {
     if (botDelay > 2000)
         {
 	hogExit();
         return 0;
         }
     sleep1000(botDelay);
     }
 
 trackCounter = hashNew(0);
 cartEmptyShellNoContent(doMiddle, hUserCookie(), excludeVars, oldVars);
+cgiExitTime("hubApi", enteredMainTime);
 return 0;
 }