046c975630f6576722e8cbfa843e68f6068d854e
hiram
  Thu Mar 21 15:06:42 2019 -0700
somewhat better for sample data URLs refs #18869

diff --git src/hg/hubApi/hubApi.c src/hg/hubApi/hubApi.c
index 76ff73b..e03035b 100644
--- src/hg/hubApi/hubApi.c
+++ src/hg/hubApi/hubApi.c
@@ -3,31 +3,35 @@
 
 /*
 +------------------+------------------+------+-----+---------+-------+
 | Field            | Type             | Null | Key | Default | Extra |
 +------------------+------------------+------+-----+---------+-------+
 | hubUrl           | longblob         | NO   | PRI | NULL    |       |
 | shortLabel       | varchar(255)     | NO   |     | NULL    |       |
 | longLabel        | varchar(255)     | NO   |     | NULL    |       |
 | registrationTime | varchar(255)     | NO   |     | NULL    |       |
 | dbCount          | int(10) unsigned | NO   |     | NULL    |       |
 | dbList           | blob             | YES  |     | NULL    |       |
 | descriptionUrl   | longblob         | YES  |     | NULL    |       |
 +------------------+------------------+------+-----+---------+-------+
 */
 
-/* Global Variables */
+/* Global Variables for all modules */
+
+int maxItemsOutput = 1000;	/* can be set in URL maxItemsOutput=N */
+
+/* Global only to this one source file */
 static boolean debug = FALSE;	/* can be set in URL debug=1 */
 static struct cart *cart;             /* CGI and other variables */
 static struct hash *oldVars = NULL;
 static struct hash *trackCounter = NULL;
 static long totalTracks = 0;
 static boolean measureTiming = FALSE;	/* set by CGI parameters */
 static boolean allTrackSettings = FALSE;	/* checkbox setting */
 static char **shortLabels = NULL;	/* public hub short labels in array */
 // struct hubPublic *publicHubList = NULL;
 static int publicHubCount = 0;
 static char *defaultHub = "Plants";
 static char *defaultDb = "ce11";
 static long enteredMainTime = 0;	/* will become = clock1000() on entry */
 		/* to allow calculation of when to bail out, taking too long */
 static long timeOutSeconds = 100;
@@ -43,40 +47,40 @@
 slAddHead(&supportedTypes, el);
 el = newSlName("wig");
 slAddHead(&supportedTypes, el);
 el = newSlName("broadPeak");
 slAddHead(&supportedTypes, el);
 el = newSlName("narrowPeak");
 slAddHead(&supportedTypes, el);
 el = newSlName("bigBed");
 slAddHead(&supportedTypes, el);
 el = newSlName("bigWig");
 slAddHead(&supportedTypes, el);
 el = newSlName("bigNarrowPeak");
 slAddHead(&supportedTypes, el);
 el = newSlName("bigGenePred");
 slAddHead(&supportedTypes, el);
-el = newSlName("bigPsl");
-slAddHead(&supportedTypes, el);
-el = newSlName("bigBarChart");
-slAddHead(&supportedTypes, el);
-el = newSlName("bigInteract");
-slAddHead(&supportedTypes, el);
-el = newSlName("bigMaf");
-slAddHead(&supportedTypes, el);
-el = newSlName("bigChain");
-slAddHead(&supportedTypes, el);
+// el = newSlName("bigPsl");
+// slAddHead(&supportedTypes, el);
+// el = newSlName("bigBarChart");
+// slAddHead(&supportedTypes, el);
+// el = newSlName("bigInteract");
+// slAddHead(&supportedTypes, el);
+// el = newSlName("bigMaf");
+// slAddHead(&supportedTypes, el);
+// el = newSlName("bigChain");
+// slAddHead(&supportedTypes, el);
 slNameSort(&supportedTypes);
 }
 
 static boolean isSupportedType(char *type)
 /* is given type in the supportedTypes list ? */
 {
 boolean ret = FALSE;
 if (startsWith("wigMaf", type))	/* not wigMaf at this time */
     return ret;
 struct slName *el;
 for (el = supportedTypes; el; el = el->next)
     {
     if (startsWith(el->name, type))
 	{
 	ret = TRUE;
@@ -188,75 +192,93 @@
     hashIncInt(countTracks, "superTrack child");
     hashIncInt(countTracks, stripType);
     hashIncInt(countTracks, "track count");
     }
 else if (isEmpty(tdb->type))
     hashIncInt(countTracks, "no type specified");
 else
     {
     hashIncInt(countTracks, stripType);
     hashIncInt(countTracks, "track count");
     }
 freeMem(stripType);
 // showCounts(countTracks);
 }
 
-static void sampleUrl(char *db, struct trackDb *tdb, char *chrom, long long chromSize)
+static void sampleUrl(struct trackHub *hub, char *db, struct trackDb *tdb, char *chrom, long long chromSize)
 /* print out a sample getData URL */
 {
 unsigned start = chromSize / 4;
 unsigned end = start + 10000;
 if (end > chromSize)
     end = chromSize;
+char *genome = NULL;
+if (hub)
+    genome = hub->genomeList->name;
 
 if (db)
     {
+    if (hub)
+	{
+	if (tdb->parent)
+	    hPrintf("<li>%s : %s subtrack of parent: %s <a href='%s/getData/track?hubUrl=%s&amp;genome=%s&amp;chrom=%s&amp;track=%s&amp;start=%u&amp;end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, tdb->parent->track, urlPrefix, hub->url, genome, chrom, tdb->track, start, end );
+	else
+	    hPrintf("<li>%s : %s <a href='%s/getData/track?hubUrl=%s&amp;genome=%s&amp;chrom=%s&amp;track=%s&amp;start=%u&amp;end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, urlPrefix, hub->url, genome, chrom, tdb->track, start, end );
+	}
+    else
+	{
 	if (tdb->parent)
 	    hPrintf("<li>%s : %s subtrack of parent: %s <a href='%s/getData/track?db=%s&amp;chrom=%s&amp;track=%s&amp;start=%u&amp;end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, tdb->parent->track, urlPrefix, db, chrom, tdb->track, start, end );
 	else
 	    hPrintf("<li>%s : %s <a href='%s/getData/track?db=%s&amp;chrom=%s&amp;track=%s&amp;start=%u&amp;end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, urlPrefix, db, chrom, tdb->track, start, end );
 	}
+    }
+else if (hub)
+    {
+    if (tdb->parent)
+	hPrintf("<li>%s : %s subtrack of parent: %s <a href='%s/getData/track?hubUrl=%s&amp;genome=%s&amp;chrom=%s&amp;track=%s&amp;start=%u&amp;end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, tdb->parent->track, urlPrefix, hub->url, genome, chrom, tdb->track, start, end );
     else
-    hPrintf("<li>%s : %s not db track</li>\n", tdb->track, tdb->type);
+	hPrintf("<li>%s : %s <a href='%s/getData/track?hubUrl=%s&amp;genome=%s&amp;chrom=%s&amp;track=%s&amp;start=%u&amp;end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, urlPrefix, hub->url, genome, chrom, tdb->track, start, end );
+    }
+else
+    hPrintf("<li>%s : %s not db hub track ?</li>\n", tdb->track, tdb->type);
 }
 
-static void showSubTracks(char *db, struct trackDb *tdb, struct hash *countTracks,
+static void showSubTracks(struct trackHub *hub, char *db, struct trackDb *tdb, struct hash *countTracks,
     char *chromName, long long chromSize)
 /* tdb has subtracks, show only subTracks, no details */
 {
 hPrintf("    <li><ul>\n");
+if (debug)
     hPrintf("    <li>subtracks for '%s' db: '%s'</li>\n", tdb->track, db);
 if (tdb->subtracks)
     {
     struct trackDb *tdbEl = NULL;
     for (tdbEl = tdb->subtracks; tdbEl; tdbEl = tdbEl->next)
 	{
         if (tdbIsCompositeView(tdbEl))
-	    hPrintf("<li>%s : %s : composite YYY view of parent: %s</li>\n", tdbEl->track, tdbEl->type, tdbEl->parent->track);
+	    hPrintf("<li>%s : %s : composite view of parent: %s</li>\n", tdbEl->track, tdbEl->type, tdbEl->parent->track);
 	else
 	    {
 	    if (isSupportedType(tdbEl->type))
-		{
-		sampleUrl(db, tdbEl, chromName, chromSize);
-//            hPrintf("<li>%s : %s : subtrack of parent: %s (with sample)</li>\n", tdbEl->track, tdbEl->type, tdbEl->parent->track, getData);
-		}
+		sampleUrl(hub, db, tdbEl, chromName, chromSize);
 	    else
 		hPrintf("<li>%s : %s : subtrack of parent: %s</li>\n", tdbEl->track, tdbEl->type, tdbEl->parent->track);
 	    }
 	hashCountTrack(tdbEl, countTracks);
         if (tdbEl->subtracks)
-	    showSubTracks(db, tdbEl, countTracks, chromName, chromSize);
+	    showSubTracks(hub, db, tdbEl, countTracks, chromName, chromSize);
 	}
     }
 hPrintf("    </ul></li>\n");
 }
 
 static void trackSettings(struct trackDb *tdb, struct hash *countTracks)
 /* process the settingsHash for a trackDb, recursive when subtracks */
 {
 hPrintf("    <li><ul>\n");
 // if (tdb->children)  haven't yet seen a track with children ?
 //   hPrintf("    <li>%s: has children</li>\n", tdb->track);
 // else
 //   hPrintf("    <li>%s: NO children</li>\n", tdb->track);
 struct hashEl *hel;
 struct hashCookie hc = hashFirst(tdb->settingsHash);
@@ -407,190 +429,175 @@
 return retVal;
 }	/* static int bbiBriefMeasure() */
 
 static void hubSampleUrl(struct trackHub *hub, struct trackDb *tdb,
     long chromCount, long itemCount, char *chromName, unsigned chromSize,
       char *genome)
 {
 unsigned start = chromSize / 4;
 unsigned end = start + 10000;
 if (end > chromSize)
     end = chromSize;
 
 if (isSupportedType(tdb->type))
     {
         if (startsWithWord("bigBed", tdb->type))
-            hPrintf("    <li>%s : %s : %ld chroms : %ld item count <a href='%s/getData/track?hubUrl=%s&amp;genome=%s&amp;track=%s&amp;chrom=%s&amp;start=%u&amp;end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, chromCount, itemCount, urlPrefix, hub->url, genome, tdb->track, chromName, start, end);
+            hPrintf("    <li>%s : %s : %ld chroms : %ld item count <a href='%s/getData/track?hubUrl=%s&amp;genome=%s&amp;track=%s&amp;chrom=%s&amp;start=%u&amp;end=%u' target=_blank>(Xsample getData)</a></li>\n", tdb->track, tdb->type, chromCount, itemCount, urlPrefix, hub->url, genome, tdb->track, chromName, start, end);
         else if (startsWithWord("bigWig", tdb->type))
-            hPrintf("    <li>%s : %s : %ld chroms : %ld bases covered <a href='%s/getData/track?hubUrl=%s&amp;genome=%s&amp;track=%s&amp;chrom=%s&amp;start=%u&amp;end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, chromCount, itemCount, urlPrefix, hub->url, genome, tdb->track, chromName, start, end);
+            hPrintf("    <li>%s : %s : %ld chroms : %ld bases covered <a href='%s/getData/track?hubUrl=%s&amp;genome=%s&amp;track=%s&amp;chrom=%s&amp;start=%u&amp;end=%u' target=_blank>(Xsample getData)</a></li>\n", tdb->track, tdb->type, chromCount, itemCount, urlPrefix, hub->url, genome, tdb->track, chromName, start, end);
         else
-            hPrintf("    <li>%s : %s : %ld chroms : %ld count <<a href='%s/getData/track?hubUrl=%s&amp;genome=%s&amp;track=%s&amp;chrom=%s&amp;start=%u&amp;end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, chromCount, itemCount, urlPrefix, hub->url, genome, tdb->track, chromName, start, end);
+            hPrintf("    <li>%s : %s : %ld chroms : %ld count <<a href='%s/getData/track?hubUrl=%s&amp;genome=%s&amp;track=%s&amp;chrom=%s&amp;start=%u&amp;end=%u' target=_blank>(Xsample getData)</a></li>\n", tdb->track, tdb->type, chromCount, itemCount, urlPrefix, hub->url, genome, tdb->track, chromName, start, end);
     }
 else
     {
         if (startsWithWord("bigBed", tdb->type))
             hPrintf("    <li>%s : %s : %ld chroms : %ld item count</li>\n", tdb->track, tdb->type, chromCount, itemCount);
         else if (startsWithWord("bigWig", tdb->type))
             hPrintf("    <li>%s : %s : %ld chroms : %ld bases covered</li>\n", tdb->track, tdb->type, chromCount, itemCount);
         else
             hPrintf("    <li>%s : %s : %ld chroms : %ld count</li>\n", tdb->track, tdb->type, chromCount, itemCount);
     }
 }
 
 static void hubCountOneTdb(struct trackHub *hub, char *db, struct trackDb *tdb,
     char *bigDataIndex, struct hash *countTracks, char *chromName,
     unsigned chromSize, char *genome)
 {
 char *bigDataUrl = trackDbSetting(tdb, "bigDataUrl");
 // char *compositeTrack = trackDbSetting(tdb, "compositeTrack");
 boolean compositeContainer = tdbIsComposite(tdb);
 boolean compositeView = tdbIsCompositeView(tdb);
 // char *superTrack = trackDbSetting(tdb, "superTrack");
 boolean superChild = tdbIsSuperTrackChild(tdb);
 boolean depthSearch = cartUsualBoolean(cart, "depthSearch", FALSE);
 hashCountTrack(tdb, countTracks);
 
-if (depthSearch && bigDataUrl)
-    {
 char *longName = NULL;
 unsigned longSize = 0;
 long chromCount = 0;
 long itemCount = 0;
+
+if (! (compositeContainer || compositeView) )
+    {
     struct dyString *errors = newDyString(1024);
     int retVal = bbiBriefMeasure(tdb->type, bigDataUrl, bigDataIndex, &chromCount, &itemCount, errors, &longName, &longSize);
     if (retVal)
-        {
         hPrintf("    <li>%s : %s : <font color='red'>ERROR: %s</font></li>\n", tdb->track, tdb->type, errors->string);
     }
-    else
+
+if (depthSearch && bigDataUrl)
+    {
+    if (isSupportedType(tdb->type))
 	{
 	if (chromSize > 0)
 	    hubSampleUrl(hub, tdb, chromCount, itemCount, chromName, chromSize, genome);
 	else
 	    hubSampleUrl(hub, tdb, chromCount, itemCount, longName, longSize, genome);
 	}
     }
 else
     {
     if (compositeContainer)
         hPrintf("    <li>%s : %s : composite track container has %d subtracks</li>\n", tdb->track, tdb->type, slCount(tdb->subtracks));
     else if (compositeView)
-        hPrintf("    <li>%s : %s : XXX composite view of parent: %s</li>\n", tdb->track, tdb->type, tdb->parent->track);
+        hPrintf("    <li>%s : %s : composite view of parent: %s</li>\n", tdb->track, tdb->type, tdb->parent->track);
     else if (superChild)
-        hPrintf("    <li>%s : %s : superTrack child of parent: %s</li>\n", tdb->track, tdb->type, tdb->parent->track);
+        hPrintf("    <li>%s : %s : superTrack child of parent: %s (sample getData)</li>\n", tdb->track, tdb->type, tdb->parent->track);
     else if (! depthSearch && bigDataUrl)
         hPrintf("    <li>%s : %s : %s</li>\n", tdb->track, tdb->type, bigDataUrl);
     else
 	{
         if (isSupportedType(tdb->type))
-            sampleUrl(db, tdb, chromName, chromSize);
+	    {
+	    if (chromSize > 0)
+		hubSampleUrl(hub, tdb, chromCount, itemCount, chromName, chromSize, genome);
+	    else
+		hubSampleUrl(hub, tdb, chromCount, itemCount, longName, longSize, genome);
+	    }
 	else
 	    hPrintf("    <li>%s : %s</li>\n", tdb->track, tdb->type);
         }
     }
 if (allTrackSettings)
     {
     hPrintf("    <li><ul>\n");
     trackSettings(tdb, countTracks); /* show all settings */
     hPrintf("    </ul></li>\n");
     }
 else if (tdb->subtracks)
     {
-    showSubTracks(db, tdb, countTracks, chromName, chromSize);
+    showSubTracks(hub, db, tdb, countTracks, chromName, chromSize);
     }
 return;
 }	/*	static void hubCountOneTdb(char *db, struct trackDb *tdb,
 	 *	char *bigDataIndex, struct hash *countTracks,
 	 *	char *chromName, long long chromSize)
 	 */
 
 
-static void countOneTdb(struct trackHub *hub, char *db, struct trackDb *tdb,
-    char *bigDataIndex, struct hash *countTracks, char *chromName,
-    unsigned chromSize, char *genome)
+static void countOneTdb(char *db, struct trackDb *tdb,
+    struct hash *countTracks, char *chromName, unsigned chromSize)
+/* for this tdb in this db, count it up and provide a sample */
 {
 char *bigDataUrl = trackDbSetting(tdb, "bigDataUrl");
 // char *compositeTrack = trackDbSetting(tdb, "compositeTrack");
 boolean compositeContainer = tdbIsComposite(tdb);
 boolean compositeView = tdbIsCompositeView(tdb);
 // char *superTrack = trackDbSetting(tdb, "superTrack");
 boolean superChild = tdbIsSuperTrackChild(tdb);
 boolean depthSearch = cartUsualBoolean(cart, "depthSearch", FALSE);
 hashCountTrack(tdb, countTracks);
 
-if (depthSearch && bigDataUrl)
-    {
-    char *longName = NULL;
-    unsigned longSize = 0;
-    long chromCount = 0;
-    long itemCount = 0;
-    struct dyString *errors = newDyString(1024);
-    int retVal = bbiBriefMeasure(tdb->type, bigDataUrl, bigDataIndex, &chromCount, &itemCount, errors, &longName, &longSize);
-    if (retVal)
-        {
-            hPrintf("    <li>%s : %s : <font color='red'>ERROR: %s</font></li>\n", tdb->track, tdb->type, errors->string);
-        }
-    else
-	{
-	if (chromSize > 0)
-	    hubSampleUrl(hub, tdb, chromCount, itemCount, chromName, chromSize, genome);
-        else
-	    hubSampleUrl(hub, tdb, chromCount, itemCount, longName, longSize, genome);
-	}
-    }
-else
-    {
 if (compositeContainer)
     hPrintf("    <li>%s : %s : composite track container has %d subtracks</li>\n", tdb->track, tdb->type, slCount(tdb->subtracks));
 else if (compositeView)
-        hPrintf("    <li>%s : %s : XXX composite view of parent: %s</li>\n", tdb->track, tdb->type, tdb->parent->track);
+    hPrintf("    <li>%s : %s : composite view of parent: %s</li>\n", tdb->track, tdb->type, tdb->parent->track);
 else if (superChild)
     hPrintf("    <li>%s : %s : superTrack child of parent: %s</li>\n", tdb->track, tdb->type, tdb->parent->track);
 else if (! depthSearch && bigDataUrl)
     hPrintf("    <li>%s : %s : %s</li>\n", tdb->track, tdb->type, bigDataUrl);
 else
     {
     if (isSupportedType(tdb->type))
-            sampleUrl(db, tdb, chromName, chromSize);
+        sampleUrl(NULL, db, tdb, chromName, chromSize);
     else
         hPrintf("    <li>%s : %s</li>\n", tdb->track, tdb->type);
     }
-    }
+
 if (allTrackSettings)
     {
     hPrintf("    <li><ul>\n");
     trackSettings(tdb, countTracks); /* show all settings */
     hPrintf("    </ul></li>\n");
     }
 else if (tdb->subtracks)
     {
-    showSubTracks(db, tdb, countTracks, chromName, chromSize);
+    showSubTracks(NULL, db, tdb, countTracks, chromName, chromSize);
     }
 return;
 }	/*	static void countOneTdb(char *db, struct trackDb *tdb,
-	 *	char *bigDataIndex, struct hash *countTracks,
-	 *	char *chromName, long long chromSize)
+	 *	struct hash *countTracks, char *chromName,
+	 *	unsigned chromSize)
 	 */
 
 static unsigned largestChrom(char *db, char **nameReturn)
 /* return the length and get the chrom name for the largest chrom
  * from chromInfo table.  For use is sample getData URLs
  */
 {
 char query[1024];
 struct sqlConnection *conn = hAllocConn(db);
-// Build a query to select the hubUrl for the given shortLabel
 sqlSafef(query, sizeof(query), "select chrom,size from chromInfo order by size desc limit 1");
 struct sqlResult *sr = sqlGetResult(conn, query);
 char **row = sqlNextRow(sr);
 unsigned length = 0;
 if (row)
    {
    *nameReturn = cloneString(row[0]);
    length = sqlLongLong(row[1]);
    }
 sqlFreeResult(&sr);
 hFreeConn(&conn);
 return length;
 }
 
 static void hubTrackList(struct trackHub *hub, struct trackDb *topTrackDb, struct trackHubGenome *genome)
@@ -812,31 +819,31 @@
 }
 
 static void tracksForUcscDb(char *db)
 /* scan the specified database for all tracks */
 {
 struct hash *countTracks = hashNew(0);
 char *chromName = NULL;
 unsigned chromSize = largestChrom(db, &chromName);
 hPrintf("<h4>Tracks in UCSC genome: '%s', longest chrom: %s:%u</h4>\n", db, chromName, chromSize);
 struct trackDb *tdbList = obtainTdb(NULL, db);
 struct trackDb *tdb;
 hPrintf("<ul>\n");
 hPrintf("<li>%s:%u</li>\n", chromName, chromSize);
 for (tdb = tdbList; tdb != NULL; tdb = tdb->next )
     {
-    countOneTdb(NULL, db, tdb, NULL, countTracks, chromName, chromSize, NULL);
+    countOneTdb(db, tdb, countTracks, chromName, chromSize);
     if (timeOutReached())
 	break;
     }
 int trackCount = ptToInt(hashFindVal(countTracks, "track count"));
 /* elCount - 1 since the 'track count' element isn't a track */
 hPrintf("    <li>%d total tracks counted, %d different track types</li>\n", trackCount, countTracks->elCount - 1);
 if (countTracks->elCount)
     {
     hPrintf("        <ol>\n");
     struct hashEl *hel, *helList = hashElListHash(countTracks);
     slSort(&helList, hashElCmpIntValDesc);
     for (hel = helList; hel; hel = hel->next)
 	{
 	if (sameOk("track count", hel->name))
 	    continue;
@@ -962,30 +969,33 @@
 /* Set up globals and make web page */
 {
 cart = theCart;
 measureTiming = hPrintStatus() && isNotEmpty(cartOptionalString(cart, "measureTiming"));
 measureTiming = TRUE;
 char *database = NULL;
 char *genome = NULL;
 
 cgiVarSet("ignoreCookie", "1");
 
 getDbAndGenome(cart, &database, &genome, oldVars);
 initGenbankTableNames(database);
 initSupportedTypes();
 initUrlPrefix();
 
+/* global variable for all workers to honor this limit */
+maxItemsOutput = cartUsualInt(cart, "maxItemsOutput", maxItemsOutput);
+
 debug = cartUsualBoolean(cart, "debug", debug);
 int timeout = cartUsualInt(cart, "udcTimeout", 300);
 if (udcCacheTimeout() < timeout)
     udcSetCacheTimeout(timeout);
 knetUdcInstall();
 
 char *pathInfo = getenv("PATH_INFO");
 /* nothing on incoming path, then display the WEB page instead */
 if (sameOk("/",pathInfo))
     pathInfo = NULL;
 
 boolean commandError = FALSE;
 /*expect no more than MAX_PATH_INFO number of words*/
 char *words[MAX_PATH_INFO];