7f4579f5b69be16ca391924dce94d17599db2873 hiram Mon Feb 25 09:49:27 2019 -0800 counting up track types for UCSC db same as hub refs #18869 diff --git src/hg/hubApi/hubApi.c src/hg/hubApi/hubApi.c index e0cb06b..dd0eddd 100644 --- src/hg/hubApi/hubApi.c +++ src/hg/hubApi/hubApi.c @@ -92,44 +92,58 @@ ++publicHubCount; } return list; } static boolean timeOutReached() { long nowTime = clock1000(); timedOut = FALSE; if ((nowTime - enteredMainTime) > (1000 * timeOutSeconds)) timedOut= TRUE; return timedOut; } static void trackSettings(struct trackDb *tdb) -/* process the settingsHash for a track */ +/* process the settingsHash for a trackDb, recursive if subtracks */ { hPrintf(" <ul>\n"); +// if (tdb->children) haven't yet seen a track with children ? +// hPrintf(" <li>%s: has children</li>\n", tdb->track); +// else +// hPrintf(" <li>%s: NO children</li>\n", tdb->track); struct hashEl *hel; struct hashCookie hc = hashFirst(tdb->settingsHash); while ((hel = hashNext(&hc)) != NULL) { + if (sameWord("track", hel->name)) + continue; // already output in header if (isEmpty((char *)hel->val)) hPrintf(" <li>%s : <empty></li>\n", hel->name); else hPrintf(" <li>%s : '%s'</li>\n", hel->name, (char *)hel->val); } if (tdb->subtracks) - trackSettings(tdb->subtracks); + { + struct trackDb *tdbEl = tdb->subtracks; + hPrintf(" <li>has %d subtrack(s)</li>\n", slCount(tdb->subtracks)); + for (tdbEl = tdb->subtracks; tdbEl; tdbEl = tdbEl->next) + { + hPrintf("<li>subtrack: %s of parent: %s</li>\n", tdbEl->track, tdbEl->parent->track); + trackSettings(tdbEl); + } + } hPrintf(" </ul>\n"); } static int bbiBriefMeasure(char *type, char *bigDataUrl, char *bigDataIndex, long *chromCount, long *itemCount, struct dyString *errors) /* check a bigDataUrl to find chrom count and item count */ { int retVal = 0; *chromCount = 0; *itemCount = 0; struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) { if (startsWithWord("bigNarrowPeak", type) || startsWithWord("bigBed", type) || startsWithWord("bigGenePred", type) @@ -204,75 +218,48 @@ retVal = 1; } } errCatchEnd(errCatch); if (errCatch->gotError) { retVal = 1; dyStringPrintf(errors, "%s", errCatch->message->string); } errCatchFree(&errCatch); return retVal; } /* static int bbiBriefMeasure() */ -#ifdef NOT -static void cloneTdb(struct trackDb *source, struct trackDb *destination) -/* TBD: is there a cloneTdb() function somewhere else ? */ -{ -destination->track = cloneString(source->track); -destination->shortLabel = cloneString(source->shortLabel); -destination->type = cloneString(source->type); -destination->longLabel = cloneString(source->longLabel); -destination->visibility = source->visibility; -destination->priority = source->priority; -destination->colorR = source->colorR; -destination->colorG = source->colorG; -destination->colorB = source->colorB; -destination->altColorR = source->altColorR; -destination->altColorG = source->altColorG; -destination->altColorB = source->altColorB; -destination->useScore = source->useScore; -destination->private = source->private; -destination->url = cloneString(source->url); -destination->html = cloneString(source->html); -destination->grp = cloneString(source->grp); -destination->canPack = source->canPack; -destination->settings = cloneString(source->settings); -destination->settingsHash = source->settingsHash; -} -#endif - static void hubTrackList(struct trackDb *topTrackDb, struct trackHubGenome *genome) /* process the track list to show all tracks, return trackDb list */ { if (topTrackDb) { struct hash *countTracks = hashNew(0); hPrintf(" <ul>\n"); struct trackDb *tdb = NULL; for ( tdb = topTrackDb; tdb; tdb = tdb->next ) { char *bigDataUrl = trackDbSetting(tdb, "bigDataUrl"); char *compositeTrack = trackDbSetting(tdb, "compositeTrack"); char *superTrack = trackDbSetting(tdb, "superTrack"); boolean depthSearch = cartUsualBoolean(cart, "depthSearch", FALSE); if (compositeTrack) hashIncInt(countTracks, "composite container"); else if (superTrack) - hashIncInt(countTracks, "superTrack container"); + hashIncInt(countTracks, "superTrack child"); else if (isEmpty(tdb->type)) hashIncInt(countTracks, "no type specified"); else hashIncInt(countTracks, tdb->type); if (depthSearch && bigDataUrl) { char *bigDataIndex = NULL; char *relIdxUrl = trackDbSetting(topTrackDb, "bigDataIndex"); if (relIdxUrl != NULL) bigDataIndex = trackHubRelativeUrl(genome->trackDbFile, relIdxUrl); long chromCount = 0; long itemCount = 0; struct dyString *errors = newDyString(1024); int retVal = bbiBriefMeasure(tdb->type, bigDataUrl, bigDataIndex, &chromCount, &itemCount, errors); @@ -283,85 +270,89 @@ else { if (startsWithWord("bigBed", tdb->type)) hPrintf(" <li>%s : %s : %ld chroms : %ld item count</li>\n", tdb->track, tdb->type, chromCount, itemCount); else if (startsWithWord("bigWig", tdb->type)) hPrintf(" <li>%s : %s : %ld chroms : %ld bases covered</li>\n", tdb->track, tdb->type, chromCount, itemCount); else hPrintf(" <li>%s : %s : %ld chroms : %ld count</li>\n", tdb->track, tdb->type, chromCount, itemCount); } } else { if (compositeTrack) hPrintf(" <li>%s : %s : composite track container</li>\n", tdb->track, tdb->type); else if (superTrack) - hPrintf(" <li>%s : %s : superTrack container</li>\n", tdb->track, tdb->type); + hPrintf(" <li>%s : %s : superTrack child</li>\n", tdb->track, tdb->type); else if (! depthSearch) hPrintf(" <li>%s : %s : %s</li>\n", tdb->track, tdb->type, bigDataUrl); else hPrintf(" <li>%s : %s</li>\n", tdb->track, tdb->type); } - if (tdb->subtracks) - hPrintf(" <li>hasSubtracks : TRUE</li>\n"); - else - hPrintf(" <li>hasSubtracks : FALSE</li>\n"); if (allTrackSettings) + { + hPrintf(" <ul>\n"); trackSettings(tdb); /* show all settings */ + hPrintf(" </ul>\n"); + } if (timeOutReached()) break; } /* for ( tdb = topTrackDb; tdb; tdb = tdb->next ) */ hPrintf(" <li>%d different track types</li>\n", countTracks->elCount); if (countTracks->elCount) { hPrintf(" <ul>\n"); struct hashEl *hel; struct hashCookie hc = hashFirst(countTracks); while ((hel = hashNext(&hc)) != NULL) { int prevCount = ptToInt(hashFindVal(trackCounter, hel->name)); totalTracks += ptToInt(hel->val); hashReplace(trackCounter, hel->name, intToPt(prevCount + ptToInt(hel->val))); hPrintf(" <li>%d - %s</li>\n", ptToInt(hel->val), hel->name); } hPrintf(" </ul>\n"); } hPrintf(" </ul>\n"); } +else + hPrintf(" <li>no trackTopDb</li>\n"); } /* static struct trackDb *hubTrackList() */ -static struct trackDb * assemblySettings(struct trackHubGenome *genome) +static void assemblySettings(struct trackHubGenome *genome) /* display all the assembly 'settingsHash' */ { -struct trackDb *retTbd = NULL; +struct trackDb *tdb = trackHubTracksForGenome(genome->trackHub, genome); +tdb = trackDbLinkUpGenerations(tdb); + +int elCount = 0; hPrintf(" <ul>\n"); struct hashEl *hel; struct hashCookie hc = hashFirst(genome->settingsHash); while ((hel = hashNext(&hc)) != NULL) { + ++elCount; hPrintf(" <li>%s : %s</li>\n", hel->name, (char *)hel->val); if (sameWord("trackDb", hel->name)) /* examine the trackDb structure */ { - struct trackDb *tdb = trackHubTracksForGenome(genome->trackHub, genome); - retTbd = tdb; hubTrackList(tdb, genome); } if (timeOutReached()) break; } +hPrintf(" <li>elCount: %d</li>\n", elCount); hPrintf(" </ul>\n"); -return retTbd; } struct slName *genomeList(struct trackHub *hubTop, struct trackDb **dbTrackList, char *selectGenome) /* follow the pointers from the trackHub to trackHubGenome and around * in a circle from one to the other to find all hub resources * return slName list of the genomes in this track hub * optionally, return the trackList from this hub for the specified genome */ { struct slName *retList = NULL; long totalAssemblyCount = 0; struct trackHubGenome *genome = hubTop->genomeList; hPrintf("<h4>genome sequences (and tracks) present in this track hub</h4>\n"); @@ -373,34 +364,31 @@ { if ( differentStringNullOk(selectGenome, genome->name) ) continue; } ++totalAssemblyCount; struct slName *el = slNameNew(genome->name); slAddHead(&retList, el); if (genome->organism) { hPrintf("<li>%s - %s - %s</li>\n", genome->organism, genome->name, genome->description); } else { /* can there be a description when organism is empty ? */ hPrintf("<li>%s</li>\n", genome->name); } - if (dbTrackList) - *dbTrackList = assemblySettings(genome); - else - (void) assemblySettings(genome); + assemblySettings(genome); if (measureTiming) { long thisTime = clock1000(); hPrintf("<em>processing time %s: %ld millis</em><br>\n", genome->name, thisTime - lastTime); } if (timeOutReached()) break; } if (trackCounter->elCount) { hPrintf(" <li>total genome assembly count: %ld</li>\n", totalAssemblyCount); hPrintf(" <li>%ld total tracks counted, %d different track types:</li>\n", totalTracks, trackCounter->elCount); hPrintf(" <ul>\n"); struct hashEl *hel; struct hashCookie hc = hashFirst(trackCounter); @@ -488,94 +476,161 @@ if (wordCount < 2) apiErrAbort("unknown endpoint command: '/%s'", pathInfo); struct hashEl *hel = hashLookup(apiFunctionHash, words[0]); if (hel == NULL) apiErrAbort("no such command: '%s' for endpoint '/%s'", words[0], pathInfo); void (*apiFunction)(char **) = hel->val; // void (*apiFunction)(char **) = hashMustFindVal(apiFunctionHash, words[0]); (*apiFunction)(words); } /* static void apiFunctionSwitch(char *pathInfo) */ static void tracksForUcscDb(char * ucscDb) { +struct hash *countTracks = hashNew(0); +struct sqlConnection *conn = hAllocConn(ucscDb); hPrintf("<p>Tracks in UCSC genome: '%s'<br>\n", ucscDb); struct trackDb *tdbList = hTrackDb(ucscDb); struct trackDb *tdb; hPrintf("<ul>\n"); for (tdb = tdbList; tdb != NULL; tdb = tdb->next ) { - hPrintf("<li>%s</li>\n", tdb->track); + char *bigDataUrl = trackDbSetting(tdb, "bigDataUrl"); + char *compositeTrack = trackDbSetting(tdb, "compositeTrack"); + char *superTrack = trackDbSetting(tdb, "superTrack"); + boolean depthSearch = cartUsualBoolean(cart, "depthSearch", FALSE); + if (compositeTrack) + hashIncInt(countTracks, "composite container"); + else if (superTrack) + hashIncInt(countTracks, "superTrack child"); + else if (isEmpty(tdb->type)) + hashIncInt(countTracks, "no type specified"); + else + hashIncInt(countTracks, tdb->type); + if (depthSearch && bigDataUrl) + { + long chromCount = 0; + long itemCount = 0; + struct dyString *errors = newDyString(1024); + int retVal = bbiBriefMeasure(tdb->type, bigDataUrl, NULL, &chromCount, &itemCount, errors); + if (retVal) + { + hPrintf(" <li>%s : %s : <font color='red'>ERROR: %s</font></li>\n", tdb->track, tdb->type, errors->string); + } + else + { + if (startsWithWord("bigBed", tdb->type)) + hPrintf(" <li>%s : %s : %ld chroms : %ld item count</li>\n", tdb->track, tdb->type, chromCount, itemCount); + else if (startsWithWord("bigWig", tdb->type)) + hPrintf(" <li>%s : %s : %ld chroms : %ld bases covered</li>\n", tdb->track, tdb->type, chromCount, itemCount); + else + hPrintf(" <li>%s : %s : %ld chroms : %ld count</li>\n", tdb->track, tdb->type, chromCount, itemCount); + } + } + else + { + if (isEmpty(bigDataUrl)) + { + int rowCount = 0; + if (sqlTableExists(conn, tdb->track)) + { + rowCount = sqlRowCount(conn, tdb->track); + hPrintf("<li>%s : %s : %d item count</li>\n", tdb->track, tdb->type, rowCount); + } + else + { + if (compositeTrack) + hPrintf("<li>%s : %s : composite container </li>\n", tdb->track, tdb->type); + else + hPrintf("<li>%s : %s</li>\n", tdb->track, tdb->type); + } + } + else + hPrintf("<li>%s : %s : %s</li>\n", tdb->track,tdb->type,bigDataUrl); + } if (allTrackSettings) trackSettings(tdb); /* show all settings */ } +hPrintf(" <li>%d different track types</li>\n", countTracks->elCount); +if (countTracks->elCount) + { + hPrintf(" <ul>\n"); + struct hashEl *hel; + struct hashCookie hc = hashFirst(countTracks); + while ((hel = hashNext(&hc)) != NULL) + { + hPrintf(" <li>%d - %s</li>\n", ptToInt(hel->val), hel->name); + } hPrintf(" </ul>\n"); -hPrintf("</p>\n"); } +hPrintf("</ul>\n"); +hPrintf("</p>\n"); +hFreeConn(&conn); +} // static void tracksForUcscDb(char * ucscDb) static void showExamples(char *url, struct trackHubGenome *hubGenome, char *ucscDb) { hPrintf("<h2>Example URLs to return json data structures:</h2>\n"); hPrintf("<ol>\n"); hPrintf("<li><a href='/cgi-bin/hubApi/list/publicHubs' target=_blank>list public hubs</a> <em>/cgi-bin/hubApi/list/publicHubs</em></li>\n"); hPrintf("<li><a href='/cgi-bin/hubApi/list/ucscGenomes' target=_blank>list database genomes</a> <em>/cgi-bin/hubApi/list/ucscGenomes</em></li>\n"); hPrintf("<li><a href='/cgi-bin/hubApi/list/hubGenomes?hubUrl=%s' target=_blank>list genomes from specified hub</a> <em>/cgi-bin/hubApi/list/hubGenomes?hubUrl=%s</em></li>\n", url, url); -hPrintf("<li><a href='/cgi-bin/hubApi/list/tracks?hubUrl=%s&genome=%s' target=_blank>list tracks from specified hub and genome</a> <em>/cgi-bin/hubApi/list/tracks?hubUrl=%s&genome=%s</em></li>\n", url, hubGenome->name, url, hubGenome->name); +hPrintf("<li><a href='/cgi-bin/hubApi/list/tracks?hubUrl=%s&hubUrl=%s&genome=%s' target=_blank>list tracks from specified hub and genome</a> <em>/cgi-bin/hubApi/list/tracks?hubUrl=%s&genome=%s</em></li>\n", url, url, hubGenome->name, url, hubGenome->name); hPrintf("<li><a href='/cgi-bin/hubApi/list/tracks?db=%s' target=_blank>list tracks from specified UCSC database</a> <em>/cgi-bin/hubApi/list/tracks?db=%s</em></li>\n", ucscDb, ucscDb); hPrintf("<li><a href='/cgi-bin/hubApi/list/chromosomes?db=%s' target=_blank>list chromosomes from specified UCSC database</a> <em>/cgi-bin/hubApi/list/chromosomes?db=%s</em></li>\n", ucscDb, ucscDb); hPrintf("<li><a href='/cgi-bin/hubApi/list/chromosomes?db=%s&track=gap' target=_blank>list chromosomes from specified track from UCSC databaset</a> <em>/cgi-bin/hubApi/list/chromosomes?db=%s&track=gap</em></li>\n", ucscDb, ucscDb); hPrintf("<li><a href='/cgi-bin/hubApi/getData/sequence?db=%s&chrom=chrM' target=_blank>get sequence from specified database and chromosome</a> <em>/cgi-bin/hubApi/getData/sequence?db=%s&chrom=chrM</em></li>\n", ucscDb, ucscDb); hPrintf("<li><a href='/cgi-bin/hubApi/getData/sequence?db=%s&chrom=chrM&start=0&end=128' target=_blank>get sequence from specified database, chromosome with start,end coordinates</a> <em>/cgi-bin/hubApi/getData/sequence?db=%s&chrom=chrM&start=0&end=128</em></li>\n", ucscDb, ucscDb); hPrintf("<li><a href='/cgi-bin/hubApi/getData/track?db=%s&track=gold' target=_blank>get entire track data from specified database and track name (gold == Assembly)</a> <em>/cgi-bin/hubApi/getData/track?db=%s&track=gold</em></li>\n", ucscDb, ucscDb); hPrintf("<li><a href='/cgi-bin/hubApi/getData/track?db=%s&chrom=chrM&track=gold' target=_blank>get track data from specified database, chromosome and track name (gold == Assembly)</a> <em>/cgi-bin/hubApi/getData/track?db=%s&chrom=chrM&track=gold</em></li>\n", ucscDb, ucscDb); hPrintf("<li><a href='/cgi-bin/hubApi/getData/track?db=%s&chrom=chrI&track=gold&start=107680&end=186148' target=_blank>get track data from specified database, chromosome, track name, start and end coordinates</a> <em>/cgi-bin/hubApi/getData/track?db=%s&chrom=chr1&track=gold&start=107680&end=186148</em></li>\n", ucscDb, ucscDb); hPrintf("<li><a href='/cgi-bin/hubApi/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/GillBejerano/hub.txt&genome=hg19&track=ultraConserved' target=_blank>get entire track data from specified hub and track name</a> <em>/cgi-bin/hubApi/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/GillBejerano/hub.txt&genome=hg19&track=ultraConserved</em></li>\n"); hPrintf("<li><a href='/cgi-bin/hubApi/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrCp&track=assembly_' target=_blank>get track data from specified hub, chromosome and track name (full chromosome)</a> <em>/cgi-bin/hubApi/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrCp&track=assembly_</em></li>\n"); hPrintf("<li><a href='/cgi-bin/hubApi/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chr1&track=assembly_&start=0&end=14309681' target=_blank>get track data from specified hub, chromosome, track name, start and end coordinates</a> <em>/cgi-bin/hubApi/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chr1&track=assembly_&start=0&end=14309681</em></li>\n"); hPrintf("<li><a href='/cgi-bin/hubApi/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&track=gc5Base_' target=_blank>get all track data from specified hub and track name</a> <em>/cgi-bin/hubApi/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&track=gc5Base</em></li>\n"); hPrintf("<li><a href='/cgi-bin/hubApi/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrMt&track=gc5Base_&start=143600&end=143685' target=_blank>get track data from specified hub, chromosome, track name, start and end coordinates</a> <em>/cgi-bin/hubApi/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrMt&track=gc5Base&start=143600&end=143685</em></li>\n"); hPrintf("</ol>\n"); hPrintf("<h2>Example URLs to generate errors:</h2>\n"); hPrintf("<li><a href='/cgi-bin/hubApi/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrI&track=assembly_&start=0&end=14309681' target=_blank>get track data from specified hub, chromosome, track name, start and end coordinates</a> <em>/cgi-bin/hubApi/getData/track?hubUrl=http://genome-test.gi.ucsc.edu/~hiram/hubs/Plants/hub.txt&genome=_araTha1&chrom=chrI&track=assembly_&start=0&end=14309681</em></li>\n"); hPrintf("<ol>\n"); hPrintf("</ol>\n"); } /* static void showExamples() */ -#ifdef NOT static void showCartDump() /* for information purposes only during development, will become obsolete */ { hPrintf("<h4>cart dump</h4>"); hPrintf("<pre>\n"); cartDump(cart); hPrintf("</pre>\n"); } -#endif static void doMiddle(struct cart *theCart) /* Set up globals and make web page */ { cart = theCart; measureTiming = hPrintStatus() && isNotEmpty(cartOptionalString(cart, "measureTiming")); measureTiming = TRUE; char *database = NULL; char *genome = NULL; +cgiVarSet("ignoreCookie", "1"); + getDbAndGenome(cart, &database, &genome, oldVars); initGenbankTableNames(database); char *docRoot = cfgOptionDefault("browser.documentRoot", DOCUMENT_ROOT); int timeout = cartUsualInt(cart, "udcTimeout", 300); if (udcCacheTimeout() < timeout) udcSetCacheTimeout(timeout); knetUdcInstall(); char *pathInfo = getenv("PATH_INFO"); if (isNotEmpty(pathInfo)) { puts("Content-Type:application/json"); @@ -621,31 +676,31 @@ long lastTime = clock1000(); struct trackHub *hub = errCatchTrackHubOpen(urlInput); if (measureTiming) { long thisTime = clock1000(); hPrintf("<em>hub open time: %ld millis</em><br>\n", thisTime - lastTime); } // hPrintf("<h3>ucscDb: '%s'</h2>\n", ucscDb); struct trackHubGenome *hubGenome = hub->genomeList; showExamples(urlInput, hubGenome, ucscDb); -// showCartDump(); +showCartDump(); hPrintf("<form action='%s' name='hubApiUrl' id='hubApiUrl' method='GET'>\n\n", "../cgi-bin/hubApi"); hPrintf("<b>Select public hub: </b>"); #define JBUFSIZE 2048 #define SMALLBUF 256 char javascript[JBUFSIZE]; struct slPair *events = NULL; safef(javascript, sizeof(javascript), "this.lastIndex=this.selectedIndex;"); slPairAdd(&events, "focus", cloneString(javascript)); cgiMakeDropListClassWithIdStyleAndJavascript("publicHubs", "publicHubs", shortLabels, publicHubCount, hubDropDown, NULL, "width: 400px", events); hWrites(" "); @@ -692,28 +747,28 @@ (void) genomeList(hub, NULL, NULL); /* ignore returned list */ hPrintf("</p>\n"); } if (timedOut) hPrintf("<h1>Reached time out %ld seconds</h1>", timeOutSeconds); if (measureTiming) hPrintf("<em>Overall total time: %ld millis</em><br>\n", clock1000() - enteredMainTime); cartWebEnd(); } /* void doMiddle(struct cart *theCart) */ /* Null terminated list of CGI Variables we don't want to save * permanently. */ -char *excludeVars[] = {"Submit", "submit", NULL,}; +static char *excludeVars[] = {"Submit", "submit", NULL,}; int main(int argc, char *argv[]) /* Process command line. */ { enteredMainTime = clock1000(); cgiSpoof(&argc, argv); measureTiming = TRUE; verboseTimeInit(); trackCounter = hashNew(0); cartEmptyShellNoContent(doMiddle, hUserCookie(), excludeVars, oldVars); return 0; }