046c975630f6576722e8cbfa843e68f6068d854e hiram Thu Mar 21 15:06:42 2019 -0700 somewhat better for sample data URLs refs #18869 diff --git src/hg/hubApi/hubApi.c src/hg/hubApi/hubApi.c index 76ff73b..e03035b 100644 --- src/hg/hubApi/hubApi.c +++ src/hg/hubApi/hubApi.c @@ -3,31 +3,35 @@ /* +------------------+------------------+------+-----+---------+-------+ | Field | Type | Null | Key | Default | Extra | +------------------+------------------+------+-----+---------+-------+ | hubUrl | longblob | NO | PRI | NULL | | | shortLabel | varchar(255) | NO | | NULL | | | longLabel | varchar(255) | NO | | NULL | | | registrationTime | varchar(255) | NO | | NULL | | | dbCount | int(10) unsigned | NO | | NULL | | | dbList | blob | YES | | NULL | | | descriptionUrl | longblob | YES | | NULL | | +------------------+------------------+------+-----+---------+-------+ */ -/* Global Variables */ +/* Global Variables for all modules */ + +int maxItemsOutput = 1000; /* can be set in URL maxItemsOutput=N */ + +/* Global only to this one source file */ static boolean debug = FALSE; /* can be set in URL debug=1 */ static struct cart *cart; /* CGI and other variables */ static struct hash *oldVars = NULL; static struct hash *trackCounter = NULL; static long totalTracks = 0; static boolean measureTiming = FALSE; /* set by CGI parameters */ static boolean allTrackSettings = FALSE; /* checkbox setting */ static char **shortLabels = NULL; /* public hub short labels in array */ // struct hubPublic *publicHubList = NULL; static int publicHubCount = 0; static char *defaultHub = "Plants"; static char *defaultDb = "ce11"; static long enteredMainTime = 0; /* will become = clock1000() on entry */ /* to allow calculation of when to bail out, taking too long */ static long timeOutSeconds = 100; @@ -43,40 +47,40 @@ slAddHead(&supportedTypes, el); el = newSlName("wig"); slAddHead(&supportedTypes, el); el = newSlName("broadPeak"); slAddHead(&supportedTypes, el); el = newSlName("narrowPeak"); slAddHead(&supportedTypes, el); el = newSlName("bigBed"); slAddHead(&supportedTypes, el); el = newSlName("bigWig"); slAddHead(&supportedTypes, el); el = newSlName("bigNarrowPeak"); slAddHead(&supportedTypes, el); el = newSlName("bigGenePred"); slAddHead(&supportedTypes, el); -el = newSlName("bigPsl"); -slAddHead(&supportedTypes, el); -el = newSlName("bigBarChart"); -slAddHead(&supportedTypes, el); -el = newSlName("bigInteract"); -slAddHead(&supportedTypes, el); -el = newSlName("bigMaf"); -slAddHead(&supportedTypes, el); -el = newSlName("bigChain"); -slAddHead(&supportedTypes, el); +// el = newSlName("bigPsl"); +// slAddHead(&supportedTypes, el); +// el = newSlName("bigBarChart"); +// slAddHead(&supportedTypes, el); +// el = newSlName("bigInteract"); +// slAddHead(&supportedTypes, el); +// el = newSlName("bigMaf"); +// slAddHead(&supportedTypes, el); +// el = newSlName("bigChain"); +// slAddHead(&supportedTypes, el); slNameSort(&supportedTypes); } static boolean isSupportedType(char *type) /* is given type in the supportedTypes list ? */ { boolean ret = FALSE; if (startsWith("wigMaf", type)) /* not wigMaf at this time */ return ret; struct slName *el; for (el = supportedTypes; el; el = el->next) { if (startsWith(el->name, type)) { ret = TRUE; @@ -188,75 +192,93 @@ hashIncInt(countTracks, "superTrack child"); hashIncInt(countTracks, stripType); hashIncInt(countTracks, "track count"); } else if (isEmpty(tdb->type)) hashIncInt(countTracks, "no type specified"); else { hashIncInt(countTracks, stripType); hashIncInt(countTracks, "track count"); } freeMem(stripType); // showCounts(countTracks); } -static void sampleUrl(char *db, struct trackDb *tdb, char *chrom, long long chromSize) +static void sampleUrl(struct trackHub *hub, char *db, struct trackDb *tdb, char *chrom, long long chromSize) /* print out a sample getData URL */ { unsigned start = chromSize / 4; unsigned end = start + 10000; if (end > chromSize) end = chromSize; +char *genome = NULL; +if (hub) + genome = hub->genomeList->name; if (db) { + if (hub) + { + if (tdb->parent) + hPrintf("<li>%s : %s subtrack of parent: %s <a href='%s/getData/track?hubUrl=%s&genome=%s&chrom=%s&track=%s&start=%u&end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, tdb->parent->track, urlPrefix, hub->url, genome, chrom, tdb->track, start, end ); + else + hPrintf("<li>%s : %s <a href='%s/getData/track?hubUrl=%s&genome=%s&chrom=%s&track=%s&start=%u&end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, urlPrefix, hub->url, genome, chrom, tdb->track, start, end ); + } + else + { if (tdb->parent) hPrintf("<li>%s : %s subtrack of parent: %s <a href='%s/getData/track?db=%s&chrom=%s&track=%s&start=%u&end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, tdb->parent->track, urlPrefix, db, chrom, tdb->track, start, end ); else hPrintf("<li>%s : %s <a href='%s/getData/track?db=%s&chrom=%s&track=%s&start=%u&end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, urlPrefix, db, chrom, tdb->track, start, end ); } + } +else if (hub) + { + if (tdb->parent) + hPrintf("<li>%s : %s subtrack of parent: %s <a href='%s/getData/track?hubUrl=%s&genome=%s&chrom=%s&track=%s&start=%u&end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, tdb->parent->track, urlPrefix, hub->url, genome, chrom, tdb->track, start, end ); else - hPrintf("<li>%s : %s not db track</li>\n", tdb->track, tdb->type); + hPrintf("<li>%s : %s <a href='%s/getData/track?hubUrl=%s&genome=%s&chrom=%s&track=%s&start=%u&end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, urlPrefix, hub->url, genome, chrom, tdb->track, start, end ); + } +else + hPrintf("<li>%s : %s not db hub track ?</li>\n", tdb->track, tdb->type); } -static void showSubTracks(char *db, struct trackDb *tdb, struct hash *countTracks, +static void showSubTracks(struct trackHub *hub, char *db, struct trackDb *tdb, struct hash *countTracks, char *chromName, long long chromSize) /* tdb has subtracks, show only subTracks, no details */ { hPrintf(" <li><ul>\n"); +if (debug) hPrintf(" <li>subtracks for '%s' db: '%s'</li>\n", tdb->track, db); if (tdb->subtracks) { struct trackDb *tdbEl = NULL; for (tdbEl = tdb->subtracks; tdbEl; tdbEl = tdbEl->next) { if (tdbIsCompositeView(tdbEl)) - hPrintf("<li>%s : %s : composite YYY view of parent: %s</li>\n", tdbEl->track, tdbEl->type, tdbEl->parent->track); + hPrintf("<li>%s : %s : composite view of parent: %s</li>\n", tdbEl->track, tdbEl->type, tdbEl->parent->track); else { if (isSupportedType(tdbEl->type)) - { - sampleUrl(db, tdbEl, chromName, chromSize); -// hPrintf("<li>%s : %s : subtrack of parent: %s (with sample)</li>\n", tdbEl->track, tdbEl->type, tdbEl->parent->track, getData); - } + sampleUrl(hub, db, tdbEl, chromName, chromSize); else hPrintf("<li>%s : %s : subtrack of parent: %s</li>\n", tdbEl->track, tdbEl->type, tdbEl->parent->track); } hashCountTrack(tdbEl, countTracks); if (tdbEl->subtracks) - showSubTracks(db, tdbEl, countTracks, chromName, chromSize); + showSubTracks(hub, db, tdbEl, countTracks, chromName, chromSize); } } hPrintf(" </ul></li>\n"); } static void trackSettings(struct trackDb *tdb, struct hash *countTracks) /* process the settingsHash for a trackDb, recursive when subtracks */ { hPrintf(" <li><ul>\n"); // if (tdb->children) haven't yet seen a track with children ? // hPrintf(" <li>%s: has children</li>\n", tdb->track); // else // hPrintf(" <li>%s: NO children</li>\n", tdb->track); struct hashEl *hel; struct hashCookie hc = hashFirst(tdb->settingsHash); @@ -407,190 +429,175 @@ return retVal; } /* static int bbiBriefMeasure() */ static void hubSampleUrl(struct trackHub *hub, struct trackDb *tdb, long chromCount, long itemCount, char *chromName, unsigned chromSize, char *genome) { unsigned start = chromSize / 4; unsigned end = start + 10000; if (end > chromSize) end = chromSize; if (isSupportedType(tdb->type)) { if (startsWithWord("bigBed", tdb->type)) - hPrintf(" <li>%s : %s : %ld chroms : %ld item count <a href='%s/getData/track?hubUrl=%s&genome=%s&track=%s&chrom=%s&start=%u&end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, chromCount, itemCount, urlPrefix, hub->url, genome, tdb->track, chromName, start, end); + hPrintf(" <li>%s : %s : %ld chroms : %ld item count <a href='%s/getData/track?hubUrl=%s&genome=%s&track=%s&chrom=%s&start=%u&end=%u' target=_blank>(Xsample getData)</a></li>\n", tdb->track, tdb->type, chromCount, itemCount, urlPrefix, hub->url, genome, tdb->track, chromName, start, end); else if (startsWithWord("bigWig", tdb->type)) - hPrintf(" <li>%s : %s : %ld chroms : %ld bases covered <a href='%s/getData/track?hubUrl=%s&genome=%s&track=%s&chrom=%s&start=%u&end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, chromCount, itemCount, urlPrefix, hub->url, genome, tdb->track, chromName, start, end); + hPrintf(" <li>%s : %s : %ld chroms : %ld bases covered <a href='%s/getData/track?hubUrl=%s&genome=%s&track=%s&chrom=%s&start=%u&end=%u' target=_blank>(Xsample getData)</a></li>\n", tdb->track, tdb->type, chromCount, itemCount, urlPrefix, hub->url, genome, tdb->track, chromName, start, end); else - hPrintf(" <li>%s : %s : %ld chroms : %ld count <<a href='%s/getData/track?hubUrl=%s&genome=%s&track=%s&chrom=%s&start=%u&end=%u' target=_blank>(sample getData)</a></li>\n", tdb->track, tdb->type, chromCount, itemCount, urlPrefix, hub->url, genome, tdb->track, chromName, start, end); + hPrintf(" <li>%s : %s : %ld chroms : %ld count <<a href='%s/getData/track?hubUrl=%s&genome=%s&track=%s&chrom=%s&start=%u&end=%u' target=_blank>(Xsample getData)</a></li>\n", tdb->track, tdb->type, chromCount, itemCount, urlPrefix, hub->url, genome, tdb->track, chromName, start, end); } else { if (startsWithWord("bigBed", tdb->type)) hPrintf(" <li>%s : %s : %ld chroms : %ld item count</li>\n", tdb->track, tdb->type, chromCount, itemCount); else if (startsWithWord("bigWig", tdb->type)) hPrintf(" <li>%s : %s : %ld chroms : %ld bases covered</li>\n", tdb->track, tdb->type, chromCount, itemCount); else hPrintf(" <li>%s : %s : %ld chroms : %ld count</li>\n", tdb->track, tdb->type, chromCount, itemCount); } } static void hubCountOneTdb(struct trackHub *hub, char *db, struct trackDb *tdb, char *bigDataIndex, struct hash *countTracks, char *chromName, unsigned chromSize, char *genome) { char *bigDataUrl = trackDbSetting(tdb, "bigDataUrl"); // char *compositeTrack = trackDbSetting(tdb, "compositeTrack"); boolean compositeContainer = tdbIsComposite(tdb); boolean compositeView = tdbIsCompositeView(tdb); // char *superTrack = trackDbSetting(tdb, "superTrack"); boolean superChild = tdbIsSuperTrackChild(tdb); boolean depthSearch = cartUsualBoolean(cart, "depthSearch", FALSE); hashCountTrack(tdb, countTracks); -if (depthSearch && bigDataUrl) - { char *longName = NULL; unsigned longSize = 0; long chromCount = 0; long itemCount = 0; + +if (! (compositeContainer || compositeView) ) + { struct dyString *errors = newDyString(1024); int retVal = bbiBriefMeasure(tdb->type, bigDataUrl, bigDataIndex, &chromCount, &itemCount, errors, &longName, &longSize); if (retVal) - { hPrintf(" <li>%s : %s : <font color='red'>ERROR: %s</font></li>\n", tdb->track, tdb->type, errors->string); } - else + +if (depthSearch && bigDataUrl) + { + if (isSupportedType(tdb->type)) { if (chromSize > 0) hubSampleUrl(hub, tdb, chromCount, itemCount, chromName, chromSize, genome); else hubSampleUrl(hub, tdb, chromCount, itemCount, longName, longSize, genome); } } else { if (compositeContainer) hPrintf(" <li>%s : %s : composite track container has %d subtracks</li>\n", tdb->track, tdb->type, slCount(tdb->subtracks)); else if (compositeView) - hPrintf(" <li>%s : %s : XXX composite view of parent: %s</li>\n", tdb->track, tdb->type, tdb->parent->track); + hPrintf(" <li>%s : %s : composite view of parent: %s</li>\n", tdb->track, tdb->type, tdb->parent->track); else if (superChild) - hPrintf(" <li>%s : %s : superTrack child of parent: %s</li>\n", tdb->track, tdb->type, tdb->parent->track); + hPrintf(" <li>%s : %s : superTrack child of parent: %s (sample getData)</li>\n", tdb->track, tdb->type, tdb->parent->track); else if (! depthSearch && bigDataUrl) hPrintf(" <li>%s : %s : %s</li>\n", tdb->track, tdb->type, bigDataUrl); else { if (isSupportedType(tdb->type)) - sampleUrl(db, tdb, chromName, chromSize); + { + if (chromSize > 0) + hubSampleUrl(hub, tdb, chromCount, itemCount, chromName, chromSize, genome); + else + hubSampleUrl(hub, tdb, chromCount, itemCount, longName, longSize, genome); + } else hPrintf(" <li>%s : %s</li>\n", tdb->track, tdb->type); } } if (allTrackSettings) { hPrintf(" <li><ul>\n"); trackSettings(tdb, countTracks); /* show all settings */ hPrintf(" </ul></li>\n"); } else if (tdb->subtracks) { - showSubTracks(db, tdb, countTracks, chromName, chromSize); + showSubTracks(hub, db, tdb, countTracks, chromName, chromSize); } return; } /* static void hubCountOneTdb(char *db, struct trackDb *tdb, * char *bigDataIndex, struct hash *countTracks, * char *chromName, long long chromSize) */ -static void countOneTdb(struct trackHub *hub, char *db, struct trackDb *tdb, - char *bigDataIndex, struct hash *countTracks, char *chromName, - unsigned chromSize, char *genome) +static void countOneTdb(char *db, struct trackDb *tdb, + struct hash *countTracks, char *chromName, unsigned chromSize) +/* for this tdb in this db, count it up and provide a sample */ { char *bigDataUrl = trackDbSetting(tdb, "bigDataUrl"); // char *compositeTrack = trackDbSetting(tdb, "compositeTrack"); boolean compositeContainer = tdbIsComposite(tdb); boolean compositeView = tdbIsCompositeView(tdb); // char *superTrack = trackDbSetting(tdb, "superTrack"); boolean superChild = tdbIsSuperTrackChild(tdb); boolean depthSearch = cartUsualBoolean(cart, "depthSearch", FALSE); hashCountTrack(tdb, countTracks); -if (depthSearch && bigDataUrl) - { - char *longName = NULL; - unsigned longSize = 0; - long chromCount = 0; - long itemCount = 0; - struct dyString *errors = newDyString(1024); - int retVal = bbiBriefMeasure(tdb->type, bigDataUrl, bigDataIndex, &chromCount, &itemCount, errors, &longName, &longSize); - if (retVal) - { - hPrintf(" <li>%s : %s : <font color='red'>ERROR: %s</font></li>\n", tdb->track, tdb->type, errors->string); - } - else - { - if (chromSize > 0) - hubSampleUrl(hub, tdb, chromCount, itemCount, chromName, chromSize, genome); - else - hubSampleUrl(hub, tdb, chromCount, itemCount, longName, longSize, genome); - } - } -else - { if (compositeContainer) hPrintf(" <li>%s : %s : composite track container has %d subtracks</li>\n", tdb->track, tdb->type, slCount(tdb->subtracks)); else if (compositeView) - hPrintf(" <li>%s : %s : XXX composite view of parent: %s</li>\n", tdb->track, tdb->type, tdb->parent->track); + hPrintf(" <li>%s : %s : composite view of parent: %s</li>\n", tdb->track, tdb->type, tdb->parent->track); else if (superChild) hPrintf(" <li>%s : %s : superTrack child of parent: %s</li>\n", tdb->track, tdb->type, tdb->parent->track); else if (! depthSearch && bigDataUrl) hPrintf(" <li>%s : %s : %s</li>\n", tdb->track, tdb->type, bigDataUrl); else { if (isSupportedType(tdb->type)) - sampleUrl(db, tdb, chromName, chromSize); + sampleUrl(NULL, db, tdb, chromName, chromSize); else hPrintf(" <li>%s : %s</li>\n", tdb->track, tdb->type); } - } + if (allTrackSettings) { hPrintf(" <li><ul>\n"); trackSettings(tdb, countTracks); /* show all settings */ hPrintf(" </ul></li>\n"); } else if (tdb->subtracks) { - showSubTracks(db, tdb, countTracks, chromName, chromSize); + showSubTracks(NULL, db, tdb, countTracks, chromName, chromSize); } return; } /* static void countOneTdb(char *db, struct trackDb *tdb, - * char *bigDataIndex, struct hash *countTracks, - * char *chromName, long long chromSize) + * struct hash *countTracks, char *chromName, + * unsigned chromSize) */ static unsigned largestChrom(char *db, char **nameReturn) /* return the length and get the chrom name for the largest chrom * from chromInfo table. For use is sample getData URLs */ { char query[1024]; struct sqlConnection *conn = hAllocConn(db); -// Build a query to select the hubUrl for the given shortLabel sqlSafef(query, sizeof(query), "select chrom,size from chromInfo order by size desc limit 1"); struct sqlResult *sr = sqlGetResult(conn, query); char **row = sqlNextRow(sr); unsigned length = 0; if (row) { *nameReturn = cloneString(row[0]); length = sqlLongLong(row[1]); } sqlFreeResult(&sr); hFreeConn(&conn); return length; } static void hubTrackList(struct trackHub *hub, struct trackDb *topTrackDb, struct trackHubGenome *genome) @@ -812,31 +819,31 @@ } static void tracksForUcscDb(char *db) /* scan the specified database for all tracks */ { struct hash *countTracks = hashNew(0); char *chromName = NULL; unsigned chromSize = largestChrom(db, &chromName); hPrintf("<h4>Tracks in UCSC genome: '%s', longest chrom: %s:%u</h4>\n", db, chromName, chromSize); struct trackDb *tdbList = obtainTdb(NULL, db); struct trackDb *tdb; hPrintf("<ul>\n"); hPrintf("<li>%s:%u</li>\n", chromName, chromSize); for (tdb = tdbList; tdb != NULL; tdb = tdb->next ) { - countOneTdb(NULL, db, tdb, NULL, countTracks, chromName, chromSize, NULL); + countOneTdb(db, tdb, countTracks, chromName, chromSize); if (timeOutReached()) break; } int trackCount = ptToInt(hashFindVal(countTracks, "track count")); /* elCount - 1 since the 'track count' element isn't a track */ hPrintf(" <li>%d total tracks counted, %d different track types</li>\n", trackCount, countTracks->elCount - 1); if (countTracks->elCount) { hPrintf(" <ol>\n"); struct hashEl *hel, *helList = hashElListHash(countTracks); slSort(&helList, hashElCmpIntValDesc); for (hel = helList; hel; hel = hel->next) { if (sameOk("track count", hel->name)) continue; @@ -962,30 +969,33 @@ /* Set up globals and make web page */ { cart = theCart; measureTiming = hPrintStatus() && isNotEmpty(cartOptionalString(cart, "measureTiming")); measureTiming = TRUE; char *database = NULL; char *genome = NULL; cgiVarSet("ignoreCookie", "1"); getDbAndGenome(cart, &database, &genome, oldVars); initGenbankTableNames(database); initSupportedTypes(); initUrlPrefix(); +/* global variable for all workers to honor this limit */ +maxItemsOutput = cartUsualInt(cart, "maxItemsOutput", maxItemsOutput); + debug = cartUsualBoolean(cart, "debug", debug); int timeout = cartUsualInt(cart, "udcTimeout", 300); if (udcCacheTimeout() < timeout) udcSetCacheTimeout(timeout); knetUdcInstall(); char *pathInfo = getenv("PATH_INFO"); /* nothing on incoming path, then display the WEB page instead */ if (sameOk("/",pathInfo)) pathInfo = NULL; boolean commandError = FALSE; /*expect no more than MAX_PATH_INFO number of words*/ char *words[MAX_PATH_INFO];