56ec64d6672d5c2ef118bd5ef929ac8bf72b6e61 chmalee Mon Jul 29 15:19:36 2019 -0700 First cut of having hubCheck output javascript arrays for jstree, refs #13428 diff --git src/hg/utils/hubCheck/hubCheck.c src/hg/utils/hubCheck/hubCheck.c index cc916fc..7d7a632 100644 --- src/hg/utils/hubCheck/hubCheck.c +++ src/hg/utils/hubCheck/hubCheck.c @@ -50,47 +50,51 @@ ); } static struct optionSpec options[] = { {"version", OPTION_STRING}, {"level", OPTION_STRING}, {"extra", OPTION_STRING}, {"noTracks", OPTION_BOOLEAN}, {"settings", OPTION_BOOLEAN}, {"checkSettings", OPTION_BOOLEAN}, {"test", OPTION_BOOLEAN}, {"printMeta", OPTION_BOOLEAN}, {"udcDir", OPTION_STRING}, {"specHost", OPTION_STRING}, {"cacheTime", OPTION_INT}, + // intentionally undocumented option for hgHubConnect + {"htmlOut", OPTION_BOOLEAN}, {NULL, 0}, }; struct trackHubCheckOptions /* How to check track hub */ { boolean checkFiles; /* check remote files exist and are correct type */ boolean checkSettings; /* check trackDb settings to spec */ boolean printMeta; /* print out the metadata for each track */ char *version; /* hub spec version to check */ char *specHost; /* server hosting hub spec */ char *level; /* check hub is valid to this support level */ char *extraFile; /* name of extra file/url with additional settings to accept */ /* intermediate data */ struct hash *settings; /* supported settings for this version */ struct hash *extra; /* additional trackDb settings to accept */ struct slName *suggest; /* list of supported settings for suggesting */ + /* hgHubConnect only */ + boolean htmlOut; /* put special formatted text into the errors dyString */ }; struct trackHubSettingSpec /* Setting name and support level, from trackDbHub.html (the spec) */ { struct trackHubSettingSpec *next; char *name; /* setting name */ char *level; /* support level (required, base, full, new, deprecated) */ }; /* Mini English spell-check using axt sequence alignment code! From JK * Works in this context when thresholded high. */ static struct axtScoreScheme *scoreSchemeEnglish() @@ -443,89 +447,236 @@ { /* check level */ struct trackHubSettingSpec *checkLevel = NULL; AllocVar(checkLevel); checkLevel->level = options->level; if (trackHubSettingLevel(hubSetting) < trackHubSettingLevel(checkLevel)) { dyStringPrintf(errors, "Setting '%s' is level '%s'\n", setting, hubSetting->level); retVal = 1; } freez(&checkLevel); } return retVal; } -int hubCheckCompositeSettings(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb, struct dyString *errors) -/* Check composite level settings like subgroups, dimensions, sortOrder, etc. - * Note that this function returns an int because we want to warn about all errors in a single - * composite stanza rather than errAbort on the first error */ +char *makeFolderObjectString(char *id, char *text, char *parent, char *title, boolean children, boolean openFolder) +/* Construct a folder item for one of the jstree arrays */ +{ +struct dyString *folderString = dyStringNew(0); +dyStringPrintf(folderString, "{icon: '../../images/folderC.png', id: '%s', " + "text:\"%s\", parent:'%s'," + "li_attr:{title:'%s'}, children:%s, state: {opened: %s}}", + id, text, parent, title, children ? "true" : "false", openFolder ? "true" : "false"); +return dyStringCannibalize(&folderString); +} + +char *makeChildObjectString(char *id, char *title, char *shortLabel, char *longLabel, + char *color, char *name, char *text, char *parent) +/* Construct a single child item for one of the jstree arrays */ +{ +struct dyString *item = dyStringNew(0); +dyStringPrintf(item, "{icon: 'fa fa-plus', id:'%s', li_attr:{title: '%s', " + "shortLabel: '%s', longLabel: '%s', color: '%s', name:'%s'}, " + "text:\"%s\", parent: '%s', state: {opened: true}}", + id, title, shortLabel, longLabel, color, name, text, parent); +return dyStringCannibalize(&item); +} + +void hubErr(struct dyString *errors, char *message, struct trackHub *hub) +/* Construct the right javascript for the jstree for a top level hub.txt error. */ +{ +char *sl; +char *strippedMessage = NULL; +static int count = 0; // force a unique id for the jstree object +char id[512]; +//TODO: Choose better default labels +if (hub && hub->shortLabel != NULL) + { + sl = hub->shortLabel; + } +else + sl = "Hub Error"; +if (message) + strippedMessage = cloneString(message); +stripChar(strippedMessage, '\n'); +safef(id, sizeof(id), "%s%d", sl, count); + +// make the error message +dyStringPrintf(errors, "trackData['%s'] = [%s];\n", sl, + makeChildObjectString(id, "Hub Error", sl, sl, "#550073", sl, strippedMessage, sl)); + +// display it by default +dyStringPrintf(errors, "trackData['#'] = [%s];\n", + makeFolderObjectString(sl, "Error getting hub or genomes configuration", "#", "Click to open node", TRUE, TRUE)); +count++; +} + +void genomeErr(struct dyString *errors, char *message, struct trackHub *hub, + struct trackHubGenome *genome) +/* Construct the right javascript for the jstree for a top-level genomes.txt error or + * error opening a trackDb.txt file */ +{ +static int count = 0; // forces unique ID's which the jstree object needs +char id[512]; +char *strippedMessage = NULL; +char *genomeName = trackHubSkipHubName(genome->name); +if (message) + strippedMessage = cloneString(message); +stripChar(strippedMessage, '\n'); +safef(id, sizeof(id), "%s%d", genomeName, count); + +dyStringPrintf(errors, "trackData['%s'] = [%s", genomeName, + makeChildObjectString(id, "Error Getting TrackDb", genomeName, genomeName, "#550073", genomeName, strippedMessage, genomeName)); +count++; +} + +void trackDbErr(struct dyString *errors, char *message, struct trackHubGenome *genome, struct trackDb *tdb, boolean doHtml) +/* Adds the right object for a jstree object of trackDb configuration errors. */ +{ +if (!doHtml) + { + dyStringPrintf(errors, "%s", message); + } +else + { + char *strippedMessage = NULL; + char *parentOrTrackString = trackHubSkipHubName(tdb->track); + char id[512]; + static int count = 0; // forces unique ID's which the jstree object needs + + if (message) + strippedMessage = cloneString(message); + + stripChar(strippedMessage, '\n'); + safef(id, sizeof(id), "%s%d", trackHubSkipHubName(tdb->track), count); + dyStringPrintf(errors, "%s,", + makeChildObjectString(id, "TrackDb Error", tdb->shortLabel, tdb->longLabel, + "#550073", trackHubSkipHubName(tdb->track), strippedMessage, parentOrTrackString)); + count++; + } +} + +boolean checkEmptyMembersForAll(membersForAll_t *membersForAll, struct trackDb *parentTdb) +/* membersForAll may be allocated and exist but not have any actual members defined. */ +{ +int i; +for (i = 0; i < ArraySize(membersForAll->members); i++) + { + if (membersForAll->members[i] != NULL) + return TRUE; + } +return FALSE; +} + +int hubCheckSubtrackSettings(struct trackHubGenome *genome, struct trackDb *tdb, struct dyString *errors, struct trackHubCheckOptions *options) +/* Check that 'subgroups' are consistent with what is defined at the parent level */ { int retVal = 0; -if (!tdbIsComposite(tdb)) +if (!tdbIsSubtrack(tdb)) return retVal; +int i; +char *subtrackName = trackHubSkipHubName(tdb->track); sortOrder_t *sortOrder = NULL; membership_t *membership = NULL; -struct slRef *subtrackRef, *subtrackRefList = NULL; - -// check that if a sortOrder is defined, then subtracks exist in the subgroup +membersForAll_t *membersForAll = NULL; struct errCatch *errCatch = errCatchNew(); + if (errCatchStart(errCatch)) { - (void)membersForAllSubGroupsGet(tdb, NULL); - } -errCatchEnd(errCatch); -if (errCatch->gotWarning || errCatch->gotError) + membersForAll = membersForAllSubGroupsGet(tdb->parent, NULL); + + // membersForAllSubGroupsGet() warns about the parent stanza, turn it into an errAbort + if (errCatch->gotWarning) { - // don't add a new line because one will already be inserted by the errCatch->message - dyStringPrintf(errors, "track %s error: %s", tdb->shortLabel, errCatch->message->string); - retVal = 1; + char *temp = cloneString(errCatch->message->string); + stripChar(temp, '\n'); + dyStringClear(errCatch->message); + errAbort("%s", temp); } -if (errCatchStart(errCatch)) + if (membersForAll && checkEmptyMembersForAll(membersForAll, tdb->parent)) { - sortOrder = sortOrderGet(NULL, tdb); - if (sortOrder) + membership = subgroupMembershipGet(tdb); + sortOrder = sortOrderGet(NULL, tdb->parent); + + if (membership == NULL) { - subtrackRefList = trackDbListGetRefsToDescendantLeaves(tdb->subtracks); - for (subtrackRef = subtrackRefList; subtrackRef != NULL; subtrackRef = subtrackRef->next) + errAbort("missing 'subgroups' setting for subtrack %s", subtrackName); + } + + // if a sortOrder is defined, make sure every subtrack has that membership + if (sortOrder) { - struct trackDb *subtrack = subtrackRef->val; - membership = subgroupMembershipGet(subtrack); - int i; for (i = 0; i < sortOrder->count; i++) { char *col = sortOrder->column[i]; if ( (!sameString(col, SUBTRACK_COLOR_SUBGROUP)) && (membership == NULL || stringArrayIx(col, membership->subgroups, membership->count) == -1)) + errAbort("%s not a member of sortOrder subgroup %s", subtrackName, col); + } + } + + // now check that this subtrack is a member of every defined subgroup + for (i = 0; i < ArraySize(membersForAll->members); i++) { - dyStringPrintf(errors, "sortOrder %s defined for all subtracks of the composite track \"%s\", but the track \"%s\" is not a member of this subGroup\n", - col, tdb->shortLabel, subtrack->shortLabel); - retVal = 1; + if (membersForAll->members[i] != NULL) + { + char *subgroupName = membersForAll->members[i]->groupTag; + if (stringArrayIx(subgroupName, membership->subgroups, membership->count) == -1) + { + errAbort("subtrack %s not a member of subgroup %s", subtrackName, subgroupName); } } } } } errCatchEnd(errCatch); -if (errCatch->gotWarning || errCatch->gotError) +if (errCatch->gotError) { - // don't add a new line because one will already be inserted by the errCatch->message - dyStringPrintf(errors, "track %s error: %s", tdb->shortLabel, errCatch->message->string); retVal = 1; + trackDbErr(errors, errCatch->message->string, genome, tdb, options->htmlOut); } errCatchFree(&errCatch); + +return retVal; +} + +int hubCheckCompositeSettings(struct trackHubGenome *genome, struct trackDb *tdb, struct dyString *errors, struct trackHubCheckOptions *options) +/* Check composite level settings like subgroups, dimensions, sortOrder, etc. + * Note that this function returns an int because we want to warn about all errors in a single + * composite stanza rather than errAbort on the first error */ +{ +int retVal = 0; +if (!tdbIsComposite(tdb)) + return retVal; + +// check that subgroup lines are syntactically correct: "subGroup name Title tag1=value1 ..." +struct errCatch *errCatch = errCatchNew(); +if (errCatchStart(errCatch)) + { + (void)membersForAllSubGroupsGet(tdb, NULL); + } +errCatchEnd(errCatch); + +if (errCatch->gotError || errCatch->gotWarning) + { + // don't add a new line because one will already be inserted by the errCatch->message + trackDbErr(errors, errCatch->message->string, genome, tdb, options->htmlOut); + retVal = 1; + } + return retVal; } void hubCheckParentsAndChildren(struct trackDb *tdb) /* Check that a single trackDb stanza has the correct parent and subtrack pointers */ { if (tdbIsSuper(tdb) || tdbIsComposite(tdb) || tdbIsCompositeView(tdb) || tdbIsContainer(tdb)) { if (tdb->subtracks == NULL) { errAbort("Track \"%s\" is declared superTrack, compositeTrack, view or " "container, but has no subtracks", tdb->track); } // Containers should not have a bigDataUrl setting @@ -535,162 +686,267 @@ "container, and also has a bigDataUrl", tdb->track); } } else if (tdb->subtracks != NULL) { errAbort("Track \"%s\" has children tracks (e.g: \"%s\"), but is not a " "compositeTrack, container, view or superTrack", tdb->track, tdb->subtracks->track); } } int hubCheckTrack(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb, struct trackHubCheckOptions *options, struct dyString *errors) /* Check track settings and optionally, files */ { int retVal = 0; +int trackDbErrorCount = 0; if (options->checkSettings && options->settings) { //verbose(3, "Found %d settings to check to spec\n", slCount(settings)); verbose(3, "Checking track: %s\n", tdb->shortLabel); verbose(3, "Found %d settings to check to spec\n", hashNumEntries(tdb->settingsHash)); struct hashEl *hel; struct hashCookie cookie = hashFirst(tdb->settingsHash); while ((hel = hashNext(&cookie)) != NULL) retVal |= hubCheckTrackSetting(hub, tdb, hel->name, options, errors); /* TODO: ? also need to check settings not in this list (other tdb fields) */ } if (options->printMeta) { struct slPair *metaPairs = trackDbMetaPairs(tdb); if (metaPairs != NULL) { printf("%s\n", trackHubSkipHubName(tdb->track)); struct slPair *pair; for(pair = metaPairs; pair; pair = pair->next) { printf("\t%s : %s\n", pair->name, (char *)pair->val); } printf("\n"); } slPairFreeValsAndList(&metaPairs); } - +struct trackDb *tempTdb = NULL; +char *idName, *textName, *parentName = NULL; struct errCatch *errCatch = errCatchNew(); -if (errCatchStart(errCatch)) +boolean trackIsContainer = (tdbIsComposite(tdb) || tdbIsCompositeView(tdb) || tdbIsContainer(tdb)); + +// first get down into the subtracks +if (tdb->subtracks != NULL) { - hubCheckParentsAndChildren(tdb); - if (tdbIsComposite(tdb)) + for (tempTdb = tdb->subtracks; tempTdb != NULL; tempTdb = tempTdb->next) + retVal |= hubCheckTrack(hub, genome, tempTdb, options, errors); + } + +if (options->htmlOut) { - retVal |= hubCheckCompositeSettings(hub, genome, tdb, errors); + dyStringPrintf(errors, "trackData['%s'] = [", trackHubSkipHubName(tdb->track)); } - if (options->checkFiles) + +if (errCatchStart(errCatch)) { + hubCheckParentsAndChildren(tdb); + if (trackIsContainer) + retVal |= hubCheckCompositeSettings(genome, tdb, errors, options); + + if (tdbIsSubtrack(tdb)) + retVal |= hubCheckSubtrackSettings(genome, tdb, errors, options); + + if (options->checkFiles) hubCheckBigDataUrl(hub, genome, tdb); } - } errCatchEnd(errCatch); if (errCatch->gotError) { + trackDbErrorCount += 1; retVal = 1; + if (!options->htmlOut) dyStringPrintf(errors, "%s", errCatch->message->string); } errCatchFree(&errCatch); -if (tdb->subtracks != NULL) +if (options->htmlOut) + { + if (trackIsContainer) { - retVal |= hubCheckTrack(hub, genome, tdb->subtracks, options, errors); + for (tempTdb = tdb->subtracks; tempTdb != NULL; tempTdb = tempTdb->next) + { + idName = trackHubSkipHubName(tempTdb->track); + textName = trackHubSkipHubName(tempTdb->longLabel); + parentName = trackHubSkipHubName(tdb->track); + dyStringPrintf(errors, "%s,", makeFolderObjectString(idName, textName, parentName, "TRACK", TRUE, retVal)); + } + } + else if (!retVal) + { + // add "Error" to the trackname to force uniqueness for the jstree + idName = trackHubSkipHubName(tdb->track); + dyStringPrintf(errors, "{icon: 'fa fa-plus', " + "id:'%sError', text:'No trackDb configuration errors', parent:'%s'}", idName, idName); + } + dyStringPrintf(errors, "];\n"); } return retVal; } int hubCheckGenome(struct trackHub *hub, struct trackHubGenome *genome, struct trackHubCheckOptions *options, struct dyString *errors) /* Check out genome within hub. */ { struct errCatch *errCatch = errCatchNew(); struct trackDb *tdbList = NULL; -int retVal = 0; +int genomeErrorCount = 0; +boolean openedGenome = FALSE; if (errCatchStart(errCatch)) { tdbList = trackHubTracksForGenome(hub, genome); tdbList = trackDbLinkUpGenerations(tdbList); tdbList = trackDbPolishAfterLinkup(tdbList, genome->name); trackHubPolishTrackNames(hub, tdbList); } errCatchEnd(errCatch); if (errCatch->gotError) { - retVal = 1; + openedGenome = TRUE; + genomeErrorCount += 1; + if (options->htmlOut) + { + genomeErr(errors, errCatch->message->string, hub, genome); + } + else dyStringPrintf(errors, "%s", errCatch->message->string); } if (errCatch->gotWarning && !errCatch->gotError) + { + openedGenome = TRUE; + if (options->htmlOut) + { + genomeErr(errors, errCatch->message->string, hub, genome); + } + else dyStringPrintf(errors, "%s", errCatch->message->string); + } errCatchFree(&errCatch); verbose(2, "%d tracks in %s\n", slCount(tdbList), genome->name); struct trackDb *tdb; +int tdbCheckVal; +static struct dyString *tdbDyString = NULL; +if (!tdbDyString) + tdbDyString = dyStringNew(0); + +// build up the track results list, keep track of number of errors, then +// open up genomes folder +char *genomeName = trackHubSkipHubName(genome->name); for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { - retVal |= hubCheckTrack(hub, genome, tdb, options, errors); + if (options->htmlOut) + { + // if we haven't already found an error then open up the array + if (!openedGenome) + { + dyStringPrintf(errors, "trackData['%s'] = [", genomeName); + openedGenome = TRUE; + } + } + // use different dyString for the actual errors generated by each track + tdbCheckVal = hubCheckTrack(hub, genome, tdb, options, tdbDyString); + genomeErrorCount += tdbCheckVal; + if (options->htmlOut) + { + char *name = trackHubSkipHubName(tdb->track); + dyStringPrintf(errors, "%s", makeFolderObjectString(name, tdb->longLabel, genomeName, "TRACK", TRUE, tdbCheckVal ? TRUE : FALSE)); + if (tdb->next != NULL) + dyStringPrintf(errors, ","); + } } +if (options->htmlOut) + dyStringPrintf(errors, "];\n"); -return retVal; +dyStringPrintf(errors, "%s", tdbDyString->string); + +return genomeErrorCount; } int trackHubCheck(char *hubUrl, struct trackHubCheckOptions *options, struct dyString *errors) /* Check a track data hub for integrity. Put errors in dyString. * return 0 if hub has no errors, 1 otherwise * if options->checkTracks is TRUE, check remote files of individual tracks */ { struct errCatch *errCatch = errCatchNew(); struct trackHub *hub = NULL; int retVal = 0; if (errCatchStart(errCatch)) { - hub = trackHubOpen(hubUrl, "hub_0"); + hub = trackHubOpen(hubUrl, ""); } errCatchEnd(errCatch); if (errCatch->gotError) { retVal = 1; + if (options->htmlOut) + { + hubErr(errors, errCatch->message->string, hub); + } + else dyStringPrintf(errors, "%s\n", errCatch->message->string); } if (errCatch->gotWarning && !errCatch->gotError) dyStringPrintf(errors, "%s", errCatch->message->string); errCatchFree(&errCatch); if (hub == NULL) return 1; if (options->checkSettings) retVal |= hubSettingsCheckInit(hub, options, errors); struct trackHubGenome *genome; + +if (options->htmlOut) + dyStringPrintf(errors, "trackData['#'] = ["); + +int numGenomeErrors = 0; +char genomeTitleString[128]; +struct dyString *genomeErrors = dyStringNew(0); for (genome = hub->genomeList; genome != NULL; genome = genome->next) { - retVal |= hubCheckGenome(hub, genome, options, errors); + numGenomeErrors = hubCheckGenome(hub, genome, options, genomeErrors); + if (options->htmlOut) + { + char *genomeName = trackHubSkipHubName(genome->name); + safef(genomeTitleString, sizeof(genomeTitleString), + "%s (%d configuration error%s)", genomeName, numGenomeErrors, + numGenomeErrors == 1 ? "" : "s"); + dyStringPrintf(errors, "%s,", makeFolderObjectString(genomeName, genomeTitleString, "#", + "Click to open node", TRUE, TRUE)); + } + retVal |= numGenomeErrors; } +if (options->htmlOut) + dyStringPrintf(errors, "];\n"); +dyStringPrintf(errors, "%s", dyStringCannibalize(&genomeErrors)); trackHubClose(&hub); return retVal; } static void addExtras(char *extraFile, struct trackHubCheckOptions *checkOptions) /* Add settings from extra file (e.g. for specific hub display site) */ { verbose(2, "Accepting extra settings in '%s'\n", extraFile); checkOptions->extraFile = extraFile; checkOptions->extra = hashNew(0); struct lineFile *lf = NULL; if (startsWith("http", extraFile)) { struct dyString *ds = netSlurpUrl(extraFile); @@ -769,28 +1025,38 @@ cacheTime = optionInt("cacheTime", cacheTime); udcSetCacheTimeout(cacheTime); // UDC cache dir: first check for hg.conf setting, then override with command line option if given. setUdcCacheDir(); udcSetDefaultDir(optionVal("udcDir", udcDefaultDir())); knetUdcInstall(); // make the htslib library use udc if (optionExists("settings")) { showSettings(checkOptions); return 0; } +// hgHubConnect specific option for generating a jstree of the hub errors +checkOptions->htmlOut = optionExists("htmlOut"); struct dyString *errors = newDyString(1024); -if (trackHubCheck(argv[1], checkOptions, errors)) +if (trackHubCheck(argv[1], checkOptions, errors) || checkOptions->htmlOut) + { + if (checkOptions->htmlOut) // just dump errors string to stdout + { + printf("%s", errors->string); + return 1; + } + else { // uniquify and count errors struct slName *errs = slNameListFromString(errors->string, '\n'); slUniqify(&errs, slNameCmp, slNameFree); int errCount = slCount(errs); printf("Found %d problem%s:\n", errCount, errCount == 1 ? "" : "s"); printf("%s\n", slNameListToString(errs, '\n')); return 1; } + } return 0; }