9a36c00bbb96a128296a4415bd05706c9424ad33 kate Thu May 14 15:34:14 2015 -0700 Finish -core and -extra options to hubCheck. refs #10015 diff --git src/hg/lib/trackHubCheck.c src/hg/lib/trackHubCheck.c index 03c35be..6ca105a 100644 --- src/hg/lib/trackHubCheck.c +++ src/hg/lib/trackHubCheck.c @@ -5,59 +5,58 @@ #include "bigBed.h" #include "errCatch.h" #include "vcf.h" #include "hgBam.h" #include "net.h" #include "htmshell.h" #include "htmlPage.h" #include "trackHub.h" #ifdef USE_HAL #include "halBlockViz.h" #endif static int hubCheckTrackSetting(struct trackHub *hub, struct trackDb *tdb, char *setting, struct trackHubCheckOptions *options, struct dyString *errors) -/* Check trackDb setting is by spec (by version and level). Returns non-zero if error, msg in errors */ +/* Check trackDb setting to spec (by version and level). Returns non-zero if error, msg in errors */ { +verbose(4, " Check setting '%s'\n", setting); + +/* skip internally added/used settings */ +if (sameString(setting, "polished") || sameString(setting, "group")) + return 0; + +/* check setting is in extra file of supported settings */ +if (options->extra && hashLookup(options->extra, setting)) + return 0; + +/* check setting is supported in this version */ struct trackHubSetting *hubSetting = hashFindVal(options->settings, setting); -/* check setting exists in this version */ if (hubSetting == NULL) { - if (options->extra == NULL) - { - dyStringPrintf(errors, "Unknown/unsupported trackDb setting '%s' in hub version '%s'", - setting, options->version); + dyStringPrintf(errors, "Setting '%s' is unknown/unsupported\n", setting); return 1; } - if (hashFindVal(options->extra, setting) == NULL) - { - dyStringPrintf(errors, - "Unknown/unsupported trackDb setting '%s' in hub version '%s' with extras file/url '%s'", - setting, options->version, options->extraFile); - return 1; - } - } -// check level + if (!options->strict) return 0; +// check level if (differentString(hubSetting->level, "core")) { dyStringPrintf(errors, - "Setting '%s' is level '%s' in version '%s' (not 'core')", - setting, hubSetting->level, options->version); + "Setting '%s' is level '%s'\n", setting, hubSetting->level); return 1; } return 0; } static void hubCheckTrackFile(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb) /* Check remote file exists and is of correct type. Wrap this in error catcher */ { char *relativeUrl = trackDbSetting(tdb, "bigDataUrl"); if (relativeUrl != NULL) { char *type = trackDbRequiredSetting(tdb, "type"); char *bigDataUrl = trackHubRelativeUrl(genome->trackDbFile, relativeUrl); verbose(2, "checking %s.%s type %s at %s\n", genome->name, tdb->track, type, bigDataUrl); @@ -110,37 +109,38 @@ else errAbort("unrecognized type %s in genome %s track %s", type, genome->name, tdb->track); freez(&bigDataUrl); } } static int hubCheckTrack(struct trackHub *hub, struct trackHubGenome *genome, struct trackDb *tdb, struct trackHubCheckOptions *options, struct dyString *errors) /* Check track settings and optionally, files */ { int retVal = 0; if (options->settings) { - struct slPair *settings = slPairListFromString(tdb->settings, FALSE); - struct slPair *setting; - for (setting = settings; setting != NULL; setting = setting->next) - { - retVal |= hubCheckTrackSetting(hub, tdb, setting->name, options, errors); - } - /* NOTE: also need to check settings not in this list (other tdb fields) */ + //verbose(3, "Found %d settings to check to spec\n", slCount(settings)); + verbose(3, "Checking track: %s\n", tdb->shortLabel); + verbose(3, "Found %d settings to check to spec\n", hashNumEntries(tdb->settingsHash)); + struct hashEl *hel; + struct hashCookie cookie = hashFirst(tdb->settingsHash); + while ((hel = hashNext(&cookie)) != NULL) + retVal |= hubCheckTrackSetting(hub, tdb, hel->name, options, errors); + /* TODO: ? also need to check settings not in this list (other tdb fields) */ } if (!options->checkFiles) return retVal; struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) { hubCheckTrackFile(hub, genome, tdb); } if (errCatch->gotError) { retVal = 1; dyStringPrintf(errors, "%s", errCatch->message->string); } @@ -159,36 +159,36 @@ int retVal = 0; if (errCatchStart(errCatch)) { tdbList = trackHubTracksForGenome(hub, genome); trackHubPolishTrackNames(hub, tdbList); } errCatchEnd(errCatch); if (errCatch->gotError) { retVal = 1; dyStringPrintf(errors, "%s", errCatch->message->string); } errCatchFree(&errCatch); +verbose(2, "%d tracks in %s\n", slCount(tdbList), genome->name); struct trackDb *tdb; for (tdb = tdbList; tdb != NULL; tdb = tdb->next) { retVal |= hubCheckTrack(hub, genome, tdb, options, errors); } -verbose(2, "%d tracks in %s\n", slCount(tdbList), genome->name); return retVal; } char *trackHubVersionDefault() /* Return current version of trackDb settings spec for hubs */ { // TODO: get from goldenPath/help/trackDb/trackDbHub.current.html return "v1"; // minor rev to v1a, etc. } int trackHubSettingLevel(struct trackHubSetting *spec) /* Get integer for level (core > full > new > deprecated) */ @@ -211,52 +211,62 @@ return trackHubSettingLevel(spec1) - trackHubSettingLevel(spec2); } struct trackHubSetting *trackHubSettingsForVersion(char *version) /* Return list of settings with support level. Version can be version string or spec url */ { if (version == NULL) version = trackHubVersionDefault(); char *specUrl; if (startsWith("http", version)) specUrl = version; else { char buf[256]; - // TODO: local host - safef(buf, sizeof buf, "http://hgwdev-kate.cse.ucsc.edu/goldenPath/help/trackDb/trackDbHub.%s.html", + safef(buf, sizeof buf, + //"http://genome.ucsc.edu/goldenPath/help/trackDb/trackDbHub.%s.html", + // TODO: switch to RR (and move to #define) + "http://hgwdev-kate.cse.ucsc.edu/goldenPath/help/trackDb/trackDbHub.%s.html", version); specUrl = buf; } +verbose(2, "Validating to spec at %s\n", specUrl); struct htmlPage *page = htmlPageGet(specUrl); if (page == NULL) errAbort("Can't open trackDb settings spec %s\n", specUrl); -// TODO: check return status -- this succeeds even for 404's ? -verbose(3, "Opened URL %s\n", specUrl); + +//TODO: apply page validator +//htmlPageValidateOrAbort(page); // would like to use this, but current page doesn't validate +// Would need to replace empty table (replaced by JS) with div, and assure htmlPageValidateOrAbort +// is run on any page change. + +/* TODO: validate this is a trackDbHub spec */ +/* (scan tags for tag->name="span" tag->attribute="id", attr->"value=trackDbHub_version", + * might want to limit to first N tags) */ /* Retrieve specs from file url. * Settings are the first text word within any <code> element nested in * a <div> having * attribute class="format". The support level ('level-*') is the class value of the * <code> tag. * E.g. <div class="format"><code class="level-core">boxedConfig on</code></div> produces: * setting=boxedConfig, class=core */ struct htmlTag *tag, *codeTag; struct htmlAttribute *attr, *codeAttr; struct trackHubSetting *spec, *savedSpec; struct hash *specHash = hashNew(0); -verbose(3, "Found %d tags\n", slCount(page->tags)); +verbose(5, "Found %d tags\n", slCount(page->tags)); int divCount = 0; char buf[256]; for (tag = page->tags; tag != NULL; tag = tag->next) { verbose(6, " TAG: %s\n", tag->name); if (differentWord(tag->name, "DIV")) continue; divCount++; attr = tag->attributes; if (differentWord(attr->name, "class") || differentWord(attr->val, "format")) continue; verbose(7, "Found format: tag %s\n", tag->name); // Look for one or more <code> tags in this format div for (codeTag = tag->next; codeTag != NULL && differentWord(codeTag->name,"/DIV"); codeTag = codeTag->next) @@ -284,90 +294,100 @@ if (savedSpec == NULL) { hashAdd(specHash, spec->name, spec); verbose(6, "added spec %s at level %s\n", spec->name, spec->level); } else if (trackHubSettingLevelCmp(spec, savedSpec) > 0) { hashReplace(specHash, spec->name, spec); verbose(6, "replaced spec %s at level %s, was %s\n", spec->name, spec->level, savedSpec->level); } } } verbose(5, "Found %d <div>'s\n", divCount); struct hashEl *el, *list = hashElListHash(specHash); + verbose(5, "Found %d settings's\n", slCount(list)); slSort(&list, hashElCmp); struct trackHubSetting *specs = NULL; +int coreCt = 0; for (el = list; el != NULL; el = el->next) + { + if (sameString(((struct trackHubSetting *)el->val)->level, "core")) + coreCt++; slAddHead(&specs, el->val); + } slReverse(&specs); +verbose(3, "Found %d supported settings for this version (%d core)\n", + slCount(specs), coreCt); return specs; } int trackHubCheck(char *hubUrl, struct trackHubCheckOptions *options, struct dyString *errors) /* hubCheck - Check a track data hub for integrity. Put errors in dyString. * return 0 if hub has no errors, 1 otherwise * if options->checkTracks is TRUE, check remote files of individual tracks */ { struct errCatch *errCatch = errCatchNew(); struct trackHub *hub = NULL; int retVal = 0; if (errCatchStart(errCatch)) { hub = trackHubOpen(hubUrl, "hub_0"); } errCatchEnd(errCatch); if (errCatch->gotError) { retVal = 1; dyStringPrintf(errors, "%s", errCatch->message->string); } errCatchFree(&errCatch); if (hub == NULL) return 1; -verbose(2, "hub %s\nshortLabel %s\nlongLabel %s\n", hubUrl, hub->shortLabel, hub->longLabel); -verbose(2, "%s has %d elements\n", hub->genomesFile, slCount(hub->genomeList)); if (hub->version != NULL) options->version = hub->version; else if (options->version == NULL) options->version = trackHubVersionDefault(); -options->strict = FALSE; -if (hub->level != NULL) +if (options->strict == FALSE && hub->level != NULL) { - if (sameString(hub->level, "core") || sameString(hub->level, "strict")) + if (sameString(hub->level, "core")) options->strict = TRUE; else if (differentString(hub->level, "all")) { dyStringPrintf(errors, - "Unknown hub support level: %s (expecting 'core' or 'all'). Defaulting to 'all'.", hub->level); + "Unknown hub support level: %s (expecting 'core' or 'all'). Defaulting to 'all'.\n", hub->level); retVal = 1; } } -struct trackHubSetting *settings = NULL; +verbose(2, "Checking hub '%s'%s\n", hub->longLabel, options->strict ? " for compliance to 'core' (use -settings to view)": ""); + errCatch = errCatchNew(); if (errCatchStart(errCatch)) { - settings = trackHubSettingsForVersion(options->version); + /* make hash of settings for this version, saving in options */ + struct trackHubSetting *setting, *settings = trackHubSettingsForVersion(options->version); + options->settings = newHash(0); + for (setting = settings; setting != NULL; setting = setting->next) + hashAdd(options->settings, setting->name, setting); } errCatchEnd(errCatch); if (errCatch->gotError) { retVal = 1; dyStringPrintf(errors, "%s", errCatch->message->string); } errCatchFree(&errCatch); struct trackHubGenome *genome; for (genome = hub->genomeList; genome != NULL; genome = genome->next) { retVal |= hubCheckGenome(hub, genome, options, errors); } trackHubClose(&hub);