06d7be056190c14b85e71bc12523f18ea6815b5e markd Mon Dec 7 00:50:29 2020 -0800 BLAT mmap index support merge with master diff --git src/hg/utils/hubCheck/hubCheck.c src/hg/utils/hubCheck/hubCheck.c index 234e995..7853912 100644 --- src/hg/utils/hubCheck/hubCheck.c +++ src/hg/utils/hubCheck/hubCheck.c @@ -447,30 +447,45 @@ { /* check level */ struct trackHubSettingSpec *checkLevel = NULL; AllocVar(checkLevel); checkLevel->level = options->level; if (trackHubSettingLevel(hubSetting) < trackHubSettingLevel(checkLevel)) { dyStringPrintf(errors, "Setting '%s' is level '%s'\n", setting, hubSetting->level); retVal = 1; } freez(&checkLevel); } return retVal; } +boolean extFileExists(char *path) +/* Check that a remote URL actually exists, path may be a URL or a local path relative to hub.txt */ +{ +// if the file is local check that it exists: +if (!hasProtocol(path) && udcExists(path)) + return TRUE; +else + { + // netLineFileSilentOpen will handle 301 redirects and the like + if (netLineFileSilentOpen(path) != NULL) + return TRUE; + } +return FALSE; +} + char *makeFolderObjectString(char *id, char *text, char *parent, char *title, boolean children, boolean openFolder) /* Construct a folder item for one of the jstree arrays */ { struct dyString *folderString = dyStringNew(0); dyStringPrintf(folderString, "{icon: '../../images/folderC.png', id: '%s', " "text:\"%s\", parent:'%s'," "li_attr:{title:'%s'}, children:%s, state: {opened: %s}}", htmlEncode(id), text, htmlEncode(parent), title, children ? "true" : "false", openFolder ? "true" : "false"); return dyStringCannibalize(&folderString); } char *makeChildObjectString(char *id, char *title, char *shortLabel, char *longLabel, char *color, char *name, char *text, char *parent) /* Construct a single child item for one of the jstree arrays */ { @@ -512,42 +527,47 @@ count++; } } void genomeErr(struct dyString *errors, char *message, struct trackHub *hub, struct trackHubGenome *genome, boolean doHtml) /* Construct the right javascript for the jstree for a top-level genomes.txt error or * error opening a trackDb.txt file */ { if (!doHtml) dyStringPrintf(errors, "%s", message); else { static int count = 0; // forces unique ID's which the jstree object needs char id[512]; + char *errorMessages[16]; char *strippedMessage = NULL; char *genomeName = trackHubSkipHubName(genome->name); if (message) strippedMessage = cloneString(message); - stripChar(strippedMessage, '\n'); + // multiple errors may be in a single message, chop by newline and make a node in the tree for each message + int numMessages = chopByChar(strippedMessage, '\n', errorMessages, sizeof(errorMessages)); + int i = 0; + dyStringPrintf(errors, "trackData['%s'] = [", genomeName); + for (; i < numMessages && isNotEmpty(errorMessages[i]); i++) + { safef(id, sizeof(id), "%s%d", genomeName, count); - - dyStringPrintf(errors, "trackData['%s'] = [%s,", genomeName, - makeChildObjectString(id, "Genome Error", genomeName, genomeName, "#550073", genomeName, strippedMessage, genomeName)); + dyStringPrintf(errors, "%s,", makeChildObjectString(id, "Genome Error", genomeName, genomeName, "#550073", genomeName, errorMessages[i], genomeName)); count++; } } +} void trackDbErr(struct dyString *errors, char *message, struct trackHubGenome *genome, struct trackDb *tdb, boolean doHtml) /* Adds the right object for a jstree object of trackDb configuration errors. */ { if (!doHtml) { dyStringPrintf(errors, "%s", message); } else { char *strippedMessage = NULL; char *splitMessages[16]; // SUBGROUP_MAX=9 but add a few extra just in case char parentOrTrackString[512]; char id[512]; static int count = 0; // forces unique ID's which the jstree object needs @@ -832,47 +852,58 @@ int hubCheckGenome(struct trackHub *hub, struct trackHubGenome *genome, struct trackHubCheckOptions *options, struct dyString *errors) /* Check out genome within hub. */ { struct errCatch *errCatch = errCatchNew(); struct trackDb *tdbList = NULL; int genomeErrorCount = 0; boolean openedGenome = FALSE; if (errCatchStart(errCatch)) { if (genome->twoBitPath != NULL) { + // check that twoBitPath is a valid file, warn instead of errAbort so we can continue checking + // the genome stanza + char *twoBit = genome->twoBitPath; + if (!extFileExists(twoBit)) + warn("Error: '%s' twoBitPath does not exist or is not accessible: '%s'", genome->name, twoBit); + + // groups and htmlPath are optional settings, again only warn if they are malformed + char *groupsFile = genome->groups; + if (groupsFile != NULL && !extFileExists(groupsFile)) + warn("warning: '%s' groups file does not exist or is not accessible: '%s'", genome->name, groupsFile); + char *htmlPath = hashFindVal(genome->settingsHash, "htmlPath"); if (htmlPath == NULL) warn("warning: missing htmlPath setting for assembly hub '%s'", genome->name); - else if ((!hasProtocol(htmlPath) && !udcExists(htmlPath)) || netUrlHead(htmlPath, NULL) < 0) - warn("warning: htmlPath file does not exist: '%s'", htmlPath); + else if (!extFileExists(htmlPath)) + warn("warning: '%s' htmlPath file does not exist or is not accessible: '%s'", genome->name, htmlPath); } tdbList = trackHubTracksForGenome(hub, genome); tdbList = trackDbLinkUpGenerations(tdbList); tdbList = trackDbPolishAfterLinkup(tdbList, genome->name); trackHubPolishTrackNames(hub, tdbList); } errCatchEnd(errCatch); if (errCatch->gotError || errCatch->gotWarning) { openedGenome = TRUE; genomeErr(errors, errCatch->message->string, hub, genome, options->htmlOut); - if (errCatch->gotError) + if (errCatch->gotError || errCatch->gotWarning) genomeErrorCount += 1; } errCatchFree(&errCatch); verbose(2, "%d tracks in %s\n", slCount(tdbList), genome->name); struct trackDb *tdb; int tdbCheckVal; static struct dyString *tdbDyString = NULL; if (!tdbDyString) tdbDyString = dyStringNew(0); // build up the track results list, keep track of number of errors, then // open up genomes folder char *genomeName = trackHubSkipHubName(genome->name); for (tdb = tdbList; tdb != NULL; tdb = tdb->next) @@ -911,37 +942,34 @@ int trackHubCheck(char *hubUrl, struct trackHubCheckOptions *options, struct dyString *errors) /* Check a track data hub for integrity. Put errors in dyString. * return 0 if hub has no errors, 1 otherwise * if options->checkTracks is TRUE, check remote files of individual tracks */ { struct errCatch *errCatch = errCatchNew(); struct trackHub *hub = NULL; struct dyString *hubErrors = dyStringNew(0); int retVal = 0; if (errCatchStart(errCatch)) { hub = trackHubOpen(hubUrl, ""); - // servers that don't return Content-Length header aren't found by udcExists so - // use netUrlHead too. We still want udcExists so if a file is a local file we - // can still find it char *descUrl = hub->descriptionUrl; if (descUrl == NULL) warn("warning: missing hub overview descripton page (descriptionUrl setting)"); - else if ((!hasProtocol(descUrl) && !udcExists(descUrl)) || netUrlHead(descUrl, NULL) < 0) + else if (!extFileExists(descUrl)) warn("warning: %s descriptionUrl setting does not exist", hub->descriptionUrl); } errCatchEnd(errCatch); if (errCatch->gotError || errCatch->gotWarning) { retVal = 1; hubErr(hubErrors, errCatch->message->string, hub, options->htmlOut); if (options->htmlOut) { if (hub && hub->shortLabel) { dyStringPrintf(errors, "trackData['#'] = [%s,", makeFolderObjectString(hub->shortLabel, "Hub Errors", "#", "Click to open node", TRUE, TRUE));