59b4888b34803576809588f0f6189d21e8cf6964 braney Mon Sep 14 15:56:07 2026 -0700 hgTablesTest: one unusable page should not end the whole run, refs #38356 testOneTrack called errAbort whenever hgTables returned a page it could not parse, which ended the run there and then. quickSubmit has already recorded that failure in tablesTestList by the time it returns NULL, so the abort was not preserving information -- it was destroying it, because reportSummary never ran and the Total line carrying the error counts was never written. Every log back to v490 has zero Total lines for this reason. Skip the track instead and keep going. The failure still counts as a hard error in the summary, so a bad page is now reported rather than fatal. The same applies to a track page with no main form or no table var, and to the matching cases in testOneGroup. This subsumes the bigPsl exception added in 2016, and drops a sameString() on a trackDb type that would have crashed had the lookup returned NULL. quickSubmit logged the reason a page was unusable only at verbose level 2, which the robot does not run at, so a failure reached the log as a bare "Couldn't select track X" with the cause discarded. Log it at level 1. That is what identified today's two failures as hgTables emitting truncated HTML rather than anything to do with the tracks themselves. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> diff --git src/hg/hgTablesTest/hgTablesTest.c src/hg/hgTablesTest/hgTablesTest.c index cc5c27a1c0a..8019dd500bf 100644 --- src/hg/hgTablesTest/hgTablesTest.c +++ src/hg/hgTablesTest/hgTablesTest.c @@ -1,1396 +1,1429 @@ /* hgTablesTest - Test hgTables web page. */ /* Copyright (C) 2014 The Regents of the University of California * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */ #include "common.h" #include "memalloc.h" #include "linefile.h" #include "hash.h" #include "htmshell.h" #include "portable.h" #include "options.h" #include "errCatch.h" #include "ra.h" #include "htmlPage.h" #include "../hgTables/hgTables.h" #include "hdb.h" #include "dbDb.h" #include "qa.h" #include "chromInfo.h" #include "obscure.h" #include <unistd.h> #include <limits.h> #ifndef HOST_NAME_MAX // needed for OS/X #define HOST_NAME_MAX _POSIX_HOST_NAME_MAX #endif #define MAX_ATTEMPTS 10 /* Row limit for a table tested WITH position filtering. Far above the 500000 * used when the whole table gets scanned, because a region restricts the output; * this only screens out the handful of whole-genome tables so dense that even one * test region's all-fields output can exceed the carefulAlloc ceiling. */ #define MAX_ROWS_REGION_FILTERED 250000000 /* Command line variables. */ char *clOrg = NULL; /* Organism from command line. */ char *clDb = NULL; /* DB from command line */ char *clGroup = NULL; /* Group from command line. */ char *clTrack = NULL; /* Track from command line. */ char *clTable = NULL; /* Table from command line. */ int clGroups = BIGNUM; /* Number of groups to test. */ int clTracks = 4; /* Number of track to test. */ int clTables = 2; /* Number of tables to test. */ int clDbs = 1; /* Number of databases per organism. */ int clOrgs = 2; /* Number of organisms to test. */ boolean appendLog; /* Append to log rather than create it. */ boolean noShuffle; /* Suppress shuffling of track and table lists. */ void usage() /* Explain usage and exit. */ { errAbort( "hgTablesTest - Test hgTables web page\n" "usage:\n" " hgTablesTest url log\n" "Where url is something like hgwbeta.soe.ucsc.edu/cgi-bin/hgTables\n" "and log is a file where error messages and statistics will be written\n" "options:\n" " -org=Human - Restrict to Human (or Mouse, Fruitfly, etc.)\n" " -db=hg17 - Restrict to particular database\n" " -group=genes - Restrict to a particular group\n" " -track=knownGene - Restrict to a particular track\n" " -table=knownGeneMrna - Restrict to a particular table\n" " -orgs=N - Number of organisms to test. Default %d\n" " -dbs=N - Number of databases per organism to test. Default %d\n" " -groups=N - Number of groups to test (default all)\n" " -tracks=N - Number of tracks per group to test (default %d)\n" " -tables=N - Number of tables per track to test (default %d)\n" " -verbose=N - Set to 0 for silent operation, 2 or 3 for debugging\n" " -appendLog - Append to log file rather than creating it\n" " -seed N - Specify seed for random number generator as debugging aid.\n" " -noShuffle - do not shuffle tracks and tables lists.\n" , clOrgs, clDbs, clTracks, clTables); } FILE *logFile; /* Log file. */ int seed = 0; /* seed for random number generator */ static struct optionSpec options[] = { {"org", OPTION_STRING}, {"db", OPTION_STRING}, {"group", OPTION_STRING}, {"track", OPTION_STRING}, {"table", OPTION_STRING}, {"orgs", OPTION_INT}, {"dbs", OPTION_INT}, {"search", OPTION_STRING}, {"groups", OPTION_INT}, {"tracks", OPTION_INT}, {"tables", OPTION_INT}, {"appendLog", OPTION_BOOLEAN}, {"seed", OPTION_INT}, {"noShuffle", OPTION_BOOLEAN}, {NULL, 0}, }; struct tablesTest /* Test on one column. */ { struct tablesTest *next; struct qaStatus *status; /* Result of test. */ char *info[6]; }; enum tablesTestInfoIx { ntiiType, ntiiOrg, ntiiDb, ntiiGroup, ntiiTrack, ntiiTable, ntiiTotalCount, }; char *tablesTestInfoTypes[] = { "type", "organism", "db", "group", "track", "table"}; struct tablesTest *tablesTestList = NULL; /* List of all tests, latest on top. */ struct tablesTest *tablesTestNew(struct qaStatus *status, char *type, char *org, char *db, char *group, char *track, char *table) /* Save away test test results. */ { struct tablesTest *test; AllocVar(test); test->status = status; test->info[ntiiType] = cloneString(naForNull(type)); test->info[ntiiOrg] = cloneString(naForNull(org)); test->info[ntiiDb] = cloneString(naForNull(db)); test->info[ntiiGroup] = cloneString(naForNull(group)); test->info[ntiiTrack] = cloneString(naForNull(track)); test->info[ntiiTable] = cloneString(naForNull(table)); slAddHead(&tablesTestList, test); return test; } void tablesTestLogOne(struct tablesTest *test, FILE *f) /* Log test result to file. */ { int i; for (i=0; i<ArraySize(test->info); ++i) fprintf(f, "%s ", test->info[i]); fprintf(f, "%s\n", test->status->errMessage); } struct htmlPage *quickSubmit(struct htmlPage *basePage, char *org, char *db, char *group, char *track, char *table, char *testName, char *button, char *buttonVal) /* Submit page and record info. Return NULL if a problem. */ { struct htmlPage *page = NULL; // don't get ahead of the botDelay sleep1000(5000); verbose(2, "quickSubmit(%p, %s, %s, %s, %s, %s, %s, %s, %s)\n", basePage, naForNull(org), naForNull(db), naForNull(group), naForNull(track), naForNull(table), naForNull(testName), naForNull(button), naForNull(buttonVal)); if (basePage != NULL) { struct qaStatus *qs; if (db != NULL) htmlPageSetVar(basePage, NULL, "db", db); if (org != NULL) htmlPageSetVar(basePage, NULL, "org", org); if (group != NULL) htmlPageSetVar(basePage, NULL, hgtaGroup, group); if (track != NULL) htmlPageSetVar(basePage, NULL, hgtaTrack, track); if (table != NULL) htmlPageSetVar(basePage, NULL, hgtaTable, table); qs = qaPageFromForm(basePage, basePage->forms, button, buttonVal, &page); if (!page) { verbose(2, "page is NULL, qs->errMessage=[%s]\n", qs->errMessage); if (startsWith("carefulAlloc: Allocated too much memory", qs->errMessage)) { verbose(1, "Response html page too large (500MB) (%s %s %s %s %s)\n", org, db, group, track, table); fprintf(logFile, "Response html page too large (500MB) (%s %s %s %s %s)\n", org, db, group, track, table); } + else + { + /* Without this the caller reports only which track it was on, and the + * reason the page was unusable is lost unless someone happens to re-run + * at -verbose=2. */ + verbose(1, "No usable page (%s %s %s %s %s): %s\n", + naForNull(org), naForNull(db), naForNull(group), + naForNull(track), naForNull(table), naForNull(qs->errMessage)); + fprintf(logFile, "No usable page (%s %s %s %s %s): %s\n", + naForNull(org), naForNull(db), naForNull(group), + naForNull(track), naForNull(table), naForNull(qs->errMessage)); + } } /* if (page->forms != NULL) htmlFormPrint(page->forms, stdout); */ // do not need to keep the returned structure, the answer is accumulating // in global variable: tablesTestList (void) tablesTestNew(qs, testName, org, db, group, track, table); } return page; } void serialSubmit(struct htmlPage **pPage, char *org, char *db, char *group, char *track, char *table, char *testName, char *button, char *buttonVal) /* Submit page, replacing old page with new one. */ { struct htmlPage *oldPage = *pPage; if (oldPage != NULL) { *pPage = quickSubmit(oldPage, org, db, group, track, table, testName, button, buttonVal); htmlPageFree(&oldPage); } } int tableSize(char *db, char *table) /* Return number of rows in table, or -1 if there is no such SQL table. * Plenty of tracks are offered in the table list but keep their data in a file * rather than a SQL table -- bigBed, bigWig, bigMaf and the chain/net and multiz * tracks built on them -- and counting rows on one of those used to abort the * whole process. hg38.multiz11way and mm39.netHs1 each ended a run this way. * A caller comparing against a row limit treats -1 as "no limit to apply". */ { struct sqlConnection *conn = sqlConnect(db); long size = sqlTableSizeIfExists(conn, table); sqlDisconnect(&conn); return size; } void showConnectInfo(char *db) /* Show connection info used by this program. */ { struct sqlConnection *conn = sqlConnect(db); char query[1024]; sqlSafef(query, sizeof query, "select current_user()"); char *user = sqlQuickString(conn, query); char *hostinfo = sqlHostInfo(conn); verbose(1, "Connecting as %s to database server %s\n", user, hostinfo); fprintf(logFile, "Connecting as %s to database server %s\n", user, hostinfo); fflush(logFile); sqlDisconnect(&conn); } void showRunningHostName() /* Show hostname of the machine we are running on. */ { char hostname[HOST_NAME_MAX]; if (gethostname(hostname, sizeof hostname)) { perror("gethostname"); safecpy(hostname, sizeof hostname, "error-reading-hostname"); } verbose(1, "Running on machine %s\n", hostname); fprintf(logFile, "Running on machine %s\n", hostname); fflush(logFile); } void quickErrReport() /* Report error at head of list if any */ { struct tablesTest *test = tablesTestList; if (test->status->errMessage != NULL) tablesTestLogOne(test, stderr); } void testSchema(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Make sure schema page comes up. */ /* mainForm not used */ { struct htmlPage *schemaPage = quickSubmit(tablePage, org, db, group, track, table, "schema", hgtaDoSchema, "submit"); htmlPageFree(&schemaPage); } void testSummaryStats(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Make sure summary stats page comes up. */ { if (htmlFormVarGet(mainForm, hgtaDoSummaryStats) != NULL) { struct htmlPage *statsPage = quickSubmit(tablePage, org, db, group, track, table, "summaryStats", hgtaDoSummaryStats, "submit"); htmlPageFree(&statsPage); } } boolean varIncludesType(struct htmlForm *form, char *var, char *value) /* Return TRUE if value is one of the options for var. */ { struct htmlFormVar *formVar = htmlFormVarGet(form, var); if (formVar == NULL) errAbort("Couldn't find %s variable in form", var); return slNameInList(formVar->values, value); } boolean outTypeAvailable(struct htmlForm *form, char *value) /* Return true if outType options include value. */ { return varIncludesType(form, hgtaOutputType, value); } int countNoncommentLines(char *s) /* Count number of lines in s that don't start with # */ { int count = 0; // beware, skipLeadingSpaces skips '\n' (isspace) so it skips blank lines. Check for '\n' first. if (s && *s != '\n') s = skipLeadingSpaces(s); while (s != NULL && s[0] != 0) { if (s[0] != '#') ++count; s = strchr(s, '\n'); if (s != NULL) { s++; if (*s != '\n') s = skipLeadingSpaces(s); } } return count; } int testAllFields(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Get all fields and return count of rows. */ /* mainForm not used */ { struct htmlPage *outPage; int rowCount = 0; htmlPageSetVar(tablePage, NULL, hgtaOutputType, "primaryTable"); outPage = quickSubmit(tablePage, org, db, group, track, table, "allFields", hgtaDoTopSubmit, "submit"); /* check for NULL outPage */ if (outPage == NULL) { verbose(1, "Null page in testAllFields (%s %s %s %s %s)\n", org, db, group, track, table); fprintf(logFile, "Null page in testAllFields (%s %s %s %s %s)\n", org, db, group, track, table); return -1; } rowCount = countNoncommentLines(outPage->htmlText); htmlPageFree(&outPage); return rowCount; } struct htmlFormVar *findPrefixedVar(struct htmlFormVar *list, char *prefix) /* Find first var with given prefix in list. */ { struct htmlFormVar *var; for (var = list; var != NULL; var = var->next) { if (startsWith(prefix, var->name)) return var; } return NULL; } void checkExpectedSimpleRows(struct htmlPage *outPage, int expectedRows) /* Make sure we got the number of rows we expect. */ { if (outPage != NULL) { char *results = outPage->htmlText; if (startsWith(NO_RESULTS, results)) results += strlen(NO_RESULTS); int rowCount = countNoncommentLines(results); if (rowCount != expectedRows) qaStatusSoftError(tablesTestList->status, "Got %d rows, expected %d", rowCount, expectedRows); } } void testOneField(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table, int expectedRows) /* Get one field and make sure the count agrees with expected. */ /* mainForm not used */ { struct htmlPage *outPage; struct htmlForm *form; struct htmlFormVar *var; int attempts = 0; if (tablePage->forms == NULL) errAbort("testOneField: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "selectedFields"); outPage = quickSubmit(tablePage, org, db, group, track, table, "selFieldsPage", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOneField: trying again to get selFieldsPage\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "selFieldsPage", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOneField - couldn't get outPage."); return; } if (outPage->forms == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOneField - missing form."); htmlPageFree(&outPage); return; } form = outPage->forms; var = findPrefixedVar(form->vars, "hgta_fs.check."); if (var == NULL) errAbort("No hgta_fs.check. vars in form"); htmlPageSetVar(outPage, NULL, var->name, "on"); serialSubmit(&outPage, org, db, group, track, table, "oneField", hgtaDoPrintSelectedFields, "submit"); // check that outPage != NULL checkExpectedSimpleRows(outPage, expectedRows); htmlPageFree(&outPage); } void testOutBed(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table, int expectedRows) /* Get as bed and make sure count agrees with expected. */ /* mainForm not used */ { struct htmlPage *outPage; int attempts = 0; if (tablePage->forms == NULL) errAbort("testOutBed: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "bed"); outPage = quickSubmit(tablePage, org, db, group, track, table, "bedUiPage", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOutBed: trying again to get bedUiPage\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "bedUiPage", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOneBed - couldn't get outPage."); return; } if (outPage->forms == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOneBed - missing form."); htmlPageFree(&outPage); return; } serialSubmit(&outPage, org, db, group, track, table, "outBed", hgtaDoGetBed, "submit"); // check that outPage != NULL checkExpectedSimpleRows(outPage, expectedRows); htmlPageFree(&outPage); } void testOutGff(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Get as GFF and make sure no crash. */ /* mainForm not used */ { struct htmlPage *outPage; if (tablePage->forms == NULL) errAbort("testOutGff: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "gff"); outPage = quickSubmit(tablePage, org, db, group, track, table, "outGff", hgtaDoTopSubmit, "submit"); /* no checking here */ if (outPage != NULL) htmlPageFree(&outPage); } int countTagsBetween(struct htmlPage *page, char *start, char *end, char *type) /* Count number of tags of given type (which should be upper case) * between start and end. If start is NULL it will start from * beginning of page. If end is NULL it will end at end of page. */ { int count = 0; struct htmlTag *tag; if (start == NULL) start = page->htmlText; if (end == NULL) end = start + strlen(start); for (tag = page->tags; tag != NULL; tag = tag->next) { if (tag->start >= start && tag->start < end && sameString(tag->name, type)) { ++count; } } return count; } void testOutHyperlink(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table, int expectedRows) /* Get as hyperlink and make sure count agrees with expected. */ /* mainForm not used */ { struct htmlPage *outPage; int attempts = 0; char *s; int rowCount; if (tablePage->forms == NULL) errAbort("testOutHyperlink: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "hyperlinks"); outPage = quickSubmit(tablePage, org, db, group, track, table, "outHyperlinks", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOutHyperLink: trying again to get outHyperLinks\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "outHyperlinks", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutHyperLink - couldn't get outPage."); return; } s = stringIn("<!-- +++++++++++++++++++++ CONTENT TABLES +++++++++++++++++++ -->", outPage->htmlText); if (s == NULL) errAbort("Can't find <!-- +++++++++++++++++++++ CONTENT TABLES +++++++++++++++++++ -->"); rowCount = countTagsBetween(outPage, s, NULL, "A"); if (rowCount != expectedRows) qaStatusSoftError(tablesTestList->status, "Got %d rows, expected %d", rowCount, expectedRows); htmlPageFree(&outPage); } void testOutCustomTrack(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Get as customTrack and make sure nothing explodes. */ /* mainForm not used */ { struct htmlPage *outPage; int attempts = 0; struct htmlFormVar *groupVar; if (tablePage->forms == NULL) errAbort("testOutCustomTrack: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "customTrack"); outPage = quickSubmit(tablePage, org, db, group, track, table, "customTrackUi", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOutCustomTrack: trying again to get customTrackUi\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "customTrackUi", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutCustomTrack - couldn't get outPage."); return; } serialSubmit(&outPage, org, db, group, track, table, "outCustom", hgtaDoGetCustomTrackTb, "submit"); if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutCustomTrack - serialSubmit returned null page."); return; } if (outPage->forms == NULL) { qaStatusSoftError(tablesTestList->status, "Error in custom track - no form produced."); htmlPageFree(&outPage); return; } groupVar = htmlFormVarGet(outPage->forms, hgtaGroup); if (!slNameInList(groupVar->values, "user")) { qaStatusSoftError(tablesTestList->status, "No custom track group after custom track submission"); } htmlPageFree(&outPage); } void checkFaOutput(struct htmlPage *page, int expectedCount, boolean lessOk) /* Check that page contains expected number of sequences. If lessOk is set * (needed to handle some multiply mapped cases in refSeq) then just check * that have at least one if expecting any. */ { if (page != NULL) { int count = countChars(page->htmlText, '>'); if (count != expectedCount) { if (!lessOk || count > expectedCount || (expectedCount > 0 && count <= 0)) qaStatusSoftError(tablesTestList->status, "Got %d sequences, expected %d", count, expectedCount); } } } void testOutSequence(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table, int expectedRows) /* Get as sequence and make sure count agrees with expected. */ /* mainForm not used */ { struct htmlPage *outPage; int attempts = 0; struct htmlFormVar *typeVar; if (tablePage->forms == NULL) errAbort("testOutSequence: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "sequence"); outPage = quickSubmit(tablePage, org, db, group, track, table, "seqUi1", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOutSequence: trying again to get seqUi1\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "seqUi1", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutSequence - couldn't get outPage"); return; } if (outPage->forms == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutSequence - missing form"); htmlPageFree(&outPage); return; } /* Since some genomic sequence things are huge, this will * only test in case where it's a gene prediction. */ typeVar = htmlFormVarGet(outPage->forms, hgtaGeneSeqType); if (typeVar != NULL) { struct htmlPage *seqPage; static char *types[] = {"protein", "mRNA"}; int i; for (i=0; i<ArraySize(types); ++i) { char *type = types[i]; if (slNameInList(typeVar->values, type)) { struct htmlPage *page; char testName[128]; htmlPageSetVar(outPage, NULL, hgtaGeneSeqType, type); safef(testName, sizeof(testName), "%sSeq", type); page = quickSubmit(outPage, org, db, group, track, table, testName, hgtaDoGenePredSequence, "submit"); checkFaOutput(page, expectedRows, TRUE); htmlPageFree(&page); } } htmlPageSetVar(outPage, NULL, hgtaGeneSeqType, "genomic"); serialSubmit(&outPage, org, db, group, track, table, "seqUi2", hgtaDoGenePredSequence, "submit"); // check that outPage != NULL /* On genomic page uncheck intron if it's there, then get results * and count them. */ if (htmlFormVarGet(outPage->forms, "hgSeq.intron") != NULL) htmlPageSetVar(outPage, NULL, "hgSeq.intron", NULL); seqPage = quickSubmit(outPage, org, db, group, track, table, "genomicSeq", hgtaDoGenomicDna, "submit"); // check that seqPage != NULL checkFaOutput(seqPage, expectedRows, FALSE); htmlPageFree(&seqPage); } htmlPageFree(&outPage); } boolean isObsolete(char *table) /* Some old table types we can't handle. Just warn that * they are there and skip. */ { boolean obsolete = sameString(table, "wabaCbr"); if (obsolete) qaStatusSoftError(tablesTestList->status, "Skipping obsolete table %s", table); return obsolete; } void testOneTable(struct htmlPage *trackPage, char *org, char *db, char *group, char *track, char *table) /* Test stuff on one table if we haven't already tested this table. */ { /* Why declared here and not globally? */ static struct hash *uniqHash = NULL; char fullName[256]; if (uniqHash == NULL) uniqHash = newHash(0); safef(fullName, sizeof(fullName), "%s.%s", db, table); if (!hashLookup(uniqHash, fullName)) { struct htmlPage *tablePage; struct htmlForm *mainForm; hashAdd(uniqHash, fullName, NULL); verbose(1, "Testing %s %s %s %s %s\n", naForNull(org), db, group, track, table); tablePage = quickSubmit(trackPage, org, db, group, track, table, "selectTable", hgtaTable, table); if (!isObsolete(table) && tablePage != NULL) { if ((mainForm = htmlFormGet(tablePage, "mainForm")) == NULL) { qaStatusSoftError(tablesTestList->status, "Couldn't get main form on tablePage for %s %s %s %s", db, group, track, table); } else { verbose(3, "testOneTable testSchema() got here 1.1\n"); testSchema(tablePage, mainForm, org, db, group, track, table); verbose(3, "testOneTable testSummaryStats() got here 1.2\n"); testSummaryStats(tablePage, mainForm, org, db, group, track, table); verbose(3, "testOneTable got here 1.3\n"); if (outTypeAvailable(mainForm, "bed")) { verbose(3, "testOneTable bed output avail means can filter on position got here 2\n"); /* A region bounds the output for almost every table, but a whole-genome * table of hundreds of millions of rows is dense enough that all-fields * output for the test region can still pass the carefulAlloc ceiling. * carefulAlloc exits the process outright, which forfeits every table * left in the run, so screen the worst offenders out the way the * no-position-filter branch below does. */ int tableRows = tableSize(db, table); if (tableRows >= MAX_ROWS_REGION_FILTERED) { verbose(1, "%s.%s tableRows=%d, too large >= %d even with position filtering, skipping.\n", db, table, tableRows, MAX_ROWS_REGION_FILTERED); fprintf(logFile, "%s.%s tableRows=%d, too large >= %d even with position filtering, skipping.\n", db, table, tableRows, MAX_ROWS_REGION_FILTERED); } else if (outTypeAvailable(mainForm, "primaryTable")) { verbose(3, "testOneTable got here 3\n"); int rowCount = testAllFields(tablePage, mainForm, org, db, group, track, table); if (rowCount >= 0) { testOneField(tablePage, mainForm, org, db, group, track, table, rowCount); testOutSequence(tablePage, mainForm, org, db, group, track, table, rowCount); testOutBed(tablePage, mainForm, org, db, group, track, table, rowCount); testOutHyperlink(tablePage, mainForm, org, db, group, track, table, rowCount); testOutGff(tablePage, mainForm, org, db, group, track, table); if (rowCount > 0) testOutCustomTrack(tablePage, mainForm, org, db, group, track, table); } } } else if (outTypeAvailable(mainForm, "primaryTable")) { verbose(3, "testOneTable no bed output available, so no position filtering available. got here 4\n"); /* If BED type is not available then the region will be ignored, and * we'll end up scanning whole table. Make sure table is not huge * before proceeding. */ int tableRows = tableSize(db, table); if (tableRows < 500000) { int rowCount = testAllFields(tablePage, mainForm, org, db, group, track, table); if (rowCount >= 0) testOneField(tablePage, mainForm, org, db, group, track, table, rowCount); } else { verbose(1, "%s.%s tableRows=%d, too large >= 500000, skipping.\n", db, table, tableRows); fprintf(logFile, "%s.%s tableRows=%d, too large >= 500000, skipping.\n", db, table, tableRows); } } } htmlPageFree(&tablePage); } carefulCheckHeap(); } } void testOneTrack(struct htmlPage *groupPage, char *org, char *db, char *group, char *track, int maxTables) /* Test a little something on up to maxTables in one track. */ { struct htmlPage *trackPage = quickSubmit(groupPage, org, db, group, track, NULL, "selectTrack", hgtaTrack, track); struct htmlForm *mainForm; struct htmlFormVar *tableVar; struct slName *table; int tableIx; +/* A track whose page does not come back, or comes back unusable, is skipped + * rather than fatal. quickSubmit has already recorded the failure in + * tablesTestList, so it is counted in the final summary either way. Aborting + * here used to end the whole run, which meant the summary that carries the + * error counts was never written at all, and a single bad page - often just a + * truncated response from a busy server - threw away an hour of testing. This + * also covers the old bigPsl special case (2016-06-20). */ if (trackPage == NULL) { - // is this an exception? - // exception for bigPsl (2016-06-20), may be short-lived. - struct sqlConnection *conn = sqlConnect(db); - char query[256]; - sqlSafef(query, sizeof query, "select type from trackDb where tableName='%s'", track); - char *type = sqlQuickString(conn, query); - sqlDisconnect(&conn); - if (sameString(type, "bigPsl")) - { - verbose(1, "Skipping testing track %s since type bigPsl not supported by hgTables at this time (2016-06-20)\n", track); - fprintf(logFile, "Skipping testing track %s since type bigPsl not supported by hgTables at this time (2016-06-20)\n", track); + verbose(1, "Skipping track %s: no page returned\n", track); + fprintf(logFile, "Skipping track %s: no page returned\n", track); return; } - else - errAbort("Couldn't select track %s", track); - } if ((mainForm = htmlFormGet(trackPage, "mainForm")) == NULL) - errAbort("Couldn't get main form on trackPage"); + { + verbose(1, "Skipping track %s: no main form on track page\n", track); + fprintf(logFile, "Skipping track %s: no main form on track page\n", track); + htmlPageFree(&trackPage); + return; + } if ((tableVar = htmlFormVarGet(mainForm, hgtaTable)) == NULL) - errAbort("Can't find table var"); + { + verbose(1, "Skipping track %s: no table var on track page\n", track); + fprintf(logFile, "Skipping track %s: no table var on track page\n", track); + htmlPageFree(&trackPage); + return; + } // put the tables in random order: if (!noShuffle) shuffleList(&tableVar->values); for (table = tableVar->values, tableIx = 0; table != NULL && tableIx < maxTables; table = table->next) { if (clTable && !sameString(clTable, table->name)) continue; testOneTable(trackPage, org, db, group, track, table->name); ++tableIx; } /* Clean up. */ htmlPageFree(&trackPage); } void testOneGroup(struct htmlPage *dbPage, char *org, char *db, char *group, int maxTracks) /* Test a little something on up to maxTracks in one group */ { struct htmlPage *groupPage = quickSubmit(dbPage, org, db, group, NULL, NULL, "selectGroup", hgtaGroup, group); struct htmlForm *mainForm; struct htmlFormVar *trackVar; struct slName *track; int trackIx; +/* As in testOneTrack, a group we cannot read is skipped rather than fatal, so + * that one bad page does not cost the whole run. */ if (groupPage == NULL) - errAbort("Error when changing group to %s", group); + { + verbose(1, "Skipping group %s: no page returned\n", group); + fprintf(logFile, "Skipping group %s: no page returned\n", group); + return; + } if ((mainForm = htmlFormGet(groupPage, "mainForm")) == NULL) - errAbort("Couldn't get main form on groupPage"); + { + verbose(1, "Skipping group %s: no main form on group page\n", group); + fprintf(logFile, "Skipping group %s: no main form on group page\n", group); + htmlPageFree(&groupPage); + return; + } if ((trackVar = htmlFormVarGet(mainForm, hgtaTrack)) == NULL) - errAbort("Can't find track var"); + { + verbose(1, "Skipping group %s: no track var on group page\n", group); + fprintf(logFile, "Skipping group %s: no track var on group page\n", group); + htmlPageFree(&groupPage); + return; + } // put the tracks in random order: if (!noShuffle) shuffleList(&trackVar->values); for (track = trackVar->values, trackIx = 0; track != NULL && trackIx < maxTracks; track = track->next) { if (clTrack && !sameString(track->name, clTrack)) continue; testOneTrack(groupPage, org, db, group, track->name, clTables); ++trackIx; } /* Clean up. */ htmlPageFree(&groupPage); } void testGroups(struct htmlPage *dbPage, char *org, char *db, int maxGroups) /* Test a little something in all groups for dbPage. */ { struct htmlForm *mainForm; struct htmlFormVar *groupVar; struct slName *group; int groupIx; if ((mainForm = htmlFormGet(dbPage, "mainForm")) == NULL) errAbort("Couldn't get main form on dbPage"); if ((groupVar = htmlFormVarGet(mainForm, hgtaGroup)) == NULL) errAbort("Can't find group var"); for (group = groupVar->values, groupIx=0; group != NULL && groupIx < maxGroups; group = group->next) { if (!sameString("allTables", group->name)) { if (clGroup && !sameString(clGroup, group->name)) continue; testOneGroup(dbPage, org, db, group->name, clTracks); ++groupIx; } } } void getTestRegion(char *db, char region[256], int regionSize) /* Look up first chromosome in database and grab five million bases * from the middle of it. */ { struct sqlConnection *conn = sqlConnect(db); char query[1024]; sqlSafef(query, sizeof query, "select * from chromInfo limit 1"); struct sqlResult *sr = sqlGetResult(conn, query); char **row; struct chromInfo ci; int start,end,middle; if ((row = sqlNextRow(sr)) == NULL) errAbort("Couldn't get one row from chromInfo"); chromInfoStaticLoad(row, &ci); middle = ci.size/2; start = middle-2500000; end = middle+2500000; if (start < 0) start = 0; if (end > ci.size) end = ci.size; safef(region, regionSize, "%s:%d-%d", ci.chrom, start+1, end); verbose(1, "Testing %s at position %s\n", db, region); fprintf(logFile, "Testing %s at position %s\n", db, region); sqlFreeResult(&sr); sqlDisconnect(&conn); } /* The hgTables page no longer carries clade/organism/assembly dropdowns -- the * gateway moved to a search box -- so there are no <option> tags left to scrape * for the list of organisms and assemblies. Read them from hgcentral dbDb * instead, the way the browser itself does. Only the *enumeration* was lost; * htmlFormVarSet still creates org/db on the form, so submitting works. */ boolean isTestableDb(struct dbDb *db) /* TRUE if this assembly is one we can actually drive: active, and with a * chromInfo table, which getTestRegion needs to pick a test region. GenArk * entries (hs1 and the like) are listed in dbDb and do have a database, but * keep their data in files rather than SQL tables, so they have no chromInfo * and used to be absent from the assembly dropdown we no longer read. */ { return db->active && hTableExists(db->name, "chromInfo"); } struct slName *organismsToTest(int maxOrgs) /* Return up to maxOrgs distinct testable organism names, in dbDb order. */ { struct slName *list = NULL; struct dbDb *dbList = hDbDbList(), *db; struct hash *seen = newHash(8); int count = 0; for (db = dbList; db != NULL && count < maxOrgs; db = db->next) { if (isTestableDb(db) && !hashLookup(seen, db->organism)) { hashAdd(seen, db->organism, NULL); slNameAddTail(&list, db->organism); ++count; } } hashFree(&seen); dbDbFreeList(&dbList); return list; } struct slName *dbsForOrganism(char *org, int maxDbs) /* Return up to maxDbs testable assembly names for organism, in dbDb order. */ { struct slName *list = NULL; struct dbDb *dbList = hDbDbList(), *db; int count = 0; for (db = dbList; db != NULL && count < maxDbs; db = db->next) { if (isTestableDb(db) && sameWord(db->organism, org)) { slNameAddTail(&list, db->name); ++count; } } dbDbFreeList(&dbList); return list; } void testDb(struct htmlPage *orgPage, char *org, char *db) /* Test on one database. */ { struct htmlPage *dbPage; char region[256]; htmlPageSetVar(orgPage, NULL, "db", db); getTestRegion(db, region, sizeof(region)); htmlPageSetVar(orgPage, NULL, "position", region); htmlPageSetVar(orgPage, NULL, hgtaRegionType, "range"); dbPage = quickSubmit(orgPage, org, db, NULL, NULL, NULL, "selectDb", "submit", "go"); if (dbPage != NULL) testGroups(dbPage, org, db, clGroups); htmlPageFree(&dbPage); } void testOrg(struct htmlPage *rootPage, struct htmlForm *rootForm, char *org) /* Test on organism. */ { struct slName *dbList, *db; /* There is no organism round-trip left to make: the gateway's clade/organism/ * assembly dropdowns and the "Go" button that submitted them are gone from the * page, so submitting for an "organism page" just yields a page with no * mainForm. Name the assembly directly instead, the way the -db= path does -- * testDb sets db, position and region type on each pass, so rootPage needs no * preparation here. */ dbList = dbsForOrganism(org, clDbs); if (dbList == NULL) errAbort("No active assembly in dbDb for organism %s", org); for (db = dbList; db != NULL; db = db->next) { testDb(rootPage, org, db->name); } slNameFreeList(&dbList); } void verifyJoinedFormat(char *s) /* Verify that s consists of lines with two tab-separated fields, * and that the second field has some n/a and some comma-separated lists. */ { char *e; int lineIx = 0; boolean gotCommas = FALSE, gotNa = FALSE; while (s != NULL && s[0] != 0) { char *row[3]; int fieldCount; ++lineIx; e = strchr(s, '\n'); if (e != NULL) *e++ = 0; if (s[0] != '#') { fieldCount = chopTabs(s, row); if (fieldCount != 2) { qaStatusSoftError(tablesTestList->status, "Got %d fields line %d of joined result, expected 2", fieldCount, lineIx); break; } if (sameString(row[1], "n/a")) gotNa = TRUE; if (countChars(row[1], ',') >= 2) gotCommas = TRUE; } s = e; } if (!gotCommas) qaStatusSoftError(tablesTestList->status, "Expected some rows in join to have comma separated lists."); if (!gotNa) qaStatusSoftError(tablesTestList->status, "Expected some rows in join to have n/a."); } void testJoining(struct htmlPage *rootPage) /* Simulate pressing buttons to get a reasonable join on a * couple of uniProt tables. */ { struct htmlPage *allPage, *page; char *org = NULL, *db = NULL, *group = "allTables", *track="uniProt"; int expectedCount = tableSize("uniProt", "taxon"); allPage = quickSubmit(rootPage, org, db, group, "uniProt", "uniProt.taxon", "taxonJoin1", NULL, NULL); if (allPage != NULL) { if (allPage->forms == NULL) { errAbort("uniProt page with no form"); } else { int count = testAllFields(allPage, allPage->forms, org, db, group, track, "uniProt.taxon"); if (count != expectedCount) qaStatusSoftError(tablesTestList->status, "Got %d rows in uniProt.taxon, expected %d", count, expectedCount); htmlPageSetVar(allPage, NULL, hgtaOutputType, "selectedFields"); page = quickSubmit(allPage, org, db, group, track, "uniProt.taxon", "taxonJoin2", hgtaDoTopSubmit, "submit"); htmlPageSetVar(page, NULL, "hgta_fs.linked.uniProt.commonName", "on"); serialSubmit(&page, org, db, group, track, NULL, "taxonJoin3", hgtaDoSelectFieldsMore, "submit"); if (page != NULL) { htmlPageSetVar(page, NULL, "hgta_fs.check.uniProt.taxon.binomial", "on"); htmlPageSetVar(page, NULL, "hgta_fs.check.uniProt.commonName.val", "on"); serialSubmit(&page, org, db, group, track, NULL, "taxonJoin4", hgtaDoPrintSelectedFields, "submit"); if (page != NULL) { checkExpectedSimpleRows(page, expectedCount); verifyJoinedFormat(page->htmlText); htmlPageFree(&page); } } } } htmlPageFree(&allPage); verbose(1, "Tested joining on uniProt.taxon & commonName\n"); } void checkXenopus(char *s) /* Check that all lines start with xenopus, and that we * see laevis in there somewhere. */ { char *e; boolean gotLaevis = FALSE; while (s != NULL && s[0] != 0) { s = skipLeadingSpaces(s); e = strchr(s, '\n'); if (e != NULL) *e++ = 0; if (s[0] != '#') { char *t = strchr(s, '\t'); if (t != NULL) *t = 0; if (!startsWith("Xenopus", s)) { qaStatusSoftError(tablesTestList->status, "Xenopus filter passing non-Xenopus"); return; } if (sameString(s, "Xenopus laevis")) gotLaevis = TRUE; } s = e; } if (!gotLaevis) qaStatusSoftError(tablesTestList->status, "Can't find Xenopus laevis in filtered uniProt.taxon"); } void testFilter(struct htmlPage *rootPage) /* Simulate pressing buttons to get a reasonable filter on * uniProt taxon. */ { char *org = NULL, *db = NULL, *group = "allTables", *track="uniProt", *table = "uniProt.taxon"; struct htmlPage *page; page = quickSubmit(rootPage, org, db, group, "uniProt", table, "taxonFilter1", hgtaDoFilterPage, "submit"); if (page != NULL) { struct htmlFormVar *var = htmlFormVarGet(page->forms, "hgta_fil.v.uniProt.taxon.binomial.pat"); if (var == NULL) internalErr(); htmlPageSetVar(page, NULL, "hgta_fil.v.uniProt.taxon.binomial.pat", "Xenopus*"); serialSubmit(&page, org, db, group, track, table, "taxonFilter2", hgtaDoFilterSubmit, "submit"); if (page != NULL) { htmlPageSetVar(page, NULL, hgtaOutputType, "selectedFields"); serialSubmit(&page, org, db, group, track, table, "taxonFilter3", hgtaDoTopSubmit, "submit"); if (page != NULL) { htmlPageSetVar(page, NULL, "hgta_fs.check.uniProt.taxon.binomial", "on"); serialSubmit(&page, org, db, group, track, table, "taxonFilter4", hgtaDoPrintSelectedFields, "submit"); if (page != NULL) checkXenopus(page->htmlText); htmlPageFree(&page); } } } verbose(1, "Tested filter on uniProt.taxon\n"); } void testIdentifier(struct htmlPage *rootPage) /* Do simple check on identifiers. Relies on * 8355 Xenopus laevis being stable taxon (and not being filtered out * by testFilter). */ { char *org = NULL, *db = NULL, *group = "allTables", *track="uniProt", *table = "uniProt.taxon"; struct htmlPage *page; page = quickSubmit(rootPage, org, db, group, "uniProt", table, "taxonId1", hgtaDoPasteIdentifiers, "submit"); if (page != NULL) { htmlPageSetVar(page, NULL, hgtaPastedIdentifiers, "8355"); serialSubmit(&page, org, db, group, track, table, "taxonId2", hgtaDoPastedIdentifiers, "submit"); if (page != NULL) { htmlPageSetVar(page, NULL, hgtaOutputType, "selectedFields"); serialSubmit(&page, org, db, group, track, table, "taxonId3", hgtaDoTopSubmit, "submit"); if (page != NULL) { htmlPageSetVar(page, NULL, "hgta_fs.check.uniProt.taxon.binomial", "on"); serialSubmit(&page, org, db, group, track, table, "taxonId4", hgtaDoPrintSelectedFields, "submit"); if (page != NULL) { if (!stringIn("Xenopus laevis", page->htmlText)) { qaStatusSoftError(tablesTestList->status, "Can't find Xenopus laevis in uniProt.taxon #8355"); } checkExpectedSimpleRows(page, 1); } htmlPageFree(&page); } } } verbose(1, "Tested identifier on uniProt.taxon\n"); } void statsOnSubsets(struct tablesTest *list, int subIx, FILE *f) /* Report tests of certain subtype. */ { struct tablesTest *test; struct hash *hash = newHash(0); struct slName *typeList = NULL, *type; fprintf(f, "\n%s subtotals\n", tablesTestInfoTypes[subIx]); /* Get list of all types in this field. */ for (test = list; test != NULL; test = test->next) { char *info = test->info[subIx]; if (!hashLookup(hash, info)) { type = slNameNew(info); hashAdd(hash, info, type); slAddHead(&typeList, type); } } slNameSort(&typeList); hashFree(&hash); for (type = typeList; type != NULL; type = type->next) { struct qaStatistics *stats; AllocVar(stats); for (test = list; test != NULL; test = test->next) { if (sameString(type->name, test->info[subIx])) { qaStatisticsAdd(stats, test->status); } } qaStatisticsReport(stats, type->name, f); freez(&stats); } } void reportSummary(struct tablesTest *list, FILE *f) /* Report summary of test results. */ { struct qaStatistics *stats; struct tablesTest *test; int i; AllocVar(stats); for (i=0; i<ntiiTotalCount; ++i) statsOnSubsets(list, i, f); for (test = list; test != NULL; test = test->next) qaStatisticsAdd(stats, test->status); fprintf(f, "\ngrand total\n"); qaStatisticsReport(stats, "Total", f); } void reportAll(struct tablesTest *list, FILE *f) /* Report all tests. */ { struct tablesTest *test; for (test = list; test != NULL; test = test->next) { if (test->status->errMessage != NULL) tablesTestLogOne(test, f); } } void hgTablesTest(char *url, char *logName) /* hgTablesTest - Test hgTables web page. */ { /* Get default page, and open log. */ struct htmlPage *rootPage = htmlPageGet(url); if (appendLog) logFile = mustOpen(logName, "a"); else logFile = mustOpen(logName, "w"); if (! endsWith(url, "hgTables")) warn("Warning: first argument should be a complete URL to hgTables, " "but doesn't look like one (%s)", url); fprintf(logFile,"seed=%d\n",seed); showRunningHostName(); verbose(1, "Testing URL %s\n", url); fprintf(logFile, "Testing URL %s\n", url); /* Show what database server we are connecting to. Matters for expected rows in tables. */ showConnectInfo("uniProt"); htmlPageValidateOrAbort(rootPage); /* Go test what they've specified in command line. */ if (clDb != NULL) { testDb(rootPage, NULL, clDb); } else { struct htmlForm *mainForm; if ((mainForm = htmlFormGet(rootPage, "mainForm")) == NULL) errAbort("Couldn't get main form"); if (clOrg != NULL) testOrg(rootPage, mainForm, clOrg); else { struct slName *orgList = organismsToTest(clOrgs), *org; if (orgList == NULL) errAbort("No active organisms in dbDb"); for (org = orgList; org != NULL; org = org->next) { testOrg(rootPage, mainForm, org->name); } slNameFreeList(&orgList); } } /* Do some more complex tests on uniProt. */ testJoining(rootPage); testFilter(rootPage); testIdentifier(rootPage); /* Clean up and report. */ htmlPageFree(&rootPage); slReverse(&tablesTestList); reportSummary(tablesTestList, stdout); reportAll(tablesTestList, logFile); fprintf(logFile, "---------------------------------------------\n"); reportSummary(tablesTestList, logFile); } int main(int argc, char *argv[]) /* Process command line. */ { pushCarefulMemHandler(500000000); optionInit(&argc, argv, options); if (argc != 3) usage(); seed = optionInt("seed",time(NULL)); verbose(1,"seed=%d\n",seed); srand(seed); clDb = optionVal("db", clDb); clOrg = optionVal("org", clOrg); clGroup = optionVal("group", clGroup); clTrack = optionVal("track", clTrack); clTable = optionVal("table", clTable); clDbs = optionInt("dbs", clDbs); clOrgs = optionInt("orgs", clOrgs); clGroups = optionInt("groups", clGroups); clTracks = optionInt("tracks", clTracks); clTables = optionInt("tables", clTables); appendLog = optionExists("appendLog"); noShuffle = optionExists("noShuffle"); if (clOrg != NULL) clOrgs = BIGNUM; hgTablesTest(argv[1], argv[2]); carefulCheckHeap(); return 0; }