b14d44f6f6031286fde5318abf67086dd29a1b43 braney Fri Sep 18 09:52:30 2026 -0700 hgTablesTest: follow the redirect on the starting url instead of failing later A url with no scheme is fetched over http, and both hgwdev and a sandbox answer plain http with a 301 to https. The run parsed the redirect page, found no form in it, and failed several steps later saying "Null form in htmlPageSetVar", which named neither the url nor the redirect. rootPageGet() now fetches through htmlPageForwarded(), says at verbose 1 when the url changed, and errAborts naming the code if the final status is not 200. Every later request is built from this page, so this puts the whole run on the url the server asked for, not just the first fetch. The robots were never exposed to this: both doHgTablesTestRobot.csh and preview2TablesTestRobot.csh already pass https urls. refs #38356 diff --git src/hg/hgTablesTest/hgTablesTest.c src/hg/hgTablesTest/hgTablesTest.c index ff18efcbb52..633daa7eda1 100644 --- src/hg/hgTablesTest/hgTablesTest.c +++ src/hg/hgTablesTest/hgTablesTest.c @@ -1,1556 +1,1575 @@ /* hgTablesTest - Test hgTables web page. */ /* Copyright (C) 2014 The Regents of the University of California * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */ #include "common.h" #include "memalloc.h" #include "linefile.h" #include "hash.h" #include "htmshell.h" #include "portable.h" #include "options.h" #include "errCatch.h" #include "ra.h" #include "htmlPage.h" #include "../hgTables/hgTables.h" #include "hdb.h" #include "dbDb.h" #include "qa.h" #include "chromInfo.h" #include "obscure.h" #include <unistd.h> #include <limits.h> #ifndef HOST_NAME_MAX // needed for OS/X #define HOST_NAME_MAX _POSIX_HOST_NAME_MAX #endif #define MAX_ATTEMPTS 10 /* Row limit for a table tested WITH position filtering. Far above the 500000 * used when the whole table gets scanned, because a region restricts the output; * this only screens out the handful of whole-genome tables so dense that even one * test region's all-fields output can exceed the carefulAlloc ceiling. */ #define MAX_ROWS_REGION_FILTERED 250000000 /* Command line variables. */ char *clOrg = NULL; /* Organism from command line. */ char *clDb = NULL; /* DB from command line */ char *clGroup = NULL; /* Group from command line. */ char *clTrack = NULL; /* Track from command line. */ char *clTable = NULL; /* Table from command line. */ int clGroups = BIGNUM; /* Number of groups to test. */ int clTracks = 4; /* Number of track to test. */ int clTables = 2; /* Number of tables to test. */ int clDbs = 1; /* Number of databases per organism. */ int clOrgs = 2; /* Number of organisms to test. */ boolean appendLog; /* Append to log rather than create it. */ boolean noShuffle; /* Suppress shuffling of track and table lists. */ void usage() /* Explain usage and exit. */ { errAbort( "hgTablesTest - Test hgTables web page\n" "usage:\n" " hgTablesTest url log\n" "Where url is something like hgwbeta.soe.ucsc.edu/cgi-bin/hgTables\n" "and log is a file where error messages and statistics will be written\n" "options:\n" " -org=Human - Restrict to Human (or Mouse, Fruitfly, etc.)\n" " -db=hg17 - Restrict to particular database\n" " -group=genes - Restrict to a particular group\n" " -track=knownGene - Restrict to a particular track\n" " -table=knownGeneMrna - Restrict to a particular table\n" " -orgs=N - Number of organisms to test. Default %d\n" " -dbs=N - Number of databases per organism to test. Default %d\n" " -groups=N - Number of groups to test (default all)\n" " -tracks=N - Number of tracks per group to test (default %d)\n" " -tables=N - Number of tables per track to test (default %d)\n" " -verbose=N - Set to 0 for silent operation, 2 or 3 for debugging\n" " -appendLog - Append to log file rather than creating it\n" " -seed N - Specify seed for random number generator as debugging aid.\n" " -noShuffle - do not shuffle tracks and tables lists.\n" , clOrgs, clDbs, clTracks, clTables); } FILE *logFile; /* Log file. */ int seed = 0; /* seed for random number generator */ static struct optionSpec options[] = { {"org", OPTION_STRING}, {"db", OPTION_STRING}, {"group", OPTION_STRING}, {"track", OPTION_STRING}, {"table", OPTION_STRING}, {"orgs", OPTION_INT}, {"dbs", OPTION_INT}, {"search", OPTION_STRING}, {"groups", OPTION_INT}, {"tracks", OPTION_INT}, {"tables", OPTION_INT}, {"appendLog", OPTION_BOOLEAN}, {"seed", OPTION_INT}, {"noShuffle", OPTION_BOOLEAN}, {NULL, 0}, }; struct tablesTest /* Test on one column. */ { struct tablesTest *next; struct qaStatus *status; /* Result of test. */ char *info[6]; }; enum tablesTestInfoIx { ntiiType, ntiiOrg, ntiiDb, ntiiGroup, ntiiTrack, ntiiTable, ntiiTotalCount, }; char *tablesTestInfoTypes[] = { "type", "organism", "db", "group", "track", "table"}; struct tablesTest *tablesTestList = NULL; /* List of all tests, latest on top. */ struct tablesTest *tablesTestNew(struct qaStatus *status, char *type, char *org, char *db, char *group, char *track, char *table) /* Save away test test results. */ { struct tablesTest *test; AllocVar(test); test->status = status; test->info[ntiiType] = cloneString(naForNull(type)); test->info[ntiiOrg] = cloneString(naForNull(org)); test->info[ntiiDb] = cloneString(naForNull(db)); test->info[ntiiGroup] = cloneString(naForNull(group)); test->info[ntiiTrack] = cloneString(naForNull(track)); test->info[ntiiTable] = cloneString(naForNull(table)); slAddHead(&tablesTestList, test); return test; } void tablesTestLogOne(struct tablesTest *test, FILE *f) /* Log test result to file. */ { int i; for (i=0; i<ArraySize(test->info); ++i) fprintf(f, "%s ", test->info[i]); fprintf(f, "%s\n", test->status->errMessage); } static void recordAbort(char *message, char *type, char *org, char *db, char *group, char *track, char *table) /* Turn a caught errAbort into a hard error in the summary. quickSubmit records * how the page fetch went; nothing records what a test made of the page it got, * so without this an abort inside a test would vanish from the counts. */ { struct qaStatus *qs; AllocVar(qs); qs->hardError = TRUE; qs->errMessage = cloneString(message); tablesTestNew(qs, type, org, db, group, track, table); verbose(1, "Caught abort testing %s (%s %s %s %s %s): %s\n", type, naForNull(org), naForNull(db), naForNull(group), naForNull(track), naForNull(table), naForNull(message)); fprintf(logFile, "Caught abort testing %s (%s %s %s %s %s): %s\n", type, naForNull(org), naForNull(db), naForNull(group), naForNull(track), naForNull(table), naForNull(message)); } struct htmlPage *quickSubmit(struct htmlPage *basePage, char *org, char *db, char *group, char *track, char *table, char *testName, char *button, char *buttonVal) /* Submit page and record info. Return NULL if a problem. */ { struct htmlPage *page = NULL; // don't get ahead of the botDelay sleep1000(5000); verbose(2, "quickSubmit(%p, %s, %s, %s, %s, %s, %s, %s, %s)\n", basePage, naForNull(org), naForNull(db), naForNull(group), naForNull(track), naForNull(table), naForNull(testName), naForNull(button), naForNull(buttonVal)); if (basePage != NULL) { struct qaStatus *qs; if (db != NULL) htmlPageSetVar(basePage, NULL, "db", db); if (org != NULL) htmlPageSetVar(basePage, NULL, "org", org); if (group != NULL) htmlPageSetVar(basePage, NULL, hgtaGroup, group); if (track != NULL) htmlPageSetVar(basePage, NULL, hgtaTrack, track); if (table != NULL) htmlPageSetVar(basePage, NULL, hgtaTable, table); qs = qaPageFromForm(basePage, basePage->forms, button, buttonVal, &page); if (!page) { verbose(2, "page is NULL, qs->errMessage=[%s]\n", qs->errMessage); if (startsWith("carefulAlloc: Allocated too much memory", qs->errMessage)) { verbose(1, "Response html page too large (500MB) (%s %s %s %s %s)\n", org, db, group, track, table); fprintf(logFile, "Response html page too large (500MB) (%s %s %s %s %s)\n", org, db, group, track, table); } else { /* Without this the caller reports only which track it was on, and the * reason the page was unusable is lost unless someone happens to re-run * at -verbose=2. */ verbose(1, "No usable page (%s %s %s %s %s): %s\n", naForNull(org), naForNull(db), naForNull(group), naForNull(track), naForNull(table), naForNull(qs->errMessage)); fprintf(logFile, "No usable page (%s %s %s %s %s): %s\n", naForNull(org), naForNull(db), naForNull(group), naForNull(track), naForNull(table), naForNull(qs->errMessage)); } } /* if (page->forms != NULL) htmlFormPrint(page->forms, stdout); */ // do not need to keep the returned structure, the answer is accumulating // in global variable: tablesTestList (void) tablesTestNew(qs, testName, org, db, group, track, table); } return page; } void serialSubmit(struct htmlPage **pPage, char *org, char *db, char *group, char *track, char *table, char *testName, char *button, char *buttonVal) /* Submit page, replacing old page with new one. */ { struct htmlPage *oldPage = *pPage; if (oldPage != NULL) { *pPage = quickSubmit(oldPage, org, db, group, track, table, testName, button, buttonVal); htmlPageFree(&oldPage); } } int tableSize(char *db, char *table) /* Return number of rows in table, or -1 if there is no such SQL table. * Plenty of tracks are offered in the table list but keep their data in a file * rather than a SQL table -- bigBed, bigWig, bigMaf and the chain/net and multiz * tracks built on them -- and counting rows on one of those used to abort the * whole process. hg38.multiz11way and mm39.netHs1 each ended a run this way. * A caller comparing against a row limit treats -1 as "no limit to apply". */ { struct sqlConnection *conn = sqlConnect(db); long size = sqlTableSizeIfExists(conn, table); sqlDisconnect(&conn); return size; } void showConnectInfo(char *db) /* Show connection info used by this program. */ { struct sqlConnection *conn = sqlConnect(db); char query[1024]; sqlSafef(query, sizeof query, "select current_user()"); char *user = sqlQuickString(conn, query); char *hostinfo = sqlHostInfo(conn); verbose(1, "Connecting as %s to database server %s\n", user, hostinfo); fprintf(logFile, "Connecting as %s to database server %s\n", user, hostinfo); fflush(logFile); sqlDisconnect(&conn); } void showRunningHostName() /* Show hostname of the machine we are running on. */ { char hostname[HOST_NAME_MAX]; if (gethostname(hostname, sizeof hostname)) { perror("gethostname"); safecpy(hostname, sizeof hostname, "error-reading-hostname"); } verbose(1, "Running on machine %s\n", hostname); fprintf(logFile, "Running on machine %s\n", hostname); fflush(logFile); } void quickErrReport() /* Report error at head of list if any */ { struct tablesTest *test = tablesTestList; if (test->status->errMessage != NULL) tablesTestLogOne(test, stderr); } void testSchema(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Make sure schema page comes up. */ /* mainForm not used */ { struct htmlPage *schemaPage = quickSubmit(tablePage, org, db, group, track, table, "schema", hgtaDoSchema, "submit"); htmlPageFree(&schemaPage); } void testSummaryStats(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Make sure summary stats page comes up. */ { if (htmlFormVarGet(mainForm, hgtaDoSummaryStats) != NULL) { struct htmlPage *statsPage = quickSubmit(tablePage, org, db, group, track, table, "summaryStats", hgtaDoSummaryStats, "submit"); htmlPageFree(&statsPage); } } boolean varIncludesType(struct htmlForm *form, char *var, char *value) /* Return TRUE if value is one of the options for var. */ { struct htmlFormVar *formVar = htmlFormVarGet(form, var); if (formVar == NULL) errAbort("Couldn't find %s variable in form", var); return slNameInList(formVar->values, value); } boolean outTypeAvailable(struct htmlForm *form, char *value) /* Return true if outType options include value. */ { return varIncludesType(form, hgtaOutputType, value); } int countNoncommentLines(char *s) /* Count number of lines in s that don't start with # */ { int count = 0; // beware, skipLeadingSpaces skips '\n' (isspace) so it skips blank lines. Check for '\n' first. if (s && *s != '\n') s = skipLeadingSpaces(s); while (s != NULL && s[0] != 0) { if (s[0] != '#') ++count; s = strchr(s, '\n'); if (s != NULL) { s++; if (*s != '\n') s = skipLeadingSpaces(s); } } return count; } int testAllFields(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Get all fields and return count of rows. */ /* mainForm not used */ { struct htmlPage *outPage; int rowCount = 0; htmlPageSetVar(tablePage, NULL, hgtaOutputType, "primaryTable"); outPage = quickSubmit(tablePage, org, db, group, track, table, "allFields", hgtaDoTopSubmit, "submit"); /* check for NULL outPage */ if (outPage == NULL) { verbose(1, "Null page in testAllFields (%s %s %s %s %s)\n", org, db, group, track, table); fprintf(logFile, "Null page in testAllFields (%s %s %s %s %s)\n", org, db, group, track, table); return -1; } rowCount = countNoncommentLines(outPage->htmlText); htmlPageFree(&outPage); return rowCount; } struct htmlFormVar *findPrefixedVar(struct htmlFormVar *list, char *prefix) /* Find first var with given prefix in list. */ { struct htmlFormVar *var; for (var = list; var != NULL; var = var->next) { if (startsWith(prefix, var->name)) return var; } return NULL; } void checkExpectedSimpleRows(struct htmlPage *outPage, int expectedRows) /* Make sure we got the number of rows we expect. */ { if (outPage != NULL) { char *results = outPage->htmlText; if (startsWith(NO_RESULTS, results)) results += strlen(NO_RESULTS); int rowCount = countNoncommentLines(results); if (rowCount != expectedRows) qaStatusSoftError(tablesTestList->status, "Got %d rows, expected %d", rowCount, expectedRows); } } void testOneField(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table, int expectedRows) /* Get one field and make sure the count agrees with expected. */ /* mainForm not used */ { struct htmlPage *outPage; struct htmlForm *form; struct htmlFormVar *var; int attempts = 0; if (tablePage->forms == NULL) errAbort("testOneField: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "selectedFields"); outPage = quickSubmit(tablePage, org, db, group, track, table, "selFieldsPage", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOneField: trying again to get selFieldsPage\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "selFieldsPage", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOneField - couldn't get outPage."); return; } if (outPage->forms == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOneField - missing form."); htmlPageFree(&outPage); return; } form = outPage->forms; var = findPrefixedVar(form->vars, "hgta_fs.check."); if (var == NULL) errAbort("No hgta_fs.check. vars in form"); htmlPageSetVar(outPage, NULL, var->name, "on"); serialSubmit(&outPage, org, db, group, track, table, "oneField", hgtaDoPrintSelectedFields, "submit"); // check that outPage != NULL checkExpectedSimpleRows(outPage, expectedRows); htmlPageFree(&outPage); } void testOutBed(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table, int expectedRows) /* Get as bed and make sure count agrees with expected. */ /* mainForm not used */ { struct htmlPage *outPage; int attempts = 0; if (tablePage->forms == NULL) errAbort("testOutBed: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "bed"); outPage = quickSubmit(tablePage, org, db, group, track, table, "bedUiPage", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOutBed: trying again to get bedUiPage\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "bedUiPage", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOneBed - couldn't get outPage."); return; } if (outPage->forms == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOneBed - missing form."); htmlPageFree(&outPage); return; } serialSubmit(&outPage, org, db, group, track, table, "outBed", hgtaDoGetBed, "submit"); // check that outPage != NULL checkExpectedSimpleRows(outPage, expectedRows); htmlPageFree(&outPage); } void testOutGff(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Get as GFF and make sure no crash. */ /* mainForm not used */ { struct htmlPage *outPage; if (tablePage->forms == NULL) errAbort("testOutGff: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "gff"); outPage = quickSubmit(tablePage, org, db, group, track, table, "outGff", hgtaDoTopSubmit, "submit"); /* no checking here */ if (outPage != NULL) htmlPageFree(&outPage); } int countTagsBetween(struct htmlPage *page, char *start, char *end, char *type) /* Count number of tags of given type (which should be upper case) * between start and end. If start is NULL it will start from * beginning of page. If end is NULL it will end at end of page. */ { int count = 0; struct htmlTag *tag; if (start == NULL) start = page->htmlText; if (end == NULL) end = start + strlen(start); for (tag = page->tags; tag != NULL; tag = tag->next) { if (tag->start >= start && tag->start < end && sameString(tag->name, type)) { ++count; } } return count; } void testOutHyperlink(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table, int expectedRows) /* Get as hyperlink and make sure count agrees with expected. */ /* mainForm not used */ { struct htmlPage *outPage; int attempts = 0; char *s; int rowCount; if (tablePage->forms == NULL) errAbort("testOutHyperlink: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "hyperlinks"); outPage = quickSubmit(tablePage, org, db, group, track, table, "outHyperlinks", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOutHyperLink: trying again to get outHyperLinks\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "outHyperlinks", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutHyperLink - couldn't get outPage."); return; } s = stringIn("<!-- +++++++++++++++++++++ CONTENT TABLES +++++++++++++++++++ -->", outPage->htmlText); if (s == NULL) errAbort("Can't find <!-- +++++++++++++++++++++ CONTENT TABLES +++++++++++++++++++ -->"); rowCount = countTagsBetween(outPage, s, NULL, "A"); if (rowCount != expectedRows) qaStatusSoftError(tablesTestList->status, "Got %d rows, expected %d", rowCount, expectedRows); htmlPageFree(&outPage); } void testOutCustomTrack(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Get as customTrack and make sure nothing explodes. */ /* mainForm not used */ { struct htmlPage *outPage; int attempts = 0; struct htmlFormVar *groupVar; if (tablePage->forms == NULL) errAbort("testOutCustomTrack: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "customTrack"); outPage = quickSubmit(tablePage, org, db, group, track, table, "customTrackUi", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOutCustomTrack: trying again to get customTrackUi\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "customTrackUi", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutCustomTrack - couldn't get outPage."); return; } serialSubmit(&outPage, org, db, group, track, table, "outCustom", hgtaDoGetCustomTrackTb, "submit"); if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutCustomTrack - serialSubmit returned null page."); return; } if (outPage->forms == NULL) { qaStatusSoftError(tablesTestList->status, "Error in custom track - no form produced."); htmlPageFree(&outPage); return; } groupVar = htmlFormVarGet(outPage->forms, hgtaGroup); if (!slNameInList(groupVar->values, "user")) { qaStatusSoftError(tablesTestList->status, "No custom track group after custom track submission"); } htmlPageFree(&outPage); } void checkFaOutput(struct htmlPage *page, int expectedCount, boolean lessOk) /* Check that page contains expected number of sequences. If lessOk is set * (needed to handle some multiply mapped cases in refSeq) then just check * that have at least one if expecting any. */ { if (page != NULL) { int count = countChars(page->htmlText, '>'); if (count != expectedCount) { if (!lessOk || count > expectedCount || (expectedCount > 0 && count <= 0)) qaStatusSoftError(tablesTestList->status, "Got %d sequences, expected %d", count, expectedCount); } } } void testOutSequence(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table, int expectedRows) /* Get as sequence and make sure count agrees with expected. */ /* mainForm not used */ { struct htmlPage *outPage; int attempts = 0; struct htmlFormVar *typeVar; if (tablePage->forms == NULL) errAbort("testOutSequence: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "sequence"); outPage = quickSubmit(tablePage, org, db, group, track, table, "seqUi1", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOutSequence: trying again to get seqUi1\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "seqUi1", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutSequence - couldn't get outPage"); return; } if (outPage->forms == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutSequence - missing form"); htmlPageFree(&outPage); return; } /* Since some genomic sequence things are huge, this will * only test in case where it's a gene prediction. */ typeVar = htmlFormVarGet(outPage->forms, hgtaGeneSeqType); if (typeVar != NULL) { struct htmlPage *seqPage; static char *types[] = {"protein", "mRNA"}; int i; for (i=0; i<ArraySize(types); ++i) { char *type = types[i]; if (slNameInList(typeVar->values, type)) { struct htmlPage *page; char testName[128]; htmlPageSetVar(outPage, NULL, hgtaGeneSeqType, type); safef(testName, sizeof(testName), "%sSeq", type); page = quickSubmit(outPage, org, db, group, track, table, testName, hgtaDoGenePredSequence, "submit"); checkFaOutput(page, expectedRows, TRUE); htmlPageFree(&page); } } htmlPageSetVar(outPage, NULL, hgtaGeneSeqType, "genomic"); serialSubmit(&outPage, org, db, group, track, table, "seqUi2", hgtaDoGenePredSequence, "submit"); // check that outPage != NULL /* On genomic page uncheck intron if it's there, then get results * and count them. */ if (htmlFormVarGet(outPage->forms, "hgSeq.intron") != NULL) htmlPageSetVar(outPage, NULL, "hgSeq.intron", NULL); seqPage = quickSubmit(outPage, org, db, group, track, table, "genomicSeq", hgtaDoGenomicDna, "submit"); // check that seqPage != NULL checkFaOutput(seqPage, expectedRows, FALSE); htmlPageFree(&seqPage); } htmlPageFree(&outPage); } boolean isObsolete(char *table) /* Some old table types we can't handle. Just warn that * they are there and skip. */ { boolean obsolete = sameString(table, "wabaCbr"); if (obsolete) qaStatusSoftError(tablesTestList->status, "Skipping obsolete table %s", table); return obsolete; } static void testOneTableBody(struct htmlPage *trackPage, char *org, char *db, char *group, char *track, char *table) /* Test stuff on one table if we haven't already tested this table. */ { /* Why declared here and not globally? */ static struct hash *uniqHash = NULL; char fullName[256]; if (uniqHash == NULL) uniqHash = newHash(0); safef(fullName, sizeof(fullName), "%s.%s", db, table); if (!hashLookup(uniqHash, fullName)) { struct htmlPage *tablePage; struct htmlForm *mainForm; hashAdd(uniqHash, fullName, NULL); verbose(1, "Testing %s %s %s %s %s\n", naForNull(org), db, group, track, table); tablePage = quickSubmit(trackPage, org, db, group, track, table, "selectTable", hgtaTable, table); if (!isObsolete(table) && tablePage != NULL) { if ((mainForm = htmlFormGet(tablePage, "mainForm")) == NULL) { qaStatusSoftError(tablesTestList->status, "Couldn't get main form on tablePage for %s %s %s %s", db, group, track, table); } else { verbose(3, "testOneTable testSchema() got here 1.1\n"); testSchema(tablePage, mainForm, org, db, group, track, table); verbose(3, "testOneTable testSummaryStats() got here 1.2\n"); testSummaryStats(tablePage, mainForm, org, db, group, track, table); verbose(3, "testOneTable got here 1.3\n"); if (outTypeAvailable(mainForm, "bed")) { verbose(3, "testOneTable bed output avail means can filter on position got here 2\n"); /* A region bounds the output for almost every table, but a whole-genome * table of hundreds of millions of rows is dense enough that all-fields * output for the test region can still pass the carefulAlloc ceiling. * carefulAlloc exits the process outright, which forfeits every table * left in the run, so screen the worst offenders out the way the * no-position-filter branch below does. */ int tableRows = tableSize(db, table); if (tableRows >= MAX_ROWS_REGION_FILTERED) { verbose(1, "%s.%s tableRows=%d, too large >= %d even with position filtering, skipping.\n", db, table, tableRows, MAX_ROWS_REGION_FILTERED); fprintf(logFile, "%s.%s tableRows=%d, too large >= %d even with position filtering, skipping.\n", db, table, tableRows, MAX_ROWS_REGION_FILTERED); } else if (outTypeAvailable(mainForm, "primaryTable")) { verbose(3, "testOneTable got here 3\n"); int rowCount = testAllFields(tablePage, mainForm, org, db, group, track, table); if (rowCount >= 0) { testOneField(tablePage, mainForm, org, db, group, track, table, rowCount); testOutSequence(tablePage, mainForm, org, db, group, track, table, rowCount); testOutBed(tablePage, mainForm, org, db, group, track, table, rowCount); testOutHyperlink(tablePage, mainForm, org, db, group, track, table, rowCount); testOutGff(tablePage, mainForm, org, db, group, track, table); if (rowCount > 0) testOutCustomTrack(tablePage, mainForm, org, db, group, track, table); } } } else if (outTypeAvailable(mainForm, "primaryTable")) { verbose(3, "testOneTable no bed output available, so no position filtering available. got here 4\n"); /* If BED type is not available then the region will be ignored, and * we'll end up scanning whole table. Make sure table is not huge * before proceeding. */ int tableRows = tableSize(db, table); if (tableRows < 500000) { int rowCount = testAllFields(tablePage, mainForm, org, db, group, track, table); if (rowCount >= 0) testOneField(tablePage, mainForm, org, db, group, track, table, rowCount); } else { verbose(1, "%s.%s tableRows=%d, too large >= 500000, skipping.\n", db, table, tableRows); fprintf(logFile, "%s.%s tableRows=%d, too large >= 500000, skipping.\n", db, table, tableRows); } } } htmlPageFree(&tablePage); } carefulCheckHeap(); } } void testOneTable(struct htmlPage *trackPage, char *org, char *db, char *group, char *track, char *table) /* Test one table, surviving an abort from anything it calls. Most of the * errAborts in this program are in the output tests below testOneTableBody, and * any one of them used to end the whole run. */ { struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) testOneTableBody(trackPage, org, db, group, track, table); errCatchEnd(errCatch); if (errCatch->gotError) recordAbort(errCatch->message->string, "table", org, db, group, track, table); errCatchFree(&errCatch); } static void testOneTrackBody(struct htmlPage *groupPage, char *org, char *db, char *group, char *track, int maxTables) /* Test a little something on up to maxTables in one track. */ { struct htmlPage *trackPage = quickSubmit(groupPage, org, db, group, track, NULL, "selectTrack", hgtaTrack, track); struct htmlForm *mainForm; struct htmlFormVar *tableVar; struct slName *table; int tableIx; /* A track whose page does not come back, or comes back unusable, is skipped * rather than fatal. quickSubmit has already recorded the failure in * tablesTestList, so it is counted in the final summary either way. Aborting * here used to end the whole run, which meant the summary that carries the * error counts was never written at all, and a single bad page - often just a * truncated response from a busy server - threw away an hour of testing. This * also covers the old bigPsl special case (2016-06-20). */ if (trackPage == NULL) { verbose(1, "Skipping track %s: no page returned\n", track); fprintf(logFile, "Skipping track %s: no page returned\n", track); return; } if ((mainForm = htmlFormGet(trackPage, "mainForm")) == NULL) { verbose(1, "Skipping track %s: no main form on track page\n", track); fprintf(logFile, "Skipping track %s: no main form on track page\n", track); htmlPageFree(&trackPage); return; } if ((tableVar = htmlFormVarGet(mainForm, hgtaTable)) == NULL) { verbose(1, "Skipping track %s: no table var on track page\n", track); fprintf(logFile, "Skipping track %s: no table var on track page\n", track); htmlPageFree(&trackPage); return; } // put the tables in random order: if (!noShuffle) shuffleList(&tableVar->values); for (table = tableVar->values, tableIx = 0; table != NULL && tableIx < maxTables; table = table->next) { if (clTable && !sameString(clTable, table->name)) continue; testOneTable(trackPage, org, db, group, track, table->name); ++tableIx; } /* Clean up. */ htmlPageFree(&trackPage); } void testOneTrack(struct htmlPage *groupPage, char *org, char *db, char *group, char *track, int maxTables) /* Test one track, surviving an abort. */ { struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) testOneTrackBody(groupPage, org, db, group, track, maxTables); errCatchEnd(errCatch); if (errCatch->gotError) recordAbort(errCatch->message->string, "track", org, db, group, track, NULL); errCatchFree(&errCatch); } static void testOneGroupBody(struct htmlPage *dbPage, char *org, char *db, char *group, int maxTracks) /* Test a little something on up to maxTracks in one group */ { struct htmlPage *groupPage = quickSubmit(dbPage, org, db, group, NULL, NULL, "selectGroup", hgtaGroup, group); struct htmlForm *mainForm; struct htmlFormVar *trackVar; struct slName *track; int trackIx; /* As in testOneTrack, a group we cannot read is skipped rather than fatal, so * that one bad page does not cost the whole run. */ if (groupPage == NULL) { verbose(1, "Skipping group %s: no page returned\n", group); fprintf(logFile, "Skipping group %s: no page returned\n", group); return; } if ((mainForm = htmlFormGet(groupPage, "mainForm")) == NULL) { verbose(1, "Skipping group %s: no main form on group page\n", group); fprintf(logFile, "Skipping group %s: no main form on group page\n", group); htmlPageFree(&groupPage); return; } if ((trackVar = htmlFormVarGet(mainForm, hgtaTrack)) == NULL) { verbose(1, "Skipping group %s: no track var on group page\n", group); fprintf(logFile, "Skipping group %s: no track var on group page\n", group); htmlPageFree(&groupPage); return; } // put the tracks in random order: if (!noShuffle) shuffleList(&trackVar->values); for (track = trackVar->values, trackIx = 0; track != NULL && trackIx < maxTracks; track = track->next) { if (clTrack && !sameString(track->name, clTrack)) continue; testOneTrack(groupPage, org, db, group, track->name, clTables); ++trackIx; } /* Clean up. */ htmlPageFree(&groupPage); } void testOneGroup(struct htmlPage *dbPage, char *org, char *db, char *group, int maxTracks) /* Test one group, surviving an abort. */ { struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) testOneGroupBody(dbPage, org, db, group, maxTracks); errCatchEnd(errCatch); if (errCatch->gotError) recordAbort(errCatch->message->string, "group", org, db, group, NULL, NULL); errCatchFree(&errCatch); } void testGroups(struct htmlPage *dbPage, char *org, char *db, int maxGroups) /* Test a little something in all groups for dbPage. */ { struct htmlForm *mainForm; struct htmlFormVar *groupVar; struct slName *group; int groupIx; if ((mainForm = htmlFormGet(dbPage, "mainForm")) == NULL) errAbort("Couldn't get main form on dbPage"); if ((groupVar = htmlFormVarGet(mainForm, hgtaGroup)) == NULL) errAbort("Can't find group var"); for (group = groupVar->values, groupIx=0; group != NULL && groupIx < maxGroups; group = group->next) { if (!sameString("allTables", group->name)) { if (clGroup && !sameString(clGroup, group->name)) continue; testOneGroup(dbPage, org, db, group->name, clTracks); ++groupIx; } } } void getTestRegion(char *db, char region[256], int regionSize) /* Look up first chromosome in database and grab five million bases * from the middle of it. */ { struct sqlConnection *conn = sqlConnect(db); char query[1024]; sqlSafef(query, sizeof query, "select * from chromInfo limit 1"); struct sqlResult *sr = sqlGetResult(conn, query); char **row; struct chromInfo ci; int start,end,middle; if ((row = sqlNextRow(sr)) == NULL) errAbort("Couldn't get one row from chromInfo"); chromInfoStaticLoad(row, &ci); middle = ci.size/2; start = middle-2500000; end = middle+2500000; if (start < 0) start = 0; if (end > ci.size) end = ci.size; safef(region, regionSize, "%s:%d-%d", ci.chrom, start+1, end); verbose(1, "Testing %s at position %s\n", db, region); fprintf(logFile, "Testing %s at position %s\n", db, region); sqlFreeResult(&sr); sqlDisconnect(&conn); } /* The hgTables page no longer carries clade/organism/assembly dropdowns -- the * gateway moved to a search box -- so there are no <option> tags left to scrape * for the list of organisms and assemblies. Read them from hgcentral dbDb * instead, the way the browser itself does. Only the *enumeration* was lost; * htmlFormVarSet still creates org/db on the form, so submitting works. */ boolean isTestableDb(struct dbDb *db) /* TRUE if this assembly is one we can actually drive: active, and with a * chromInfo table, which getTestRegion needs to pick a test region. GenArk * entries (hs1 and the like) are listed in dbDb and do have a database, but * keep their data in files rather than SQL tables, so they have no chromInfo * and used to be absent from the assembly dropdown we no longer read. */ { return db->active && hTableExists(db->name, "chromInfo"); } struct slName *organismsToTest(int maxOrgs) /* Return up to maxOrgs distinct testable organism names, in dbDb order. */ { struct slName *list = NULL; struct dbDb *dbList = hDbDbList(), *db; struct hash *seen = newHash(8); int count = 0; for (db = dbList; db != NULL && count < maxOrgs; db = db->next) { if (isTestableDb(db) && !hashLookup(seen, db->organism)) { hashAdd(seen, db->organism, NULL); slNameAddTail(&list, db->organism); ++count; } } hashFree(&seen); dbDbFreeList(&dbList); return list; } struct slName *dbsForOrganism(char *org, int maxDbs) /* Return up to maxDbs testable assembly names for organism, in dbDb order. */ { struct slName *list = NULL; struct dbDb *dbList = hDbDbList(), *db; int count = 0; for (db = dbList; db != NULL && count < maxDbs; db = db->next) { if (isTestableDb(db) && sameWord(db->organism, org)) { slNameAddTail(&list, db->name); ++count; } } dbDbFreeList(&dbList); return list; } static void testDbBody(struct htmlPage *orgPage, char *org, char *db) /* Test on one database. */ { struct htmlPage *dbPage; char region[256]; htmlPageSetVar(orgPage, NULL, "db", db); getTestRegion(db, region, sizeof(region)); htmlPageSetVar(orgPage, NULL, "position", region); htmlPageSetVar(orgPage, NULL, hgtaRegionType, "range"); dbPage = quickSubmit(orgPage, org, db, NULL, NULL, NULL, "selectDb", "submit", "go"); if (dbPage != NULL) testGroups(dbPage, org, db, clGroups); htmlPageFree(&dbPage); } void testDb(struct htmlPage *orgPage, char *org, char *db) /* Test one database, surviving an abort. The setup steps here - the test * region, the group list - abort on their own, and one bad database should not * cost the databases after it. */ { struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) testDbBody(orgPage, org, db); errCatchEnd(errCatch); if (errCatch->gotError) recordAbort(errCatch->message->string, "db", org, db, NULL, NULL, NULL); errCatchFree(&errCatch); } void testOrg(struct htmlPage *rootPage, struct htmlForm *rootForm, char *org) /* Test on organism. */ { struct slName *dbList, *db; /* There is no organism round-trip left to make: the gateway's clade/organism/ * assembly dropdowns and the "Go" button that submitted them are gone from the * page, so submitting for an "organism page" just yields a page with no * mainForm. Name the assembly directly instead, the way the -db= path does -- * testDb sets db, position and region type on each pass, so rootPage needs no * preparation here. */ dbList = dbsForOrganism(org, clDbs); if (dbList == NULL) errAbort("No active assembly in dbDb for organism %s", org); for (db = dbList; db != NULL; db = db->next) { testDb(rootPage, org, db->name); } slNameFreeList(&dbList); } void verifyJoinedFormat(char *s) /* Verify that s consists of lines with two tab-separated fields, * and that the second field has some n/a and some comma-separated lists. */ { char *e; int lineIx = 0; boolean gotCommas = FALSE, gotNa = FALSE; while (s != NULL && s[0] != 0) { char *row[3]; int fieldCount; ++lineIx; e = strchr(s, '\n'); if (e != NULL) *e++ = 0; if (s[0] != '#') { fieldCount = chopTabs(s, row); if (fieldCount != 2) { qaStatusSoftError(tablesTestList->status, "Got %d fields line %d of joined result, expected 2", fieldCount, lineIx); break; } if (sameString(row[1], "n/a")) gotNa = TRUE; if (countChars(row[1], ',') >= 2) gotCommas = TRUE; } s = e; } if (!gotCommas) qaStatusSoftError(tablesTestList->status, "Expected some rows in join to have comma separated lists."); if (!gotNa) qaStatusSoftError(tablesTestList->status, "Expected some rows in join to have n/a."); } void testJoining(struct htmlPage *rootPage) /* Simulate pressing buttons to get a reasonable join on a * couple of uniProt tables. */ { struct htmlPage *allPage, *page; char *org = NULL, *db = NULL, *group = "allTables", *track="uniProt"; int expectedCount = tableSize("uniProt", "taxon"); allPage = quickSubmit(rootPage, org, db, group, "uniProt", "uniProt.taxon", "taxonJoin1", NULL, NULL); if (allPage != NULL) { if (allPage->forms == NULL) { errAbort("uniProt page with no form"); } else { int count = testAllFields(allPage, allPage->forms, org, db, group, track, "uniProt.taxon"); if (count != expectedCount) qaStatusSoftError(tablesTestList->status, "Got %d rows in uniProt.taxon, expected %d", count, expectedCount); htmlPageSetVar(allPage, NULL, hgtaOutputType, "selectedFields"); page = quickSubmit(allPage, org, db, group, track, "uniProt.taxon", "taxonJoin2", hgtaDoTopSubmit, "submit"); htmlPageSetVar(page, NULL, "hgta_fs.linked.uniProt.commonName", "on"); serialSubmit(&page, org, db, group, track, NULL, "taxonJoin3", hgtaDoSelectFieldsMore, "submit"); if (page != NULL) { htmlPageSetVar(page, NULL, "hgta_fs.check.uniProt.taxon.binomial", "on"); htmlPageSetVar(page, NULL, "hgta_fs.check.uniProt.commonName.val", "on"); serialSubmit(&page, org, db, group, track, NULL, "taxonJoin4", hgtaDoPrintSelectedFields, "submit"); if (page != NULL) { checkExpectedSimpleRows(page, expectedCount); verifyJoinedFormat(page->htmlText); htmlPageFree(&page); } } } } htmlPageFree(&allPage); verbose(1, "Tested joining on uniProt.taxon & commonName\n"); } void checkXenopus(char *s) /* Check that all lines start with xenopus, and that we * see laevis in there somewhere. */ { char *e; boolean gotLaevis = FALSE; while (s != NULL && s[0] != 0) { s = skipLeadingSpaces(s); e = strchr(s, '\n'); if (e != NULL) *e++ = 0; if (s[0] != '#') { char *t = strchr(s, '\t'); if (t != NULL) *t = 0; if (!startsWith("Xenopus", s)) { qaStatusSoftError(tablesTestList->status, "Xenopus filter passing non-Xenopus"); return; } if (sameString(s, "Xenopus laevis")) gotLaevis = TRUE; } s = e; } if (!gotLaevis) qaStatusSoftError(tablesTestList->status, "Can't find Xenopus laevis in filtered uniProt.taxon"); } void testFilter(struct htmlPage *rootPage) /* Simulate pressing buttons to get a reasonable filter on * uniProt taxon. */ { char *org = NULL, *db = NULL, *group = "allTables", *track="uniProt", *table = "uniProt.taxon"; struct htmlPage *page; page = quickSubmit(rootPage, org, db, group, "uniProt", table, "taxonFilter1", hgtaDoFilterPage, "submit"); if (page != NULL) { struct htmlFormVar *var = htmlFormVarGet(page->forms, "hgta_fil.v.uniProt.taxon.binomial.pat"); if (var == NULL) internalErr(); htmlPageSetVar(page, NULL, "hgta_fil.v.uniProt.taxon.binomial.pat", "Xenopus*"); serialSubmit(&page, org, db, group, track, table, "taxonFilter2", hgtaDoFilterSubmit, "submit"); if (page != NULL) { htmlPageSetVar(page, NULL, hgtaOutputType, "selectedFields"); serialSubmit(&page, org, db, group, track, table, "taxonFilter3", hgtaDoTopSubmit, "submit"); if (page != NULL) { htmlPageSetVar(page, NULL, "hgta_fs.check.uniProt.taxon.binomial", "on"); serialSubmit(&page, org, db, group, track, table, "taxonFilter4", hgtaDoPrintSelectedFields, "submit"); if (page != NULL) checkXenopus(page->htmlText); htmlPageFree(&page); } } } verbose(1, "Tested filter on uniProt.taxon\n"); } void testIdentifier(struct htmlPage *rootPage) /* Do simple check on identifiers. Relies on * 8355 Xenopus laevis being stable taxon (and not being filtered out * by testFilter). */ { char *org = NULL, *db = NULL, *group = "allTables", *track="uniProt", *table = "uniProt.taxon"; struct htmlPage *page; page = quickSubmit(rootPage, org, db, group, "uniProt", table, "taxonId1", hgtaDoPasteIdentifiers, "submit"); if (page != NULL) { htmlPageSetVar(page, NULL, hgtaPastedIdentifiers, "8355"); serialSubmit(&page, org, db, group, track, table, "taxonId2", hgtaDoPastedIdentifiers, "submit"); if (page != NULL) { htmlPageSetVar(page, NULL, hgtaOutputType, "selectedFields"); serialSubmit(&page, org, db, group, track, table, "taxonId3", hgtaDoTopSubmit, "submit"); if (page != NULL) { htmlPageSetVar(page, NULL, "hgta_fs.check.uniProt.taxon.binomial", "on"); serialSubmit(&page, org, db, group, track, table, "taxonId4", hgtaDoPrintSelectedFields, "submit"); if (page != NULL) { if (!stringIn("Xenopus laevis", page->htmlText)) { qaStatusSoftError(tablesTestList->status, "Can't find Xenopus laevis in uniProt.taxon #8355"); } checkExpectedSimpleRows(page, 1); } htmlPageFree(&page); } } } verbose(1, "Tested identifier on uniProt.taxon\n"); } void statsOnSubsets(struct tablesTest *list, int subIx, FILE *f) /* Report tests of certain subtype. */ { struct tablesTest *test; struct hash *hash = newHash(0); struct slName *typeList = NULL, *type; fprintf(f, "\n%s subtotals\n", tablesTestInfoTypes[subIx]); /* Get list of all types in this field. */ for (test = list; test != NULL; test = test->next) { char *info = test->info[subIx]; if (!hashLookup(hash, info)) { type = slNameNew(info); hashAdd(hash, info, type); slAddHead(&typeList, type); } } slNameSort(&typeList); hashFree(&hash); for (type = typeList; type != NULL; type = type->next) { struct qaStatistics *stats; AllocVar(stats); for (test = list; test != NULL; test = test->next) { if (sameString(type->name, test->info[subIx])) { qaStatisticsAdd(stats, test->status); } } qaStatisticsReport(stats, type->name, f); freez(&stats); } } static int countHardErrors(struct tablesTest *list) /* Count the tests that ended in a hard error. */ { int count = 0; struct tablesTest *test; for (test = list; test != NULL; test = test->next) if (test->status->errMessage != NULL && test->status->hardError) ++count; return count; } void reportSummary(struct tablesTest *list, FILE *f) /* Report summary of test results. */ { struct qaStatistics *stats; struct tablesTest *test; int i; AllocVar(stats); for (i=0; i<ntiiTotalCount; ++i) statsOnSubsets(list, i, f); for (test = list; test != NULL; test = test->next) qaStatisticsAdd(stats, test->status); fprintf(f, "\ngrand total\n"); qaStatisticsReport(stats, "Total", f); } void reportAll(struct tablesTest *list, FILE *f) /* Report all tests. */ { struct tablesTest *test; for (test = list; test != NULL; test = test->next) { if (test->status->errMessage != NULL) tablesTestLogOne(test, f); } } static void catchRootTest(void (*test)(struct htmlPage *rootPage), char *name, struct htmlPage *rootPage) /* Run one of the whole-program uniProt tests, surviving an abort. These run * last, so an abort in the first of them used to take the other two and the * summary with it. */ { struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) test(rootPage); errCatchEnd(errCatch); if (errCatch->gotError) recordAbort(errCatch->message->string, name, NULL, "uniProt", NULL, NULL, NULL); errCatchFree(&errCatch); } +static struct htmlPage *rootPageGet(char *url) +/* Fetch the page the whole run starts from, following a redirect if the server + * sends one. Both hgwdev and a sandbox answer plain http with a 301 to https, + * and a url given with no scheme is fetched over http, so without this the run + * parses the redirect page, finds no form in it, and dies several steps later + * saying "Null form in htmlPageSetVar", which names neither the url nor the + * redirect. Every later request is built from this page, so following the + * redirect here also puts the rest of the run on the url the server asked for. */ +{ +struct htmlPage *page = htmlPageForwarded(url, NULL); +if (page == NULL) + errAbort("Couldn't get %s", url); +if (!sameString(page->url, url)) + verbose(1, "%s redirected to %s\n", url, page->url); +if (page->status->status != 200) + errAbort("%s returned HTTP status code %d", page->url, page->status->status); +return page; +} + int hgTablesTest(char *url, char *logName) /* hgTablesTest - Test hgTables web page. Returns the exit code: zero only if * the run finished and no test hit a hard error. */ { /* Get default page, and open log. */ -struct htmlPage *rootPage = htmlPageGet(url); +struct htmlPage *rootPage = rootPageGet(url); if (appendLog) logFile = mustOpen(logName, "a"); else logFile = mustOpen(logName, "w"); if (! endsWith(url, "hgTables")) warn("Warning: first argument should be a complete URL to hgTables, " "but doesn't look like one (%s)", url); fprintf(logFile,"seed=%d\n",seed); showRunningHostName(); -verbose(1, "Testing URL %s\n", url); -fprintf(logFile, "Testing URL %s\n", url); +verbose(1, "Testing URL %s\n", rootPage->url); +fprintf(logFile, "Testing URL %s\n", rootPage->url); /* Show what database server we are connecting to. Matters for expected rows in tables. */ showConnectInfo("uniProt"); htmlPageValidateOrAbort(rootPage); /* Go test what they've specified in command line. */ if (clDb != NULL) { testDb(rootPage, NULL, clDb); } else { struct htmlForm *mainForm; if ((mainForm = htmlFormGet(rootPage, "mainForm")) == NULL) errAbort("Couldn't get main form"); if (clOrg != NULL) testOrg(rootPage, mainForm, clOrg); else { struct slName *orgList = organismsToTest(clOrgs), *org; if (orgList == NULL) errAbort("No active organisms in dbDb"); for (org = orgList; org != NULL; org = org->next) { testOrg(rootPage, mainForm, org->name); } slNameFreeList(&orgList); } } /* Do some more complex tests on uniProt. */ catchRootTest(testJoining, "joining", rootPage); catchRootTest(testFilter, "filter", rootPage); catchRootTest(testIdentifier, "identifier", rootPage); /* Clean up and report. */ htmlPageFree(&rootPage); slReverse(&tablesTestList); reportSummary(tablesTestList, stdout); reportAll(tablesTestList, logFile); fprintf(logFile, "---------------------------------------------\n"); reportSummary(tablesTestList, logFile); /* A run that tested nothing, or that could not read a page it asked for, is * not a pass. Before #38356 the first unreadable page ended the run with * errAbort, so the caller at least saw a nonzero exit. Now that the run * carries on and counts such a page as a hard error, the exit code has to * carry the same news, or a caller reading only the exit code is told a run * that failed on every track succeeded. Soft errors are deliberately not * counted here: the page was read and the answer was wrong, which is a report * about hgTables rather than about this run. They are still in the summary. */ int testCount = slCount(tablesTestList); int hardCount = countHardErrors(tablesTestList); if (testCount == 0) { verbose(1, "No tests ran.\n"); fprintf(logFile, "No tests ran.\n"); return 1; } if (hardCount > 0) { verbose(1, "Exiting nonzero: %d of %d tests hit a hard error.\n", hardCount, testCount); fprintf(logFile, "Exiting nonzero: %d of %d tests hit a hard error.\n", hardCount, testCount); return 1; } return 0; } int main(int argc, char *argv[]) /* Process command line. */ { pushCarefulMemHandler(500000000); optionInit(&argc, argv, options); if (argc != 3) usage(); seed = optionInt("seed",time(NULL)); verbose(1,"seed=%d\n",seed); srand(seed); clDb = optionVal("db", clDb); clOrg = optionVal("org", clOrg); clGroup = optionVal("group", clGroup); clTrack = optionVal("track", clTrack); clTable = optionVal("table", clTable); clDbs = optionInt("dbs", clDbs); clOrgs = optionInt("orgs", clOrgs); clGroups = optionInt("groups", clGroups); clTracks = optionInt("tracks", clTracks); clTables = optionInt("tables", clTables); appendLog = optionExists("appendLog"); noShuffle = optionExists("noShuffle"); if (clOrg != NULL) clOrgs = BIGNUM; int status = hgTablesTest(argv[1], argv[2]); carefulCheckHeap(); return status; }