d48a1f1935917a45b20ae782f4a0ab53da13e6a9 braney Tue Sep 15 12:53:04 2026 -0700 hgTablesTest: skip an oversized page instead of dying inside the allocator, refs #38359 A dense file-backed track can hand back hundreds of megabytes for a single five-megabyte test region. hg38 hgdp returned 602MB, which took carefulAlloc past its 500MB ceiling, and carefulAlloc exits the process where it stands rather than errAborting, on the grounds that errAbort itself allocates. So the run ended with one line on stderr, nothing in the log, and every table still to come forfeited. The arm in quickSubmit meant to catch exactly this and name the track had never once run. htmlPage now takes an optional ceiling on the response it will read into memory. Past it the fetch frees what it has read and errAborts naming the url, which the robot's errCatch turns back into an ordinary return of no page. The ceiling defaults to none, which leaves hgNearTest, hgBlatTest and htmlCheck exactly as they were. hgTablesTest sets it to 100MB, a fifth of the allocator ceiling: the dyString roughly doubles as it grows and the old buffer is still live while the new one fills, and the parsed page then sits alongside its text. An oversized page is logged and skipped, not counted as an error. A track that answers a 5Mb region with 600MB is one this robot cannot test, which is the same situation the row count screen already catches before submitting; counting it would put a failure in every weekly run and leave the summary as useless a gate as the one that never failed. The log is line buffered now as well. Finding out that a run died partway through is what this robot is for, and a block of buffered lines lost on the way out is part of how the old failure left no trace of which track it was on. Co-Authored-By: Claude Opus 5 (1M context) diff --git src/hg/hgTablesTest/hgTablesTest.c src/hg/hgTablesTest/hgTablesTest.c index 633daa7eda1..49f9c24f542 100644 --- src/hg/hgTablesTest/hgTablesTest.c +++ src/hg/hgTablesTest/hgTablesTest.c @@ -1,1575 +1,1607 @@ /* hgTablesTest - Test hgTables web page. */ /* Copyright (C) 2014 The Regents of the University of California * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */ #include "common.h" #include "memalloc.h" #include "linefile.h" #include "hash.h" #include "htmshell.h" #include "portable.h" #include "options.h" #include "errCatch.h" #include "ra.h" #include "htmlPage.h" #include "../hgTables/hgTables.h" #include "hdb.h" #include "dbDb.h" #include "qa.h" #include "chromInfo.h" #include "obscure.h" #include #include #ifndef HOST_NAME_MAX // needed for OS/X #define HOST_NAME_MAX _POSIX_HOST_NAME_MAX #endif #define MAX_ATTEMPTS 10 /* Row limit for a table tested WITH position filtering. Far above the 500000 * used when the whole table gets scanned, because a region restricts the output; * this only screens out the handful of whole-genome tables so dense that even one * test region's all-fields output can exceed the carefulAlloc ceiling. */ #define MAX_ROWS_REGION_FILTERED 250000000 +/* Ceiling on the allocation the careful memory handler will permit. */ +#define MAX_CAREFUL_ALLOC 500000000 + +/* Ceiling on one response read into memory. Well under MAX_CAREFUL_ALLOC because a + * response does not cost its own size once: the dyString it accumulates in roughly + * doubles when it grows, the old buffer is still live while the new one is filled, + * and the parsed page then sits alongside the text. A page this big is a failure + * to report, not a page we want to finish parsing. */ +#define MAX_RESPONSE_BYTES 100000000 + /* Command line variables. */ char *clOrg = NULL; /* Organism from command line. */ char *clDb = NULL; /* DB from command line */ char *clGroup = NULL; /* Group from command line. */ char *clTrack = NULL; /* Track from command line. */ char *clTable = NULL; /* Table from command line. */ int clGroups = BIGNUM; /* Number of groups to test. */ int clTracks = 4; /* Number of track to test. */ int clTables = 2; /* Number of tables to test. */ int clDbs = 1; /* Number of databases per organism. */ int clOrgs = 2; /* Number of organisms to test. */ boolean appendLog; /* Append to log rather than create it. */ boolean noShuffle; /* Suppress shuffling of track and table lists. */ void usage() /* Explain usage and exit. */ { errAbort( "hgTablesTest - Test hgTables web page\n" "usage:\n" " hgTablesTest url log\n" "Where url is something like hgwbeta.soe.ucsc.edu/cgi-bin/hgTables\n" "and log is a file where error messages and statistics will be written\n" "options:\n" " -org=Human - Restrict to Human (or Mouse, Fruitfly, etc.)\n" " -db=hg17 - Restrict to particular database\n" " -group=genes - Restrict to a particular group\n" " -track=knownGene - Restrict to a particular track\n" " -table=knownGeneMrna - Restrict to a particular table\n" " -orgs=N - Number of organisms to test. Default %d\n" " -dbs=N - Number of databases per organism to test. Default %d\n" " -groups=N - Number of groups to test (default all)\n" " -tracks=N - Number of tracks per group to test (default %d)\n" " -tables=N - Number of tables per track to test (default %d)\n" " -verbose=N - Set to 0 for silent operation, 2 or 3 for debugging\n" " -appendLog - Append to log file rather than creating it\n" " -seed N - Specify seed for random number generator as debugging aid.\n" " -noShuffle - do not shuffle tracks and tables lists.\n" , clOrgs, clDbs, clTracks, clTables); } FILE *logFile; /* Log file. */ int seed = 0; /* seed for random number generator */ static struct optionSpec options[] = { {"org", OPTION_STRING}, {"db", OPTION_STRING}, {"group", OPTION_STRING}, {"track", OPTION_STRING}, {"table", OPTION_STRING}, {"orgs", OPTION_INT}, {"dbs", OPTION_INT}, {"search", OPTION_STRING}, {"groups", OPTION_INT}, {"tracks", OPTION_INT}, {"tables", OPTION_INT}, {"appendLog", OPTION_BOOLEAN}, {"seed", OPTION_INT}, {"noShuffle", OPTION_BOOLEAN}, {NULL, 0}, }; struct tablesTest /* Test on one column. */ { struct tablesTest *next; struct qaStatus *status; /* Result of test. */ char *info[6]; }; enum tablesTestInfoIx { ntiiType, ntiiOrg, ntiiDb, ntiiGroup, ntiiTrack, ntiiTable, ntiiTotalCount, }; char *tablesTestInfoTypes[] = { "type", "organism", "db", "group", "track", "table"}; struct tablesTest *tablesTestList = NULL; /* List of all tests, latest on top. */ struct tablesTest *tablesTestNew(struct qaStatus *status, char *type, char *org, char *db, char *group, char *track, char *table) /* Save away test test results. */ { struct tablesTest *test; AllocVar(test); test->status = status; test->info[ntiiType] = cloneString(naForNull(type)); test->info[ntiiOrg] = cloneString(naForNull(org)); test->info[ntiiDb] = cloneString(naForNull(db)); test->info[ntiiGroup] = cloneString(naForNull(group)); test->info[ntiiTrack] = cloneString(naForNull(track)); test->info[ntiiTable] = cloneString(naForNull(table)); slAddHead(&tablesTestList, test); return test; } void tablesTestLogOne(struct tablesTest *test, FILE *f) /* Log test result to file. */ { int i; for (i=0; iinfo); ++i) fprintf(f, "%s ", test->info[i]); fprintf(f, "%s\n", test->status->errMessage); } static void recordAbort(char *message, char *type, char *org, char *db, char *group, char *track, char *table) /* Turn a caught errAbort into a hard error in the summary. quickSubmit records * how the page fetch went; nothing records what a test made of the page it got, * so without this an abort inside a test would vanish from the counts. */ { struct qaStatus *qs; AllocVar(qs); qs->hardError = TRUE; qs->errMessage = cloneString(message); tablesTestNew(qs, type, org, db, group, track, table); verbose(1, "Caught abort testing %s (%s %s %s %s %s): %s\n", type, naForNull(org), naForNull(db), naForNull(group), naForNull(track), naForNull(table), naForNull(message)); fprintf(logFile, "Caught abort testing %s (%s %s %s %s %s): %s\n", type, naForNull(org), naForNull(db), naForNull(group), naForNull(track), naForNull(table), naForNull(message)); } struct htmlPage *quickSubmit(struct htmlPage *basePage, char *org, char *db, char *group, char *track, char *table, char *testName, char *button, char *buttonVal) /* Submit page and record info. Return NULL if a problem. */ { struct htmlPage *page = NULL; // don't get ahead of the botDelay sleep1000(5000); verbose(2, "quickSubmit(%p, %s, %s, %s, %s, %s, %s, %s, %s)\n", basePage, naForNull(org), naForNull(db), naForNull(group), naForNull(track), naForNull(table), naForNull(testName), naForNull(button), naForNull(buttonVal)); if (basePage != NULL) { struct qaStatus *qs; if (db != NULL) htmlPageSetVar(basePage, NULL, "db", db); if (org != NULL) htmlPageSetVar(basePage, NULL, "org", org); if (group != NULL) htmlPageSetVar(basePage, NULL, hgtaGroup, group); if (track != NULL) htmlPageSetVar(basePage, NULL, hgtaTrack, track); if (table != NULL) htmlPageSetVar(basePage, NULL, hgtaTable, table); qs = qaPageFromForm(basePage, basePage->forms, button, buttonVal, &page); if (!page) { verbose(2, "page is NULL, qs->errMessage=[%s]\n", qs->errMessage); - if (startsWith("carefulAlloc: Allocated too much memory", qs->errMessage)) - { - verbose(1, "Response html page too large (500MB) (%s %s %s %s %s)\n", org, db, group, track, table); - fprintf(logFile, "Response html page too large (500MB) (%s %s %s %s %s)\n", org, db, group, track, table); + /* htmlPage stops reading at MAX_RESPONSE_BYTES and errAborts with this prefix. + * Before that cap existed the response was read until carefulAlloc hit its + * ceiling and called exit(1), which took the whole run down and left nothing + * in the log, so this arm never ran and the offending track was never named. + * + * This is a skip, not an error. A track dense enough to answer a 5Mb region + * with hundreds of megabytes - hg38 hgdp returns over 600MB, mm39 jaspar2024 + * about 760MB - is simply one this robot cannot test, the same situation the + * row count screen below catches before submitting. Counting it would put an + * error in every weekly run and make the summary as useless a gate as the one + * that never failed. Clearing errMessage drops it out of the error counts. */ + if (startsWith(HTML_PAGE_TOO_BIG, qs->errMessage)) + { + verbose(1, "Response html page too large (over %d bytes), skipping (%s %s %s %s %s)\n", + MAX_RESPONSE_BYTES, naForNull(org), naForNull(db), naForNull(group), + naForNull(track), naForNull(table)); + fprintf(logFile, "Response html page too large (over %d bytes), skipping (%s %s %s %s %s)\n", + MAX_RESPONSE_BYTES, naForNull(org), naForNull(db), naForNull(group), + naForNull(track), naForNull(table)); + freez(&qs->errMessage); + qs->hardError = FALSE; } else { /* Without this the caller reports only which track it was on, and the * reason the page was unusable is lost unless someone happens to re-run * at -verbose=2. */ verbose(1, "No usable page (%s %s %s %s %s): %s\n", naForNull(org), naForNull(db), naForNull(group), naForNull(track), naForNull(table), naForNull(qs->errMessage)); fprintf(logFile, "No usable page (%s %s %s %s %s): %s\n", naForNull(org), naForNull(db), naForNull(group), naForNull(track), naForNull(table), naForNull(qs->errMessage)); } } /* if (page->forms != NULL) htmlFormPrint(page->forms, stdout); */ // do not need to keep the returned structure, the answer is accumulating // in global variable: tablesTestList (void) tablesTestNew(qs, testName, org, db, group, track, table); } return page; } void serialSubmit(struct htmlPage **pPage, char *org, char *db, char *group, char *track, char *table, char *testName, char *button, char *buttonVal) /* Submit page, replacing old page with new one. */ { struct htmlPage *oldPage = *pPage; if (oldPage != NULL) { *pPage = quickSubmit(oldPage, org, db, group, track, table, testName, button, buttonVal); htmlPageFree(&oldPage); } } int tableSize(char *db, char *table) /* Return number of rows in table, or -1 if there is no such SQL table. * Plenty of tracks are offered in the table list but keep their data in a file * rather than a SQL table -- bigBed, bigWig, bigMaf and the chain/net and multiz * tracks built on them -- and counting rows on one of those used to abort the * whole process. hg38.multiz11way and mm39.netHs1 each ended a run this way. * A caller comparing against a row limit treats -1 as "no limit to apply". */ { struct sqlConnection *conn = sqlConnect(db); long size = sqlTableSizeIfExists(conn, table); sqlDisconnect(&conn); return size; } void showConnectInfo(char *db) /* Show connection info used by this program. */ { struct sqlConnection *conn = sqlConnect(db); char query[1024]; sqlSafef(query, sizeof query, "select current_user()"); char *user = sqlQuickString(conn, query); char *hostinfo = sqlHostInfo(conn); verbose(1, "Connecting as %s to database server %s\n", user, hostinfo); fprintf(logFile, "Connecting as %s to database server %s\n", user, hostinfo); fflush(logFile); sqlDisconnect(&conn); } void showRunningHostName() /* Show hostname of the machine we are running on. */ { char hostname[HOST_NAME_MAX]; if (gethostname(hostname, sizeof hostname)) { perror("gethostname"); safecpy(hostname, sizeof hostname, "error-reading-hostname"); } verbose(1, "Running on machine %s\n", hostname); fprintf(logFile, "Running on machine %s\n", hostname); fflush(logFile); } void quickErrReport() /* Report error at head of list if any */ { struct tablesTest *test = tablesTestList; if (test->status->errMessage != NULL) tablesTestLogOne(test, stderr); } void testSchema(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Make sure schema page comes up. */ /* mainForm not used */ { struct htmlPage *schemaPage = quickSubmit(tablePage, org, db, group, track, table, "schema", hgtaDoSchema, "submit"); htmlPageFree(&schemaPage); } void testSummaryStats(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Make sure summary stats page comes up. */ { if (htmlFormVarGet(mainForm, hgtaDoSummaryStats) != NULL) { struct htmlPage *statsPage = quickSubmit(tablePage, org, db, group, track, table, "summaryStats", hgtaDoSummaryStats, "submit"); htmlPageFree(&statsPage); } } boolean varIncludesType(struct htmlForm *form, char *var, char *value) /* Return TRUE if value is one of the options for var. */ { struct htmlFormVar *formVar = htmlFormVarGet(form, var); if (formVar == NULL) errAbort("Couldn't find %s variable in form", var); return slNameInList(formVar->values, value); } boolean outTypeAvailable(struct htmlForm *form, char *value) /* Return true if outType options include value. */ { return varIncludesType(form, hgtaOutputType, value); } int countNoncommentLines(char *s) /* Count number of lines in s that don't start with # */ { int count = 0; // beware, skipLeadingSpaces skips '\n' (isspace) so it skips blank lines. Check for '\n' first. if (s && *s != '\n') s = skipLeadingSpaces(s); while (s != NULL && s[0] != 0) { if (s[0] != '#') ++count; s = strchr(s, '\n'); if (s != NULL) { s++; if (*s != '\n') s = skipLeadingSpaces(s); } } return count; } int testAllFields(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Get all fields and return count of rows. */ /* mainForm not used */ { struct htmlPage *outPage; int rowCount = 0; htmlPageSetVar(tablePage, NULL, hgtaOutputType, "primaryTable"); outPage = quickSubmit(tablePage, org, db, group, track, table, "allFields", hgtaDoTopSubmit, "submit"); /* check for NULL outPage */ if (outPage == NULL) { verbose(1, "Null page in testAllFields (%s %s %s %s %s)\n", org, db, group, track, table); fprintf(logFile, "Null page in testAllFields (%s %s %s %s %s)\n", org, db, group, track, table); return -1; } rowCount = countNoncommentLines(outPage->htmlText); htmlPageFree(&outPage); return rowCount; } struct htmlFormVar *findPrefixedVar(struct htmlFormVar *list, char *prefix) /* Find first var with given prefix in list. */ { struct htmlFormVar *var; for (var = list; var != NULL; var = var->next) { if (startsWith(prefix, var->name)) return var; } return NULL; } void checkExpectedSimpleRows(struct htmlPage *outPage, int expectedRows) /* Make sure we got the number of rows we expect. */ { if (outPage != NULL) { char *results = outPage->htmlText; if (startsWith(NO_RESULTS, results)) results += strlen(NO_RESULTS); int rowCount = countNoncommentLines(results); if (rowCount != expectedRows) qaStatusSoftError(tablesTestList->status, "Got %d rows, expected %d", rowCount, expectedRows); } } void testOneField(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table, int expectedRows) /* Get one field and make sure the count agrees with expected. */ /* mainForm not used */ { struct htmlPage *outPage; struct htmlForm *form; struct htmlFormVar *var; int attempts = 0; if (tablePage->forms == NULL) errAbort("testOneField: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "selectedFields"); outPage = quickSubmit(tablePage, org, db, group, track, table, "selFieldsPage", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOneField: trying again to get selFieldsPage\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "selFieldsPage", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOneField - couldn't get outPage."); return; } if (outPage->forms == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOneField - missing form."); htmlPageFree(&outPage); return; } form = outPage->forms; var = findPrefixedVar(form->vars, "hgta_fs.check."); if (var == NULL) errAbort("No hgta_fs.check. vars in form"); htmlPageSetVar(outPage, NULL, var->name, "on"); serialSubmit(&outPage, org, db, group, track, table, "oneField", hgtaDoPrintSelectedFields, "submit"); // check that outPage != NULL checkExpectedSimpleRows(outPage, expectedRows); htmlPageFree(&outPage); } void testOutBed(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table, int expectedRows) /* Get as bed and make sure count agrees with expected. */ /* mainForm not used */ { struct htmlPage *outPage; int attempts = 0; if (tablePage->forms == NULL) errAbort("testOutBed: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "bed"); outPage = quickSubmit(tablePage, org, db, group, track, table, "bedUiPage", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOutBed: trying again to get bedUiPage\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "bedUiPage", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOneBed - couldn't get outPage."); return; } if (outPage->forms == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOneBed - missing form."); htmlPageFree(&outPage); return; } serialSubmit(&outPage, org, db, group, track, table, "outBed", hgtaDoGetBed, "submit"); // check that outPage != NULL checkExpectedSimpleRows(outPage, expectedRows); htmlPageFree(&outPage); } void testOutGff(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Get as GFF and make sure no crash. */ /* mainForm not used */ { struct htmlPage *outPage; if (tablePage->forms == NULL) errAbort("testOutGff: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "gff"); outPage = quickSubmit(tablePage, org, db, group, track, table, "outGff", hgtaDoTopSubmit, "submit"); /* no checking here */ if (outPage != NULL) htmlPageFree(&outPage); } int countTagsBetween(struct htmlPage *page, char *start, char *end, char *type) /* Count number of tags of given type (which should be upper case) * between start and end. If start is NULL it will start from * beginning of page. If end is NULL it will end at end of page. */ { int count = 0; struct htmlTag *tag; if (start == NULL) start = page->htmlText; if (end == NULL) end = start + strlen(start); for (tag = page->tags; tag != NULL; tag = tag->next) { if (tag->start >= start && tag->start < end && sameString(tag->name, type)) { ++count; } } return count; } void testOutHyperlink(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table, int expectedRows) /* Get as hyperlink and make sure count agrees with expected. */ /* mainForm not used */ { struct htmlPage *outPage; int attempts = 0; char *s; int rowCount; if (tablePage->forms == NULL) errAbort("testOutHyperlink: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "hyperlinks"); outPage = quickSubmit(tablePage, org, db, group, track, table, "outHyperlinks", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOutHyperLink: trying again to get outHyperLinks\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "outHyperlinks", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutHyperLink - couldn't get outPage."); return; } s = stringIn("", outPage->htmlText); if (s == NULL) errAbort("Can't find "); rowCount = countTagsBetween(outPage, s, NULL, "A"); if (rowCount != expectedRows) qaStatusSoftError(tablesTestList->status, "Got %d rows, expected %d", rowCount, expectedRows); htmlPageFree(&outPage); } void testOutCustomTrack(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table) /* Get as customTrack and make sure nothing explodes. */ /* mainForm not used */ { struct htmlPage *outPage; int attempts = 0; struct htmlFormVar *groupVar; if (tablePage->forms == NULL) errAbort("testOutCustomTrack: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "customTrack"); outPage = quickSubmit(tablePage, org, db, group, track, table, "customTrackUi", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOutCustomTrack: trying again to get customTrackUi\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "customTrackUi", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutCustomTrack - couldn't get outPage."); return; } serialSubmit(&outPage, org, db, group, track, table, "outCustom", hgtaDoGetCustomTrackTb, "submit"); if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutCustomTrack - serialSubmit returned null page."); return; } if (outPage->forms == NULL) { qaStatusSoftError(tablesTestList->status, "Error in custom track - no form produced."); htmlPageFree(&outPage); return; } groupVar = htmlFormVarGet(outPage->forms, hgtaGroup); if (!slNameInList(groupVar->values, "user")) { qaStatusSoftError(tablesTestList->status, "No custom track group after custom track submission"); } htmlPageFree(&outPage); } void checkFaOutput(struct htmlPage *page, int expectedCount, boolean lessOk) /* Check that page contains expected number of sequences. If lessOk is set * (needed to handle some multiply mapped cases in refSeq) then just check * that have at least one if expecting any. */ { if (page != NULL) { int count = countChars(page->htmlText, '>'); if (count != expectedCount) { if (!lessOk || count > expectedCount || (expectedCount > 0 && count <= 0)) qaStatusSoftError(tablesTestList->status, "Got %d sequences, expected %d", count, expectedCount); } } } void testOutSequence(struct htmlPage *tablePage, struct htmlForm *mainForm, char *org, char *db, char *group, char *track, char *table, int expectedRows) /* Get as sequence and make sure count agrees with expected. */ /* mainForm not used */ { struct htmlPage *outPage; int attempts = 0; struct htmlFormVar *typeVar; if (tablePage->forms == NULL) errAbort("testOutSequence: Missing form (tablePage)"); htmlPageSetVar(tablePage, NULL, hgtaOutputType, "sequence"); outPage = quickSubmit(tablePage, org, db, group, track, table, "seqUi1", hgtaDoTopSubmit, "submit"); while (outPage == NULL && attempts < MAX_ATTEMPTS) { printf("testOutSequence: trying again to get seqUi1\n"); outPage = quickSubmit(tablePage, org, db, group, track, table, "seqUi1", hgtaDoTopSubmit, "submit"); attempts++; } if (outPage == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutSequence - couldn't get outPage"); return; } if (outPage->forms == NULL) { qaStatusSoftError(tablesTestList->status, "Error in testOutSequence - missing form"); htmlPageFree(&outPage); return; } /* Since some genomic sequence things are huge, this will * only test in case where it's a gene prediction. */ typeVar = htmlFormVarGet(outPage->forms, hgtaGeneSeqType); if (typeVar != NULL) { struct htmlPage *seqPage; static char *types[] = {"protein", "mRNA"}; int i; for (i=0; ivalues, type)) { struct htmlPage *page; char testName[128]; htmlPageSetVar(outPage, NULL, hgtaGeneSeqType, type); safef(testName, sizeof(testName), "%sSeq", type); page = quickSubmit(outPage, org, db, group, track, table, testName, hgtaDoGenePredSequence, "submit"); checkFaOutput(page, expectedRows, TRUE); htmlPageFree(&page); } } htmlPageSetVar(outPage, NULL, hgtaGeneSeqType, "genomic"); serialSubmit(&outPage, org, db, group, track, table, "seqUi2", hgtaDoGenePredSequence, "submit"); // check that outPage != NULL /* On genomic page uncheck intron if it's there, then get results * and count them. */ if (htmlFormVarGet(outPage->forms, "hgSeq.intron") != NULL) htmlPageSetVar(outPage, NULL, "hgSeq.intron", NULL); seqPage = quickSubmit(outPage, org, db, group, track, table, "genomicSeq", hgtaDoGenomicDna, "submit"); // check that seqPage != NULL checkFaOutput(seqPage, expectedRows, FALSE); htmlPageFree(&seqPage); } htmlPageFree(&outPage); } boolean isObsolete(char *table) /* Some old table types we can't handle. Just warn that * they are there and skip. */ { boolean obsolete = sameString(table, "wabaCbr"); if (obsolete) qaStatusSoftError(tablesTestList->status, "Skipping obsolete table %s", table); return obsolete; } static void testOneTableBody(struct htmlPage *trackPage, char *org, char *db, char *group, char *track, char *table) /* Test stuff on one table if we haven't already tested this table. */ { /* Why declared here and not globally? */ static struct hash *uniqHash = NULL; char fullName[256]; if (uniqHash == NULL) uniqHash = newHash(0); safef(fullName, sizeof(fullName), "%s.%s", db, table); if (!hashLookup(uniqHash, fullName)) { struct htmlPage *tablePage; struct htmlForm *mainForm; hashAdd(uniqHash, fullName, NULL); verbose(1, "Testing %s %s %s %s %s\n", naForNull(org), db, group, track, table); tablePage = quickSubmit(trackPage, org, db, group, track, table, "selectTable", hgtaTable, table); if (!isObsolete(table) && tablePage != NULL) { if ((mainForm = htmlFormGet(tablePage, "mainForm")) == NULL) { qaStatusSoftError(tablesTestList->status, "Couldn't get main form on tablePage for %s %s %s %s", db, group, track, table); } else { verbose(3, "testOneTable testSchema() got here 1.1\n"); testSchema(tablePage, mainForm, org, db, group, track, table); verbose(3, "testOneTable testSummaryStats() got here 1.2\n"); testSummaryStats(tablePage, mainForm, org, db, group, track, table); verbose(3, "testOneTable got here 1.3\n"); if (outTypeAvailable(mainForm, "bed")) { verbose(3, "testOneTable bed output avail means can filter on position got here 2\n"); /* A region bounds the output for almost every table, but a whole-genome * table of hundreds of millions of rows is dense enough that all-fields * output for the test region can still pass the carefulAlloc ceiling. * carefulAlloc exits the process outright, which forfeits every table * left in the run, so screen the worst offenders out the way the * no-position-filter branch below does. */ int tableRows = tableSize(db, table); if (tableRows >= MAX_ROWS_REGION_FILTERED) { verbose(1, "%s.%s tableRows=%d, too large >= %d even with position filtering, skipping.\n", db, table, tableRows, MAX_ROWS_REGION_FILTERED); fprintf(logFile, "%s.%s tableRows=%d, too large >= %d even with position filtering, skipping.\n", db, table, tableRows, MAX_ROWS_REGION_FILTERED); } else if (outTypeAvailable(mainForm, "primaryTable")) { verbose(3, "testOneTable got here 3\n"); int rowCount = testAllFields(tablePage, mainForm, org, db, group, track, table); if (rowCount >= 0) { testOneField(tablePage, mainForm, org, db, group, track, table, rowCount); testOutSequence(tablePage, mainForm, org, db, group, track, table, rowCount); testOutBed(tablePage, mainForm, org, db, group, track, table, rowCount); testOutHyperlink(tablePage, mainForm, org, db, group, track, table, rowCount); testOutGff(tablePage, mainForm, org, db, group, track, table); if (rowCount > 0) testOutCustomTrack(tablePage, mainForm, org, db, group, track, table); } } } else if (outTypeAvailable(mainForm, "primaryTable")) { verbose(3, "testOneTable no bed output available, so no position filtering available. got here 4\n"); /* If BED type is not available then the region will be ignored, and * we'll end up scanning whole table. Make sure table is not huge * before proceeding. */ int tableRows = tableSize(db, table); if (tableRows < 500000) { int rowCount = testAllFields(tablePage, mainForm, org, db, group, track, table); if (rowCount >= 0) testOneField(tablePage, mainForm, org, db, group, track, table, rowCount); } else { verbose(1, "%s.%s tableRows=%d, too large >= 500000, skipping.\n", db, table, tableRows); fprintf(logFile, "%s.%s tableRows=%d, too large >= 500000, skipping.\n", db, table, tableRows); } } } htmlPageFree(&tablePage); } carefulCheckHeap(); } } void testOneTable(struct htmlPage *trackPage, char *org, char *db, char *group, char *track, char *table) /* Test one table, surviving an abort from anything it calls. Most of the * errAborts in this program are in the output tests below testOneTableBody, and * any one of them used to end the whole run. */ { struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) testOneTableBody(trackPage, org, db, group, track, table); errCatchEnd(errCatch); if (errCatch->gotError) recordAbort(errCatch->message->string, "table", org, db, group, track, table); errCatchFree(&errCatch); } static void testOneTrackBody(struct htmlPage *groupPage, char *org, char *db, char *group, char *track, int maxTables) /* Test a little something on up to maxTables in one track. */ { struct htmlPage *trackPage = quickSubmit(groupPage, org, db, group, track, NULL, "selectTrack", hgtaTrack, track); struct htmlForm *mainForm; struct htmlFormVar *tableVar; struct slName *table; int tableIx; /* A track whose page does not come back, or comes back unusable, is skipped * rather than fatal. quickSubmit has already recorded the failure in * tablesTestList, so it is counted in the final summary either way. Aborting * here used to end the whole run, which meant the summary that carries the * error counts was never written at all, and a single bad page - often just a * truncated response from a busy server - threw away an hour of testing. This * also covers the old bigPsl special case (2016-06-20). */ if (trackPage == NULL) { verbose(1, "Skipping track %s: no page returned\n", track); fprintf(logFile, "Skipping track %s: no page returned\n", track); return; } if ((mainForm = htmlFormGet(trackPage, "mainForm")) == NULL) { verbose(1, "Skipping track %s: no main form on track page\n", track); fprintf(logFile, "Skipping track %s: no main form on track page\n", track); htmlPageFree(&trackPage); return; } if ((tableVar = htmlFormVarGet(mainForm, hgtaTable)) == NULL) { verbose(1, "Skipping track %s: no table var on track page\n", track); fprintf(logFile, "Skipping track %s: no table var on track page\n", track); htmlPageFree(&trackPage); return; } // put the tables in random order: if (!noShuffle) shuffleList(&tableVar->values); for (table = tableVar->values, tableIx = 0; table != NULL && tableIx < maxTables; table = table->next) { if (clTable && !sameString(clTable, table->name)) continue; testOneTable(trackPage, org, db, group, track, table->name); ++tableIx; } /* Clean up. */ htmlPageFree(&trackPage); } void testOneTrack(struct htmlPage *groupPage, char *org, char *db, char *group, char *track, int maxTables) /* Test one track, surviving an abort. */ { struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) testOneTrackBody(groupPage, org, db, group, track, maxTables); errCatchEnd(errCatch); if (errCatch->gotError) recordAbort(errCatch->message->string, "track", org, db, group, track, NULL); errCatchFree(&errCatch); } static void testOneGroupBody(struct htmlPage *dbPage, char *org, char *db, char *group, int maxTracks) /* Test a little something on up to maxTracks in one group */ { struct htmlPage *groupPage = quickSubmit(dbPage, org, db, group, NULL, NULL, "selectGroup", hgtaGroup, group); struct htmlForm *mainForm; struct htmlFormVar *trackVar; struct slName *track; int trackIx; /* As in testOneTrack, a group we cannot read is skipped rather than fatal, so * that one bad page does not cost the whole run. */ if (groupPage == NULL) { verbose(1, "Skipping group %s: no page returned\n", group); fprintf(logFile, "Skipping group %s: no page returned\n", group); return; } if ((mainForm = htmlFormGet(groupPage, "mainForm")) == NULL) { verbose(1, "Skipping group %s: no main form on group page\n", group); fprintf(logFile, "Skipping group %s: no main form on group page\n", group); htmlPageFree(&groupPage); return; } if ((trackVar = htmlFormVarGet(mainForm, hgtaTrack)) == NULL) { verbose(1, "Skipping group %s: no track var on group page\n", group); fprintf(logFile, "Skipping group %s: no track var on group page\n", group); htmlPageFree(&groupPage); return; } // put the tracks in random order: if (!noShuffle) shuffleList(&trackVar->values); for (track = trackVar->values, trackIx = 0; track != NULL && trackIx < maxTracks; track = track->next) { if (clTrack && !sameString(track->name, clTrack)) continue; testOneTrack(groupPage, org, db, group, track->name, clTables); ++trackIx; } /* Clean up. */ htmlPageFree(&groupPage); } void testOneGroup(struct htmlPage *dbPage, char *org, char *db, char *group, int maxTracks) /* Test one group, surviving an abort. */ { struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) testOneGroupBody(dbPage, org, db, group, maxTracks); errCatchEnd(errCatch); if (errCatch->gotError) recordAbort(errCatch->message->string, "group", org, db, group, NULL, NULL); errCatchFree(&errCatch); } void testGroups(struct htmlPage *dbPage, char *org, char *db, int maxGroups) /* Test a little something in all groups for dbPage. */ { struct htmlForm *mainForm; struct htmlFormVar *groupVar; struct slName *group; int groupIx; if ((mainForm = htmlFormGet(dbPage, "mainForm")) == NULL) errAbort("Couldn't get main form on dbPage"); if ((groupVar = htmlFormVarGet(mainForm, hgtaGroup)) == NULL) errAbort("Can't find group var"); for (group = groupVar->values, groupIx=0; group != NULL && groupIx < maxGroups; group = group->next) { if (!sameString("allTables", group->name)) { if (clGroup && !sameString(clGroup, group->name)) continue; testOneGroup(dbPage, org, db, group->name, clTracks); ++groupIx; } } } void getTestRegion(char *db, char region[256], int regionSize) /* Look up first chromosome in database and grab five million bases * from the middle of it. */ { struct sqlConnection *conn = sqlConnect(db); char query[1024]; sqlSafef(query, sizeof query, "select * from chromInfo limit 1"); struct sqlResult *sr = sqlGetResult(conn, query); char **row; struct chromInfo ci; int start,end,middle; if ((row = sqlNextRow(sr)) == NULL) errAbort("Couldn't get one row from chromInfo"); chromInfoStaticLoad(row, &ci); middle = ci.size/2; start = middle-2500000; end = middle+2500000; if (start < 0) start = 0; if (end > ci.size) end = ci.size; safef(region, regionSize, "%s:%d-%d", ci.chrom, start+1, end); verbose(1, "Testing %s at position %s\n", db, region); fprintf(logFile, "Testing %s at position %s\n", db, region); sqlFreeResult(&sr); sqlDisconnect(&conn); } /* The hgTables page no longer carries clade/organism/assembly dropdowns -- the * gateway moved to a search box -- so there are no