2dffcd60f62e3fb25bbe8b009b4235bb3b9eb44f galt Sun Mar 9 14:54:38 2014 -0700 Adding better handling of auto-upgrade for the situation where simultaneous CGIs would try to all upgrade at once. It uses an advisory lock to make sure only one process is trying to upgrade at a time, and it checks again that the field does not exist already before adding the new sessionkey field, it writes the result of success or failure with the error message to a TRASH file so that other CGIs may access it. To prevent them all from repeatedly trying to do the same upgrade with every CGI execution, it checks the time-stamp on the AUTOUPGRADE trash file, and if it is older than 3 minutes, then it will try again to see if the admins have given it the permissions needed - which will then re-write the trash file with the results and its time will be updated. If an error occurs during sql upgrade, it traps the error instead of errAborting and the message is saved to be included in the trash file, and the system proceeds for now without the new sessionKey field. diff --git src/hg/lib/cartDb.c src/hg/lib/cartDb.c index 5a3a0d2..f103917 100644 --- src/hg/lib/cartDb.c +++ src/hg/lib/cartDb.c @@ -1,314 +1,449 @@ /* cartDb.c was originally generated by the autoSql program, which also * generated cartDb.h and cartDb.sql. This module links the database and * the RAM representation of objects. */ /* - * TODO + * TODO REMOVE AFTER AUTOUPGRADE COMPLETE: (added 2014-03-09) * - * autodetect - * or even better auto-upgrade of the userDb and sessionDb tables + * autoupgrade of the userDb and sessionDb tables * alter table userDb add column sessionKey varchar(255) NOT NULL default ''; * alter table sessionDb add column sessionKey varchar(255) NOT NULL default ''; * - * find and modify the .as and .sql corresponding to cartDb - * - * */ #include "common.h" #include "linefile.h" #include "dystring.h" #include "jksql.h" #include "hgConfig.h" #include "base64.h" #include "cartDb.h" #include "cart.h" +#include "errCatch.h" +#include "portable.h" +#include "obscure.h" +#include <sys/time.h> + extern DbConnector cartDefaultConnector; extern DbDisconnect cartDefaultDisconnector; static boolean userDbInitialized = FALSE; static boolean sessionDbInitialized = FALSE; +struct dyString *dyUpgradeError = NULL; + boolean cartDbHasSessionKey(struct sqlConnection *conn, char *table) /* Check to see if the table has the sessionKey field */ { static boolean userDbHasSessionKey = FALSE; static boolean sessionDbHasSessionKey = FALSE; if (sameString(table, "userDb")) { if (!userDbInitialized) { userDbInitialized = TRUE; if (sqlFieldIndex(conn, table, "sessionKey") >= 0) { userDbHasSessionKey = TRUE; } } return userDbHasSessionKey; } else if (sameString(table, "sessionDb")) { if (!sessionDbInitialized) { sessionDbInitialized = TRUE; if (sqlFieldIndex(conn, table, "sessionKey") >= 0) { sessionDbHasSessionKey = TRUE; } } return sessionDbHasSessionKey; } else errAbort("Unknown table %s", table); return FALSE; } +#define AUTOUPGRPATHSIZE 256 +static char *makeResultName(char *tableName, char *path) +/* return path in trash for corresponding autoupgrade result file */ +{ +safef(path, AUTOUPGRPATHSIZE, "../trash/AUTO_UPGRADE_RESULT_%s", tableName); +return cloneString(path); +} + +static boolean checkAutoUpgradeTableResultTimeIsOld(char *tableName) +/* Has enough time passed since the last upgrade check? + * The idea is to only check once every few minutes + * rather than each time the CGI runs. */ +{ +char path[AUTOUPGRPATHSIZE]; +makeResultName(tableName, path); +if (!fileExists(path)) + return TRUE; // If there is no result yet we should test and make one +struct timeval rawtime; +gettimeofday( &rawtime, NULL ); +time_t now = rawtime.tv_sec; +time_t fMT = fileModTime(path); +double diff = difftime(now, fMT); +if (diff > (3 * 60)) // 3 minutes in seconds + return TRUE; // The result is old we should test it again in case situation changed +return FALSE; +} + +static char * readAutoUpgradeTableResult(char *tableName) +/* Read table upgrade result */ +{ +char path[AUTOUPGRPATHSIZE]; +char *result = NULL; +makeResultName(tableName, path); +if (!fileExists(path)) + return NULL; // There is no result yet +readInGulp(path, &result, NULL); +return result; +} + +static void writeAutoUpgradeTableResult(char *tableName, char *result) +/* Write table upgrade result */ +{ +char path[AUTOUPGRPATHSIZE]; +makeResultName(tableName, path); +writeGulp(path, result, strlen(result)); +} + + +void autoUpgradeTableAddSesssionKey(struct sqlConnection *conn, char *tableName) +/* Try to upgrade the table by adding sessionKey field + * in a safe way handling success failures and retries + * with multiple CGIs running. */ +{ + +boolean testAgain = checkAutoUpgradeTableResultTimeIsOld(tableName); +if (testAgain) + { + // Get the advisory lock for this table + // This prevents multiple CGI processes from trying to upgrade simultaneously + char lockName[256]; + safef(lockName, sizeof lockName, "AUTO_UPGRADE_%s", tableName); + sqlGetLock(conn, lockName); + + // Make sure that the table has not been already upgraded by some earlier process. + // We do not want to upgrade more than once. + if (sqlFieldIndex(conn, tableName, "sessionKey") == -1) + { + + char result[4096]; + + // Put a catch around the table upgrade attempt, + // both to allow us to continue if it fails, + // and to catch the error message and save it in the results file + struct errCatch *errCatch = errCatchNew(); + if (errCatchStart(errCatch)) + { + char query[256]; + sqlSafef(query, sizeof query, "alter table %s add column sessionKey varchar(255) NOT NULL default ''", tableName); + sqlUpdate(conn, query); + } + errCatchEnd(errCatch); + if (errCatch->gotError) + { + safef(result, sizeof result, "AUTOUPGRADE FAILED\n%s", errCatch->message->string); + } + else + { + safef(result, sizeof result, "OK\n"); + } + + errCatchFree(&errCatch); + + writeAutoUpgradeTableResult(tableName, result); + + } + + // Release the advisory lock for this table + sqlReleaseLock(conn, lockName); + + } +else + { // in the interests of speed, just use the old result + char *oldResult = readAutoUpgradeTableResult(tableName); + if (oldResult) + { + if (startsWith("AUTOUPGRADE FAILED", oldResult)) + { + // cannot do this here since it is too early and the warn handler does not work right + // it ends up writing html and javascript before the cookie and response header have even been finished. + // warn("%s", oldResult); + // instead, save the error message for access later from select places like hgGateway + if (!dyUpgradeError) + dyUpgradeError = dyStringNew(256); + else + dyStringPrintf(dyUpgradeError,"\n"); + dyStringPrintf(dyUpgradeError,"%s", oldResult); + } + } + } + +} + + + boolean cartDbUseSessionKey() /* Check settings and and state to determine if sessionKey is in use */ { static boolean initialized = FALSE; static boolean useSessionKey = FALSE; if (!initialized) { initialized = TRUE; char *sessionKey = cfgOption2("browser", "sessionKey"); if (!sessionKey) - sessionKey = "off"; // DEFAULT but this might change to another value + sessionKey = "on"; // DEFAULT but this might change to another value if (sameString(sessionKey, "on")) { useSessionKey = TRUE; struct sqlConnection *conn = cartDefaultConnector(); boolean userDbHasSessionKey = cartDbHasSessionKey(conn, "userDb"); boolean sessionDbHasSessionKey = cartDbHasSessionKey(conn, "sessionDb"); if ( ! (userDbHasSessionKey && sessionDbHasSessionKey) ) { - //errAbort("brower.sessionKey=on but userDb and sesionDb are missing the sessionKey field."); + //errAbort("brower.sessionKey=on but userDb and sessionDb are missing the sessionKey field."); // AUTO-UPGRADE tables to add missing sessionKey field here. if (!userDbHasSessionKey) - sqlUpdate(conn, "NOSQLINJ alter table userDb add column sessionKey varchar(255) NOT NULL default ''"); - if (!sessionDbHasSessionKey) - sqlUpdate(conn, "NOSQLINJ alter table sessionDb add column sessionKey varchar(255) NOT NULL default ''"); + { + autoUpgradeTableAddSesssionKey(conn, "userDb"); userDbInitialized = FALSE; + userDbHasSessionKey = cartDbHasSessionKey(conn, "userDb"); + } + if (!sessionDbHasSessionKey) + { + autoUpgradeTableAddSesssionKey(conn, "sessionDb"); sessionDbInitialized = FALSE; + sessionDbHasSessionKey = cartDbHasSessionKey(conn, "sessionDb"); + } + if ( ! (userDbHasSessionKey && sessionDbHasSessionKey) ) + useSessionKey = FALSE; } cartDefaultDisconnector(&conn); } else if (sameString(sessionKey, "off")) { useSessionKey = FALSE; } else if (sameString(sessionKey, "autodetect")) { errAbort("brower.sessionKey=autodetect has not implemented yet."); // TODO } } return useSessionKey; } char *cartDbMakeRandomKey(int numBits) /* Generate base64 encoding of a random key of at least size numBits returning string to be freed when done */ { int numBytes = (numBits + 7) / 8; // round up to nearest whole byte. numBytes = ((numBytes+2)/3)*3; // round up to the nearest multiple of 3 to avoid equals-char padding in base64 output FILE *f = mustOpen("/dev/urandom", "r"); // open random system device for read-only access. char *binaryString = needMem(numBytes); mustRead(f, binaryString, numBytes); carefulClose(&f); char * result = base64Encode(binaryString, numBytes); // converts 3 binary bytes into 4 printable characters int len = strlen(result); -memSwapChar(result, len, '+', '-'); // replace + and / with characters that are URL-friendly. -memSwapChar(result, len, '/', '*'); +memSwapChar(result, len, '+', 'A'); // replace + and / with characters that are URL-friendly. +memSwapChar(result, len, '/', 'a'); freeMem(binaryString); return result; } void cartDbSecureId(char *buf, int bufSize, struct cartDb *cartDb) /* Return combined string of session id plus sessionKey in buf if turned on.*/ { if (cartDbUseSessionKey() && !sameString(cartDb->sessionKey,"")) safef(buf, bufSize, "%d_%s", cartDb->id, cartDb->sessionKey); else safef(buf, bufSize, "%d", cartDb->id); } unsigned int cartDbParseId(char *id, char **pSessionKey) /* Parse out the numeric id and id_sessionKey string if present. */ { unsigned int result = 0; char *e = strchr(id, '_'); if (e) *e = 0; result = sqlUnsigned(id); if (e) { *e = '_'; if (pSessionKey) *pSessionKey = e+1; } else { if (pSessionKey) *pSessionKey = NULL; } return result; } void cartDbStaticLoad(char **row, struct cartDb *ret) /* Load a row from cartDb table into ret. The contents of ret will * be replaced at the next call to this function. */ { ret->id = sqlUnsigned(row[0]); ret->contents = row[1]; ret->reserved = sqlSigned(row[2]); ret->firstUse = row[3]; ret->lastUse = row[4]; ret->useCount = sqlSigned(row[5]); if (cartDbUseSessionKey()) ret->sessionKey = row[6]; } struct cartDb *cartDbLoad(char **row) /* Load a cartDb from row fetched with select * from cartDb * from database. Dispose of this with cartDbFree(). */ { struct cartDb *ret; AllocVar(ret); ret->id = sqlUnsigned(row[0]); ret->contents = cloneString(row[1]); ret->reserved = sqlSigned(row[2]); ret->firstUse = cloneString(row[3]); ret->lastUse = cloneString(row[4]); ret->useCount = sqlSigned(row[5]); if (cartDbUseSessionKey()) ret->sessionKey = cloneString(row[6]); return ret; } struct cartDb *cartDbLoadAll(char *fileName) /* Load all cartDb from a tab-separated file. * Dispose of this with cartDbFreeList(). */ { struct cartDb *list = NULL, *el; struct lineFile *lf = lineFileOpen(fileName, TRUE); char *row[6]; while (lineFileRow(lf, row)) { el = cartDbLoad(row); slAddHead(&list, el); } lineFileClose(&lf); slReverse(&list); return list; } struct cartDb *cartDbLoadWhere(struct sqlConnection *conn, char *table, char *where) /* Load all cartDb from table that satisfy where clause. The * where clause may be NULL in which case whole table is loaded * Dispose of this with cartDbFreeList(). */ { struct cartDb *list = NULL, *el; struct dyString *query = dyStringNew(256); struct sqlResult *sr; char **row; sqlDyStringPrintf(query, "select * from %s", table); if (where != NULL) dyStringPrintf(query, " where %-s", where); // the where clause must be checked by caller for sqli sr = sqlGetResult(conn, query->string); while ((row = sqlNextRow(sr)) != NULL) { el = cartDbLoad(row); slAddHead(&list, el); } slReverse(&list); sqlFreeResult(&sr); dyStringFree(&query); return list; } struct cartDb *cartDbCommaIn(char **pS, struct cartDb *ret) /* Create a cartDb out of a comma separated string. * This will fill in ret if non-null, otherwise will * return a new cartDb */ { char *s = *pS; if (ret == NULL) AllocVar(ret); ret->id = sqlUnsignedComma(&s); ret->contents = sqlStringComma(&s); ret->reserved = sqlSignedComma(&s); ret->firstUse = sqlStringComma(&s); ret->lastUse = sqlStringComma(&s); ret->useCount = sqlSignedComma(&s); if (cartDbUseSessionKey()) ret->sessionKey = sqlStringComma(&s); *pS = s; return ret; } void cartDbFree(struct cartDb **pEl) /* Free a single dynamically allocated cartDb such as created * with cartDbLoad(). */ { struct cartDb *el; if ((el = *pEl) == NULL) return; freeMem(el->contents); freeMem(el->firstUse); freeMem(el->lastUse); if (cartDbUseSessionKey()) freeMem(el->sessionKey); freez(pEl); } void cartDbFreeList(struct cartDb **pList) /* Free a list of dynamically allocated cartDb's */ { struct cartDb *el, *next; for (el = *pList; el != NULL; el = next) { next = el->next; cartDbFree(&el); } *pList = NULL; } void cartDbOutput(struct cartDb *el, FILE *f, char sep, char lastSep) /* Print out cartDb. Separate fields with sep. Follow last field with lastSep. */ { if (sep == ',') fputc('"',f); fprintf(f, "%u", el->id); if (sep == ',') fputc('"',f); fputc(sep,f); if (sep == ',') fputc('"',f); fprintf(f, "%s", el->contents); if (sep == ',') fputc('"',f); fputc(sep,f); fprintf(f, "%d", el->reserved); fputc(sep,f); if (sep == ',') fputc('"',f); fprintf(f, "%s", el->firstUse); if (sep == ',') fputc('"',f); fputc(sep,f); if (sep == ',') fputc('"',f); fprintf(f, "%s", el->lastUse); if (sep == ',') fputc('"',f); fputc(sep,f); fprintf(f, "%d", el->useCount); if (cartDbUseSessionKey()) { fputc(sep,f); fprintf(f, "%s", el->sessionKey); } fputc(lastSep,f); }