3228e85fa927785f7b52ad8320ef9f3484509424 galt Wed Aug 5 01:00:31 2015 -0700 Initial check-in for Add-SSL-Support-Options to jksql.c and to hgsql-and-family functions. diff --git src/hg/lib/jksql.c src/hg/lib/jksql.c index b489288..3179ca6 100644 --- src/hg/lib/jksql.c +++ src/hg/lib/jksql.c @@ -46,30 +46,38 @@ /* statistics */ static unsigned totalNumConnects = 0; static unsigned maxNumConnections = 0; struct sqlProfile /* a configuration profile for connecting to a server */ { struct sqlProfile *next; char *name; // name of profile char *host; // host name for database server unsigned int port; // port for database server char *socket; // unix-domain socket path for database server char *user; // database server user name char *password; // database server password struct slName *dbs; // database associated with profile, can be NULL. + // ssl + char *key; // path to ssl client key.pem + char *cert; // path to ssl client cert.pem + char *ca; // path to ssl certificate authority ca.pem + char *caPath; // path to directory containing ssl .pem certs + char *cipher; // list of permissible ciphers to use + char *verifyServerCert; // Client will check server cert Subject CN={host}. + // Boolean connection flag, if NON-NULL and != "0" then it is on. }; struct sqlConnection /* This is an item on a list of sql open connections. */ { MYSQL *conn; /* Connection. Can be NULL if not connected yet. */ struct sqlProfile *profile; /* profile, or NULL if not opened via a profile */ struct dlNode *node; /* Pointer to list node. */ struct dlList *resultList; /* Any open results. */ boolean hasHardLock; /* TRUE if table has a non-advisory lock. */ boolean inCache; /* debugging flag to indicate it's in a cache */ boolean isFree; /* is this connection free for reuse; alway FALSE * unless managed by a cache */ int hasTableCache; /* to avoid repeated checks for cache table name existence, -1 if not initialized yet, otherwise like a boolean */ @@ -102,97 +110,176 @@ char *query, ResGetter *getter, boolean abort); static void sqlConnectIfUnconnected(struct sqlConnection *sc, bool abort); bool sqlConnMustUseFailover(struct sqlConnection *sc); static char *envOverride(char *envName, char *defaultVal) /* look up envName in environment, if it exists and is non-empty, return its * value, otherwise return defaultVal */ { char *val = getenv(envName); if (isEmpty(val)) return defaultVal; else return val; } -static struct sqlProfile *sqlProfileNew(char *profileName, char *host, unsigned int port, - char *socket, char *user, char *password) -/* create a new profile object */ +struct sqlProfile *sqlProfileNew(char *name, char *host, unsigned int port, + char *socket, char *user, char *password, + char *key, char *cert, char *ca, char *caPath, char *cipher, char *verifyServerCert) +/* create a new profile object (does not include ->dbs) */ { struct sqlProfile *sp; AllocVar(sp); -sp->name = cloneString(profileName); +sp->name = cloneString(name); sp->host = cloneString(host); sp->port = port; sp->socket = cloneString(socket); sp->user = cloneString(user); sp->password = cloneString(password); +sp->key = cloneString(key); +sp->cert = cloneString(cert); +sp->ca = cloneString(ca); +sp->caPath = cloneString(caPath); +sp->cipher = cloneString(cipher); +sp->verifyServerCert = cloneString(verifyServerCert); return sp; } +static struct sqlProfile *sqlProfileClone(struct sqlProfile *o) +/* clone profile object (does not include ->dbs) */ +{ +struct sqlProfile *sp; +AllocVar(sp); +sp->name = cloneString(o->name); +sp->host = cloneString(o->host); +sp->port = o->port; +sp->socket = cloneString(o->socket); +sp->user = cloneString(o->user); +sp->password = cloneString(o->password); +sp->key = cloneString(o->key); +sp->cert = cloneString(o->cert); +sp->ca = cloneString(o->ca); +sp->caPath = cloneString(o->caPath); +sp->cipher = cloneString(o->cipher); +sp->verifyServerCert = cloneString(o->verifyServerCert); +return sp; +} + +struct sqlProfile *sqlProfileFromPairs(struct slPair *pairs) +/* create a new profile object (does not include ->dbs) */ +{ +struct sqlProfile *sp; +AllocVar(sp); +struct slPair *p; +for(p=pairs; p; p=p->next) + { + char *value = (char *)p->val; + if (sameString(p->name,"name")) + sp->name = cloneString(value); + if (sameString(p->name,"host")) + sp->host = cloneString(value); + if (sameString(p->name,"port")) + sp->port = atoi(value); + if (sameString(p->name,"socket")) + sp->socket = cloneString(value); + if (sameString(p->name,"user")) + sp->user = cloneString(value); + if (sameString(p->name,"password")) + sp->password = cloneString(value); + if (sameString(p->name,"key")) + sp->key = cloneString(value); + if (sameString(p->name,"cert")) + sp->cert = cloneString(value); + if (sameString(p->name,"ca")) + sp->ca = cloneString(value); + if (sameString(p->name,"caPath")) + sp->caPath = cloneString(value); + if (sameString(p->name,"cipher")) + sp->cipher = cloneString(value); + if (sameString(p->name,"verifyServerCert")) + sp->verifyServerCert = cloneString(value); + } +return sp; +} + + static void sqlProfileAssocDb(struct sqlProfile *sp, char *db) /* associate a db with a profile. If it is already associated with this * profile, don't do anything.*/ { struct sqlProfile *sp2 = hashFindVal(dbToProfile, db); if ((sp2 != NULL) && (sp2 != sp)) errAbort("databases %s already associated with profile %s, trying to associated it with %s", db, sp2->name, sp->name); if (sp2 == NULL) { hashAdd(dbToProfile, db, sp); slSafeAddHead(&sp->dbs, slNameNew(db)); } } -static void sqlProfileCreate(char *profileName, char *host, unsigned int port, - char *socket, char *user, char *password) +static void sqlProfileCreate(struct sqlProfile *sp) /* create a profile and add to global data structures */ { -struct sqlProfile *sp = sqlProfileNew(profileName, host, port, socket, user, password); hashAdd(profiles, sp->name, sp); if (sameString(sp->name, defaultProfileName)) defaultProfile = sp; // save default } static void sqlProfileAddProfIf(char *profileName) /* check if a config prefix is a profile, and if so, add a * sqlProfile object for it if doesn't already exist. */ { char *host = cfgOption2(profileName, "host"); char *portstr = cfgOption2(profileName, "port"); char *socket = cfgOption2(profileName, "socket"); char *user = cfgOption2(profileName, "user"); char *password = cfgOption2(profileName, "password"); +// ssl +char *key = cfgOption2(profileName, "key"); +char *cert = cfgOption2(profileName, "cert"); +char *ca = cfgOption2(profileName, "ca"); +char *caPath = cfgOption2(profileName, "caPath"); +char *cipher = cfgOption2(profileName, "cipher"); +char *verifyServerCert = cfgOption2(profileName, "verifyServerCert"); + unsigned int port = 0; if ((host != NULL) && (user != NULL) && (password != NULL) && (hashLookup(profiles, profileName) == NULL)) { /* for the default profile, allow environment variable override */ if (sameString(profileName, defaultProfileName)) { host = envOverride("HGDB_HOST", host); portstr = envOverride("HGDB_PORT", portstr); socket = envOverride("HGDB_SOCKET", socket); user = envOverride("HGDB_USER", user); password = envOverride("HGDB_PASSWORD", password); + // ssl + key = envOverride("HGDB_KEY", key); + cert = envOverride("HGDB_CERT", cert); + ca = envOverride("HGDB_CA", ca); + caPath = envOverride("HGDB_CAPATH", caPath); + cipher = envOverride("HGDB_CIPHER", cipher); + verifyServerCert = envOverride("HGDB_CIPHER", verifyServerCert); } if (portstr != NULL) port = atoi(portstr); - sqlProfileCreate(profileName, host, port, socket, user, password); + struct sqlProfile *sp = sqlProfileNew(profileName, host, port, socket, user, password, key, cert, ca, caPath, cipher, verifyServerCert); + sqlProfileCreate(sp); } } static void sqlProfileAddProfs(struct slName *cnames) /* load the profiles from list of config names */ { struct slName *cname; for (cname = cnames; cname != NULL; cname = cname->next) { char *dot1 = strchr(cname->name, '.'); // first dot in name if ((dot1 != NULL) && sameString(dot1, ".host")) { *dot1 = '\0'; sqlProfileAddProfIf(cname->name); *dot1 = '.'; @@ -329,55 +416,100 @@ struct slName *names = NULL; struct hashCookie cookie = hashFirst(profiles); struct hashEl* hel; while ((hel = hashNext(&cookie)) != NULL) slAddHead(&names, slNameNew(hel->name)); return names; } static void replaceStr(char **str, char *val) /* free str and replace with clone new value */ { freeMem(*str); *str = cloneString(val); } -void sqlProfileConfig(char *profileName, char *host, unsigned int port, - char *socket, char *user, char *password) +void sqlProfileConfig(struct slPair *pairs) /* Set configuration for the profile. This overrides an existing profile in * hg.conf or defines a new one. Results are unpredictable if a connect cache * has been established for this profile. */ { -struct sqlProfile* sp = sqlProfileGet(profileName, NULL); +struct sqlProfile *spIn = sqlProfileFromPairs(pairs); +struct sqlProfile *sp = sqlProfileGet(spIn->name, NULL); if (sp == NULL) - return sqlProfileCreate(profileName, host, port, socket, user, password); -replaceStr(&sp->host, host); -replaceStr(&sp->socket, socket); -sp->port = port; -replaceStr(&sp->user, user); -replaceStr(&sp->password, password); -} - -void sqlProfileConfigDefault(char *host, unsigned int port, char *socket, - char *user, char *password) + return sqlProfileCreate(spIn); +replaceStr(&sp->host, spIn->host); +replaceStr(&sp->socket, spIn->socket); +sp->port = spIn->port; +replaceStr(&sp->user, spIn->user); +replaceStr(&sp->password, spIn->password); +replaceStr(&sp->key, spIn->key); +replaceStr(&sp->cert, spIn->cert); +replaceStr(&sp->ca, spIn->ca); +replaceStr(&sp->caPath, spIn->caPath); +replaceStr(&sp->cipher, spIn->cipher); +replaceStr(&sp->verifyServerCert, spIn->verifyServerCert); +} + +void sqlProfileConfigDefault(struct slPair *pairs) /* Set configuration for the default profile. This overrides an existing * profile in hg.conf or defines a new one. Results are unpredictable if a * connect cache has been established for this profile. */ { -sqlProfileConfig(defaultProfileName, host, port, socket, user, password); +struct slPair *found = slPairFind(pairs, "name"); +if (found) + found->val = defaultProfileName; +else + slPairAdd(&pairs, "name", defaultProfileName); +sqlProfileConfig(pairs); } +char *sqlProfileToMyCnf(char *profileName) +/* Read in profile named, + * and create a multi-line setting string usable in my.cnf files. + * Return Null if profile not found. */ +{ +struct sqlProfile *sp = sqlProfileGet(profileName, NULL); +if (!sp) + return NULL; +struct dyString *dy = dyStringNew(256); +if (sp->host) + dyStringPrintf(dy, "host=%s\n", sp->host); +if (sp->user) + dyStringPrintf(dy, "user=%s\n", sp->user); +if (sp->password) + dyStringPrintf(dy, "password=%s\n", sp->password); +if (sp->port) + dyStringPrintf(dy, "port=%d\n", sp->port); +if (sp->socket) + dyStringPrintf(dy, "socket=%s\n", sp->socket); +if (sp->key) + dyStringPrintf(dy, "ssl-key=%s\n", sp->key); +if (sp->cert) + dyStringPrintf(dy, "ssl-cert=%s\n", sp->cert); +if (sp->ca) + dyStringPrintf(dy, "ssl-ca=%s\n", sp->ca); +if (sp->caPath) + dyStringPrintf(dy, "ssl-capath=%s\n", sp->caPath); +if (sp->cipher) + dyStringPrintf(dy, "ssl-cipher=%s\n", sp->cipher); +if (sp->verifyServerCert && !sameString(sp->verifyServerCert,"0")) + dyStringPrintf(dy, "ssl-verify-server-cert\n"); +return dyStringCannibalize(&dy); +} + + static void monitorInit(void) /* initialize monitoring on the first call */ { unsigned flags = 0; char *val; /* there is special code in cheap.cgi to pass these from cgiOption to env */ val = getenv("JKSQL_TRACE"); if ((val != NULL) && sameString(val, "on")) flags |= JKSQL_TRACE; val = getenv("JKSQL_PROF"); if ((val != NULL) && sameString(val, "on")) flags |= JKSQL_PROF; if (flags != 0) @@ -889,233 +1021,276 @@ } freeDlList(&sqlOpenConnections); } } static void sqlInitTracking(void) /* Initialize tracking and freeing of resources. */ { if (sqlOpenConnections == NULL) { sqlOpenConnections = newDlList(); atexit(sqlCleanupAll); } } -static struct sqlConnection *sqlConnRemoteFillIn(struct sqlConnection *sc, char *host, unsigned int port, char *socket, - char *user, char *password, +static struct sqlConnection *sqlConnRemoteFillIn(struct sqlConnection *sc, + struct sqlProfile *sp, char *database, boolean abort, boolean addAsOpen) /* Fill the sqlConnection object: Connect to database somewhere as somebody. * Database maybe NULL to just connect to the server. If abort is set display * error message and abort on error. This is the core function that connects to * a MySQL server. */ { MYSQL *conn; long deltaTime; sqlInitTracking(); sc->resultList = newDlList(); if (addAsOpen) sc->node = dlAddValTail(sqlOpenConnections, sc); long oldTime = monitorEnterTime; monitorEnterTime = 0; monitorEnter(); if ((sc->conn = conn = mysql_init(NULL)) == NULL) // no need for monitorLeave here errAbort("Couldn't connect to mySQL."); // Fix problem where client LOCAL setting is disabled by default for security mysql_options(conn, MYSQL_OPT_LOCAL_INFILE, NULL); + +// Boolean option to tell client to verify that the host server certificate Subject CN equals the hostname. +// If turned on this can defeat Man-In-The-Middle attacks. +if (sp->verifyServerCert && !sameString(sp->verifyServerCert,"0")) + { + my_bool flag = TRUE; + mysql_options(conn, MYSQL_OPT_SSL_VERIFY_SERVER_CERT, &flag); + } + +// TODO GALT should be optional based on some mysql variable setting? +if (sp->key || sp->cert || sp->ca || sp->caPath || sp->cipher) + mysql_ssl_set(conn, sp->key, sp->cert, sp->ca, sp->caPath, sp->cipher); + if (mysql_real_connect( conn, - host, /* host */ - user, /* user name */ - password, /* password */ + sp->host, /* host */ + sp->user, /* user name */ + sp->password, /* password */ database, /* database */ - port, /* port */ - socket, /* socket */ + sp->port, /* port */ + sp->socket, /* socket */ 0) /* flags */ == NULL) { monitorLeave(); monitorEnterTime = oldTime; if (abort) errAbort("Couldn't connect to database %s on %s as %s.\n%s", - database, host, user, mysql_error(conn)); + database, sp->host, sp->user, mysql_error(conn)); else if (sqlParanoid) fprintf(stderr, "ASH: Couldn't connect to database %s on %s as %s. " "mysql: %s pid=%ld\n", - database, host, user, mysql_error(conn), (long)getpid()); + database, sp->host, sp->user, mysql_error(conn), (long)getpid()); return NULL; } /* Make sure the db is correct in the connect, think usually happens if there * is a mismatch between MySQL library and code. If this happens, please * figure out what is going on. Contact markd if you need help. */ if (((conn->db != NULL) && !sameString(database, conn->db)) || ((conn->db == NULL) && (database != NULL))) errAbort("apparent mismatch between mysql.h used to compile jksql.c and libmysqlclient"); sc->db=cloneString(database); if (monitorFlags & JKSQL_TRACE) - monitorPrint(sc, "SQL_CONNECT", "%s %s", host, user); + monitorPrint(sc, "SQL_CONNECT", "%s %s", sp->host, sp->user); deltaTime = monitorLeave(); if (monitorFlags & JKSQL_TRACE) monitorPrint(sc, "SQL_TIME", "%0.3fs", ((double)deltaTime)/1000.0); monitorEnterTime = oldTime; sqlNumOpenConnections++; if (sqlNumOpenConnections > maxNumConnections) maxNumConnections = sqlNumOpenConnections; totalNumConnects++; sc->hasTableCache=-1; // -1 => not determined return sc; } -static struct sqlConnection *sqlConnRemote(char *host, unsigned int port, char *socket, - char *user, char *password, - char *database, boolean abort) +static struct sqlConnection *sqlConnRemote(struct sqlProfile* sp, char *database, boolean abort) /* Connect to database somewhere as somebody. Database maybe NULL to just * connect to the server. If abort is set display error message and abort on * error. */ { struct sqlConnection *sc; AllocVar(sc); -return sqlConnRemoteFillIn(sc, host, port, socket, user, password, database, abort, TRUE); +return sqlConnRemoteFillIn(sc, sp, database, abort, TRUE); } struct sqlConnection *sqlConnectRemote(char *host, char *user, char *password, char *database) /* Connect to database somewhere as somebody. Database maybe NULL to - * just connect to the server. Abort on error. */ + * just connect to the server. Abort on error. + * This only takes limited connection parameters. Use Full version for access to all.*/ { -return sqlConnRemote(host, 0, NULL, user, password, database, TRUE); +struct sqlProfile* sp = sqlProfileNew( + NULL, + host, 0, NULL, user, password, + NULL, NULL, NULL, NULL, NULL, NULL // ssl params + ); +return sqlConnRemote(sp, database, TRUE); } + struct sqlConnection *sqlMayConnectRemote(char *host, char *user, char *password, char *database) /* Connect to database somewhere as somebody. Database maybe NULL to - * just connect to the server. Return NULL can't connect */ + * just connect to the server. Return NULL if can't connect. + * This only takes limited connection parameters. Use Full version for access to all.*/ { -return sqlConnRemote(host, 0, NULL, user, password, database, FALSE); +struct sqlProfile* sp = sqlProfileNew( + NULL, + host, 0, NULL, user, password, + NULL, NULL, NULL, NULL, NULL, NULL // ssl params + ); +return sqlConnRemote(sp, database, FALSE); } +struct sqlConnection *sqlConnectRemoteFull(struct slPair *pairs, char *database) +/* Connect to database somewhere as somebody. Database maybe NULL to + * just connect to the server. Abort on error. + * Connection parameter pairs contains a list of name/values. */ +{ +struct sqlProfile *sp = sqlProfileFromPairs(pairs); +return sqlConnRemote(sp, database, TRUE); +} + +struct sqlConnection *sqlMayConnectRemoteFull(struct slPair *pairs, char *database) +/* Connect to database somewhere as somebody. Database maybe NULL to + * just connect to the server. + * Connection parameter pairs contains a list of name/values. Return NULL if can't connect.*/ +{ +struct sqlProfile *sp = sqlProfileFromPairs(pairs); +return sqlConnRemote(sp, database, FALSE); +} + + static struct sqlConnection *sqlUnconnectedConn(struct sqlProfile* profile, char* database) /* create a sqlConnection object that has all information to connect but is actually * not connected yet, as indicated by a NULL mysql connection pointer */ { static struct sqlConnection *sc; AllocVar(sc); sc->conn = NULL; sc->profile = profile; // remember the profile, needed to connect later sc->db = cloneString(database); sc->hasTableCache = -1; // -1 => undefined return sc; } static struct sqlConnection *sqlConnProfile(struct sqlProfile* sp, char *database, boolean abort) /* Connect to database using the profile. Database maybe NULL to connect to * the server. Optionally abort on failure. */ { bool mainAbort = abort; struct sqlConnection *sc; // get the failover profile for the profile, if it exists struct sqlProfile *failoverProf = sqlProfileGetFailover(sp, database); // if we have a failover profile, don't abort right away if (failoverProf!=NULL) mainAbort = FALSE; // connect with the default profile -sc = sqlConnRemote(sp->host, sp->port, sp->socket, sp->user, sp->password, database, mainAbort); +sc = sqlConnRemote(sp, database, mainAbort); if (failoverProf==NULL) // the default case, without a failover connection: just return sc, can be NULL return sc; // we still have a failover profile to setup: // if the requested database exists only on the failover connection, then the main connect // failed. We just connect again without a database, but note the database if (sc==NULL) { if (monitorFlags & JKSQL_TRACE) fprintf(stderr, "SQL_CONNECT_MAIN_FAIL %s\n", database); - sc = sqlConnRemote(sp->host, sp->port, sp->socket, sp->user, sp->password, NULL, TRUE); + sc = sqlConnRemote(sp, NULL, TRUE); sc->db = cloneString(database); } sc->profile = sp; // remember the profile // don't connect the failOver connection yet: lazily connect later when needed sc->failoverConn = sqlUnconnectedConn(failoverProf, database); return sc; } struct sqlConnection *sqlMayConnect(char *database) /* Connect to database on default host as default user. * Return NULL (don't abort) on failure. */ { return sqlConnProfile(sqlProfileMustGet(NULL, database), database, FALSE); } static void sqlConnectIfUnconnected(struct sqlConnection *sc, bool abort) /* Take a yet unconnected sqlConnection object and connect it to the sql server. */ { if (sc->conn!=NULL) return; char *profName = NULL; if (sc->profile) profName = sc->profile->name; struct sqlProfile *sp = sqlProfileMustGet(profName, sc->db); -sqlConnRemoteFillIn(sc, sp->host, sp->port, sp->socket, sp->user, sp->password, sc->db, abort, FALSE); +sqlConnRemoteFillIn(sc, sp, sc->db, abort, FALSE); } struct sqlConnection *sqlConnect(char *database) /* Connect to database on default host as default user. */ { struct sqlProfile *defProf = sqlProfileMustGet(NULL, database); return sqlConnProfile(defProf, database, TRUE); } struct sqlConnection *sqlConnectProfile(char *profileName, char *database) /* Connect to profile or database using the specified profile. Can specify * profileName, database, or both. The profile is the prefix to the host, * user, and password variables in .hg.conf. For the default profile of "db", * the environment variables HGDB_HOST, HGDB_USER, and HGDB_PASSWORD can * override. */ { struct sqlProfile* sp = sqlProfileMustGet(profileName, database); -return sqlConnRemote(sp->host, sp->port, sp->socket, sp->user, sp->password, database, TRUE); +return sqlConnRemote(sp, database, TRUE); } struct sqlConnection *sqlMayConnectProfile(char *profileName, char *database) /* Connect to profile or database using the specified profile. Can specify * profileName, database, or both. The profile is the prefix to the host, * user, and password variables in .hg.conf. For the default profile of "db", * the environment variables HGDB_HOST, HGDB_USER, and HGDB_PASSWORD can * override. Return NULL if connection fails or profile is not found. */ { struct sqlProfile* sp = sqlProfileGet(profileName, database); if (sp == NULL) return NULL; -return sqlConnRemote(sp->host, sp->port, sp->socket, sp->user, sp->password, database, FALSE); +return sqlConnRemote(sp, database, FALSE); } void sqlVaWarn(struct sqlConnection *sc, char *format, va_list args) /* Default error message handler. */ { MYSQL *conn = sc->conn; if (format != NULL) { vaWarn(format, args); } warn("mySQL error %d: %s (profile=%s, host=%s, db=%s)", mysql_errno(conn), mysql_error(conn), scConnProfile(sc), sqlGetHost(sc), scConnDb(sc)); } void sqlWarn(struct sqlConnection *sc, char *format, ...) /* Printf formatted error message that adds on sql @@ -2398,32 +2573,39 @@ } } return NULL; } static struct sqlConnCacheEntry *sqlConnCacheAddNew(struct sqlConnCache *cache, struct sqlProfile *profile, char *database, boolean abort) /* create and add a new connect to the cache */ { struct sqlConnection *conn; if (cache->entryCnt >= sqlConnCacheMax) errAbort("Too many open sqlConnections for cache"); if (cache->host != NULL) - conn = sqlConnRemote(cache->host, 0, NULL, cache->user, - cache->password, database, abort); + { + struct sqlProfile *clone = sqlProfileClone(profile); + clone->host = cache->host; + clone->port = 0; + clone->socket = NULL; + clone->user = cache->user; + clone->password = cache->password; + conn = sqlConnRemote(clone, database, abort); + } else { conn = sqlConnProfile(profile, database, abort); } if (conn != NULL) return sqlConnCacheAdd(cache, profile, conn); else { assert(!abort); return NULL; } } static struct sqlConnection *sqlConnCacheDoAlloc(struct sqlConnCache *cache, char *profileName,