3b7af84b7d8ebe9d30811103accd61584add9d9a
galt
  Sun Mar 19 22:24:29 2017 -0700
Initial check-in of proxy stuff. Added support for https_proxy for configuring https proxy. Added no_proxy for configuring domain suffixes which should be excluded from proxying. src/product/README.proxy updated. There are probably some documentation pages that will still need updating.

diff --git src/lib/https.c src/lib/https.c
index 9c613c9..d8960ed 100644
--- src/lib/https.c
+++ src/lib/https.c
@@ -40,30 +40,31 @@
 int numLocks = CRYPTO_num_locks();
 AllocArray(mutexes, numLocks);
 for (i = 0;  i < numLocks;  i++)
     pthread_mutex_init(&mutexes[i], NULL);
 CRYPTO_set_id_callback(openssl_id_callback);
 CRYPTO_set_locking_callback(openssl_locking_callback);
 }
  
 
 struct netConnectHttpsParams
 /* params to pass to thread */
 {
 pthread_t thread;
 char *hostName;
 int port;
+boolean noProxy;
 int sv[2]; /* the pair of socket descriptors */
 };
 
 static void xerrno(char *msg)
 {
 fprintf(stderr, "%s : %s\n", strerror(errno), msg); fflush(stderr);
 }
 
 static void xerr(char *msg)
 {
 fprintf(stderr, "%s\n", msg); fflush(stderr);
 }
 
 void openSslInit()
 /* do only once */
@@ -82,104 +83,161 @@
     }
 pthread_mutex_unlock( &osiMutex );
 }
 
 
 void *netConnectHttpsThread(void *threadParam)
 /* use a thread to run socket back to user */
 {
 /* child */
 
 struct netConnectHttpsParams *params = threadParam;
 
 pthread_detach(params->thread);  // this thread will never join back with it's progenitor
 
 int fd=0;
+char *proxyUrl = getenv("https_proxy");
+if (params->noProxy)
+    proxyUrl = NULL;
+char *connectHost;
+int connectPort;
 
-char hostnameProto[256];
-
-BIO *sbio;
+BIO *sbio, *ssbio;
 SSL_CTX *ctx;
 SSL *ssl;
 
 openSslInit();
 
 ctx = SSL_CTX_new(SSLv23_client_method());
 
 fd_set readfds;
 fd_set writefds;
 int err;
 struct timeval tv;
 
 
 /* TODO checking certificates 
 
 char *certFile = NULL;
 char *certPath = NULL;
 if (certFile || certPath)
     {
     SSL_CTX_load_verify_locations(ctx,certFile,certPath);
 #if (OPENSSL_VERSION_NUMBER < 0x0090600fL)
     SSL_CTX_set_verify_depth(ctx,1);
 #endif
     }
 
 // verify paths and mode.
 
 */
 
 
-sbio = BIO_new_ssl_connect(ctx);
 
-BIO_get_ssl(sbio, &ssl);
-if(!ssl) 
+// Don't want any retries since we are non-blocking bio now
+// This is available on newer versions of openssl
+//SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
+
+// Support for Http Proxy
+struct netParsedUrl pxy;
+if (proxyUrl)
     {
-    xerr("Can't locate SSL pointer");
+    netParseUrl(proxyUrl, &pxy);
+    if (!sameString(pxy.protocol, "http"))
+	errAbort("Unknown proxy protocol %s in %s.", pxy.protocol, proxyUrl);
+    connectHost = pxy.host;
+    connectPort = atoi(pxy.port);
+    }
+else
+    {
+    connectHost = params->hostName;
+    connectPort = params->port;
+    }
+fd = netConnect(connectHost,connectPort);
+if (fd == -1)
+    {
+    xerr("netConnect() failed");
     goto cleanup;
     }
 
+if (proxyUrl)
+    {
+    verbose(2, "CONNECT %s:%d HTTP/1.0 via %s:%d\n", params->hostName, params->port, connectHost,connectPort);
+    struct dyString *dy = newDyString(512);
+    dyStringPrintf(dy, "CONNECT %s:%d HTTP/1.0\r\n", params->hostName, params->port);
+    setAuthorization(pxy, "Proxy-Authorization", dy);
+    dyStringAppend(dy, "\r\n");
+    mustWriteFd(fd, dy->string, dy->stringSize);
+    dyStringFree(&dy);
+    // verify response
+    char *newUrl = NULL;
+    boolean success = netSkipHttpHeaderLinesWithRedirect(fd, proxyUrl, &newUrl);
+    if (!success) 
+	{
+	xerr("proxy server response failed");
+	goto cleanup;
+	}
+    if (newUrl) /* no redirects */
+	{
+	xerr("proxy server response should not be a redirect");
+	goto cleanup;
+	}
+    }
 
 
-/* Don't want any retries since we are non-blocking bio now */
-//SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
-
+sbio=BIO_new_socket(fd,BIO_NOCLOSE);
+if (sbio == NULL)
+    {
+    xerr("BIO_new_socket() failed");
+    goto cleanup;
+    }
+if ((ssbio = BIO_new_ssl(ctx, 1)) == NULL) 
+    {
+    xerr("BIO_new_ssl() failed");
+    goto cleanup;
+    }
+sbio = BIO_push(ssbio, sbio);
+BIO_get_ssl(sbio, &ssl);
+if(!ssl) 
+    {
+    xerr("Can't locate SSL pointer");
+    goto cleanup;
+    }
 
-safef(hostnameProto,sizeof(hostnameProto),"%s:%d",params->hostName,params->port);
-BIO_set_conn_hostname(sbio, hostnameProto);
 
 /* 
 Server Name Indication (SNI)
 Required to complete tls ssl negotiation for systems which house multiple domains. (SNI)
 This is common when serving HTTPS requests with a wildcard certificate (*.domain.tld).
 This line will allow the ssl connection to send the hostname at tls negotiation time.
 It tells the remote server which hostname the client is connecting to.
 The hostname must not be an IP address.
 */ 
 if (!internetIsDottedQuad(params->hostName))
     SSL_set_tlsext_host_name(ssl,params->hostName);
 
 BIO_set_nbio(sbio, 1);     /* non-blocking mode */
 
 while (1) 
     {
-    if (BIO_do_connect(sbio) == 1) 
+    if (BIO_do_handshake(sbio) == 1) 
 	{
 	break;  /* Connected */
 	}
     if (! BIO_should_retry(sbio)) 
 	{
-	xerr("BIO_do_connect() failed");
+	xerr("BIO_do_handshake() failed");
 	char s[256];	
 	safef(s, sizeof s, "SSL error: %s", ERR_reason_error_string(ERR_get_error()));
 	xerr(s);
 	goto cleanup;
 	}
 
     fd = BIO_get_fd(sbio, NULL);
     if (fd == -1) 
 	{
 	xerr("unable to get BIO descriptor");
 	goto cleanup;
 	}
     FD_ZERO(&readfds);
     FD_ZERO(&writefds);
     if (BIO_should_read(sbio)) 
@@ -370,48 +428,49 @@
 		    if ((errno != 104)  // udcCache often closes causing "Connection reset by peer"
 		     && (errno !=  32)) // udcCache often closes causing "Broken pipe"
 			xerrno("error writing https data back to user pipe");
 		    goto cleanup;
 		    }
 		bwt += bwtx;
 		}
 	    brd = 0;
 	    bwt = 0;
 	    }
 	}
     }
 
 cleanup:
 
-BIO_free_all(sbio);
+BIO_free_all(sbio);  // will free entire chain of bios
 close(params->sv[1]);  /* we are done with it */
 
 return NULL;
 }
 
-int netConnectHttps(char *hostName, int port)
+int netConnectHttps(char *hostName, int port, boolean noProxy)
 /* Return socket for https connection with server or -1 if error. */
 {
 
 fflush(stdin);
 fflush(stdout);
 fflush(stderr);
 
 struct netConnectHttpsParams *params;
 AllocVar(params);
 params->hostName = cloneString(hostName);
 params->port = port;
+params->noProxy = noProxy;
 
 socketpair(AF_UNIX, SOCK_STREAM, 0, params->sv);
 
 // netBlockBrokenPipes(); works, but is heavy handed 
 //  and ignores SIGPIPE on all connections for all threads in the entire application. 
 // We are trying something more subtle and library and thread-friendly instead.
 
 int rc;
 rc = pthread_create(&params->thread, NULL, netConnectHttpsThread, (void *)params);
 if (rc)
     {
     errAbort("Unexpected error %d from pthread_create(): %s",rc,strerror(rc));
     }
 
 /* parent */