e81efb1074d9786436ed8cdb626fc3bdeb6ac14c
max
  Thu Aug 6 09:20:30 2026 -0700
hgLogin: fix the social-login/email-link code review issues from #38008

#Preview2 week - bugs introduced now will need a build patch to fix
Brian, thanks for the thorough review - every one of these was real. Here is what
I changed for the six items that stayed on the ticket (the pre-existing XSS and the
site-wide httpsCertCheck default went to #38011 and #38012).

1. Reflected XSS in the account chooser and the other new pages. Every address and
username now goes through htmlEncode() before it lands in the HTML or an attribute
(chooseAccountPage, completeAccountPage, emailLinkPage, changeEmailPage and the
confirmation pages). I also gated the email-link side of the chooser on
emailLinkEnabled(), so your chooseAccount + emailLogin_email=<img ...> URL now
renders the tag as text and does nothing at all where the feature is off.

2. Registering someone else's address to steal their social login. The two OAuth
email-match queries (resolveIdentity and chooseAccount) now require
accountActivated='Y', so an unactivated row someone planted with a victim's address
can no longer be matched or linked. completeAccount only marks the new account
activated when the provider actually verified the address and the user kept it;
otherwise it creates the account inactive and sends the usual confirmation mail, so
an unverified address can never be planted as a trusted one.

3. OAuth requests not enforcing the server certificate. Rather than poke the env var,
I added a small library knob, httpsSetCertCheck() in lib/https.c, that pins the
cert-check mode for the rest of the process and is not overwritten by openSslInit()
or hg.conf. hgLogin's httpRequest() calls it with "abort", so those requests refuse
a bad certificate no matter how the site is configured, and it no longer depends on
being the first HTTPS connection.

4. The pending-identity signature. It now also covers the hguid (which survives the
provider redirect, unlike the hgsid) and the time it was minted, with a 15-minute
expiry, and it is cleared on the failure paths too. A signature that leaks into a
saved or shared session is now useless to another browser and dies quickly anyway.

5. changeEmail. It now asks for the current password where the account has one, and it
no longer changes the address on the spot - it emails a one-time signed confirmation
link to the new address and only applies the change when that link is opened. When the
change lands it also mails the OLD address to say the account's email was changed and
who to contact if that wasn't them, so a hijack gets noticed. One honest caveat: an
account with no password (social-only) still can't be re-checked before the change, so
a stolen cookie could still start it - but the old-address alert now gives the owner a
way to catch it. Expiring login cookies is the deeper fix and feels like its own ticket.

6. isalnum() on a signed char in suggestUsername - now cast to unsigned char.

Build is clean, no new warnings. Set back to you for another look.

refs #38008

diff --git src/lib/https.c src/lib/https.c
index ece524484b8..9c8e09e3ccb 100644
--- src/lib/https.c
+++ src/lib/https.c
@@ -1,988 +1,1004 @@
 /* Connect via https. */
 
 /* Copyright (C) 2012 The Regents of the University of California 
  * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */
 
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 
 #include <openssl/x509v3.h>
 #include <openssl/x509_vfy.h>
 
 #include <sys/socket.h>
 #include <unistd.h>
 #include <pthread.h>
 #include <signal.h>
 
 #include "common.h"
 #include "internet.h"
 #include "errAbort.h"
 #include "hash.h"
 #include "net.h"
 
 char *https_cert_check = "log";                 // DEFAULT certificate check is log.
 char *https_cert_check_depth = "9";             // DEFAULT depth check level is 9.
 char *https_cert_check_verbose = "off";         // DEFAULT verbose is off.
 char *https_cert_check_domain_exceptions = "";  // DEFAULT space separated list is empty string.
+static boolean https_cert_check_forced = FALSE; // TRUE once httpsSetCertCheck() pins the mode,
+                                                 // so openSslInit() won't override it from env.
 
 char *https_proxy = NULL;
 char *log_proxy = NULL;
 
 char *SCRIPT_NAME = NULL;
 
 // For use with callback. Set a variable into the connection itself,
 // and then use that during the callback.
 struct myData
     {
     char *hostName;
     };
 
 int myDataIndex = -1;
 
 static pthread_mutex_t *mutexes = NULL;
  
 unsigned long openssl_id_callback(void)
 {
 return ((unsigned long)pthread_self());
 }
  
 void openssl_locking_callback(int mode, int n, const char * file, int line)
 {
 if (mode & CRYPTO_LOCK)
     pthread_mutex_lock(&mutexes[n]);
 else
     pthread_mutex_unlock(&mutexes[n]);
 }
  
 static void openssl_pthread_setup(void)
 {
 int i;
 int numLocks = CRYPTO_num_locks();
 AllocArray(mutexes, numLocks);
 for (i = 0;  i < numLocks;  i++)
     pthread_mutex_init(&mutexes[i], NULL);
 CRYPTO_set_id_callback(openssl_id_callback);
 CRYPTO_set_locking_callback(openssl_locking_callback);
 }
  
 
 struct netConnectHttpsParams
 /* params to pass to thread */
 {
 pthread_t thread;
 int sv[2]; /* the pair of socket descriptors */
 BIO *sbio;  // ssl bio
 };
 
 static void xerrno(char *msg)
 {
 fprintf(stderr, "%s : %s\n", strerror(errno), msg); fflush(stderr);
 }
 
 static void xerr(char *msg)
 {
 fprintf(stderr, "%s\n", msg); fflush(stderr);
 }
 
 void initDomainWhiteListHash();   // forward declaration
 
 void myGetenv(char **pMySetting, char *envSetting)
 /* avoid setenv which causes problems in multi-threaded programs
  * cloning the env var helps isolate it from other threads activity. */
 {
 char *value = getenv(envSetting);
 if (value)
      *pMySetting = cloneString(value);
 }
 
+void httpsSetCertCheck(char *mode)
+/* Pin the TLS certificate-check mode ("abort", "warn", or "log") for every HTTPS connection this
+ * process makes from here on, overriding hg.conf's httpsCertCheck and the https_cert_check env
+ * var.  Use where a caller must never accept an unverified certificate however the site is
+ * configured (e.g. hgLogin's OAuth requests, which carry a client secret).  Order does not
+ * matter: openSslInit() will not overwrite a pinned value from the environment, and
+ * verify_callback reads the live value on each connection, so this takes effect whether it is
+ * called before or after the first HTTPS connection. */
+{
+https_cert_check = cloneString(mode);
+https_cert_check_forced = TRUE;
+}
+
 void openSslInit()
 /* do only once */
 {
 static boolean done = FALSE;
 static pthread_mutex_t osiMutex = PTHREAD_MUTEX_INITIALIZER;
 pthread_mutex_lock( &osiMutex );
 if (!done)
     {
     // setenv avoided since not thread-safe
+    if (!https_cert_check_forced)   // httpsSetCertCheck() wins over the env var
         myGetenv(&https_cert_check,               "https_cert_check");
     myGetenv(&https_cert_check_depth,             "https_cert_check_depth");
     myGetenv(&https_cert_check_verbose,           "https_cert_check_verbose");
     myGetenv(&https_cert_check_domain_exceptions, "https_cert_check_domain_exceptions");
     myGetenv(&https_proxy, "https_proxy");
     myGetenv(&log_proxy,   "log_proxy");
     myGetenv(&SCRIPT_NAME, "SCRIPT_NAME");
 
     SSL_library_init();
     ERR_load_crypto_strings();
     SSL_load_error_strings();   // ERR_load_SSL_strings(); deprecated.
     OpenSSL_add_all_algorithms();
     openssl_pthread_setup();
     myDataIndex = SSL_get_ex_new_index(0, "myDataIndex", NULL, NULL, NULL);
     initDomainWhiteListHash();
     done = TRUE;
     }
 pthread_mutex_unlock( &osiMutex );
 }
 
 
 void *netConnectHttpsThread(void *threadParam)
 /* use a thread to run socket back to user */
 {
 /* child */
 
 struct netConnectHttpsParams *params = threadParam;
 
 pthread_detach(params->thread);  // this thread will never join back with it's progenitor
 
 
 /* we need to wait on both the user's socket and the BIO SSL socket 
  * to see if we need to ferry data from one to the other */
 
 fd_set readfds;
 fd_set writefds;
 
 struct timeval tv;
 int err;
 
 char sbuf[32768];  // socket buffer sv[1] to user
 char bbuf[32768];  // bio buffer
 int srd = 0;
 int swt = 0;
 int brd = 0;
 int bwt = 0;
 int fd = 0;
 
 while (1) 
     {
     // Do not move this to before the loop.
     /* Get underlying file descriptor, needed for select call */
     fd = BIO_get_fd(params->sbio, NULL);
     if (fd == -1) 
 	{
 	xerr("BIO doesn't seem to be initialized in https, unable to get descriptor.");
 	goto cleanup;
 	}
 
     // The earlier call to BIO_set_nbio() should have turned non-blocking io on already.
 #if defined(__APPLE__) && defined(__clang__)
     if (fcntl(fd, F_SETFL, O_NONBLOCK) == -1)
 #else
     if (fcntl(fd, F_SETFL, SOCK_NONBLOCK) == -1)
 #endif
         {
 	xerr("Could not switch to non-blocking.\n");
 	goto cleanup;
 	}
 
     FD_ZERO(&readfds);
     FD_ZERO(&writefds);
 
     if (brd == 0)
 	FD_SET(fd, &readfds);
     if (swt < srd)
 	FD_SET(fd, &writefds);
     if (srd == 0)
 	FD_SET(params->sv[1], &readfds);
 
     tv.tv_sec = 90; // timeout 90 seconds needed for slow CGIs respsonse time.
     tv.tv_usec = 0;
 
     err = select(max(fd,params->sv[1]) + 1, &readfds, &writefds, NULL, &tv);
 
     /* Evaluate select() return code */
     if (err < 0) 
 	{
 	xerr("error during select()");
 	goto cleanup;
 	}
     else if (err == 0) 
 	{
 	/* Timed out - just quit */
 	xerr("https timeout expired");
 	goto cleanup;
 	}
 
     else 
 	{
 	if (FD_ISSET(params->sv[1], &readfds))
 	    {
 	    swt = 0;
 	    srd = read(params->sv[1], sbuf, 32768);
 	    if (srd == -1)
 		{
 		if (errno != 104) // udcCache often closes causing "Connection reset by peer"
 		    xerrno("error reading user pipe for https socket");
 		goto cleanup;
 		}
 	    if (srd == 0) 
 		break;  // user closed socket, we are done
 	    }
 
 	if (FD_ISSET(fd, &writefds))
 	    {
             if (swt < srd) 
 		{
 		int swtx = BIO_write(params->sbio, sbuf+swt, srd-swt);
 		if (swtx <= 0)
 		    {
 		    if (!BIO_should_write(params->sbio))
 			{
 			ERR_print_errors_fp(stderr);
 			xerr("Error writing SSL connection");
 			goto cleanup;
 			}
 		    }
 		else
 		    {
 		    swt += swtx;
 		    if (swt >= srd)
 			{
 			swt = 0;
 			srd = 0;
 			}
 		    }
 		}
 	    }
 
 	if (FD_ISSET(fd, &readfds))
 	    {
 	    bwt = 0;
 	    brd = BIO_read(params->sbio, bbuf, 32768);
 
 	    if (brd <= 0)
 		{
 		if (BIO_should_read(params->sbio))
 		    {
 		    brd = 0;
 		    continue;
 		    }
 		else
 		    {
 		    if (brd == 0) break;
 		    ERR_print_errors_fp(stderr);
 		    xerr("Error reading SSL connection");
 		    goto cleanup;
 		    }
 		}
 	    // write the https data received immediately back on socket to user, and it's ok if it blocks.
 	    while(bwt < brd)
 		{
 		int bwtx = write(params->sv[1], bbuf+bwt, brd-bwt);
 		if (bwtx == -1)
 		    {
 		    if ((errno != 104)  // udcCache often closes causing "Connection reset by peer"
 		     && (errno !=  32)) // udcCache often closes causing "Broken pipe"
 			xerrno("error writing https data back to user pipe");
 		    goto cleanup;
 		    }
 		bwt += bwtx;
 		}
 	    brd = 0;
 	    bwt = 0;
 	    }
 	}
     }
 
 cleanup:
 
 
 BIO_free_all(params->sbio);  // will free entire chain of bios
 close(fd);     // Needed because we use BIO_NOCLOSE above. Someday might want to re-use a connection.
 close(params->sv[1]);  /* we are done with it */
 
 return NULL;
 }
 
 
 static int verify_callback(int preverify_ok, X509_STORE_CTX *ctx)
 {
 char    buf[256];
 X509   *cert;
 int     err, depth;
 
 struct myData *myData;
 SSL    *ssl;
 
 cert = X509_STORE_CTX_get_current_cert(ctx);
 err = X509_STORE_CTX_get_error(ctx);
 depth = X509_STORE_CTX_get_error_depth(ctx);
 
 /*
 * Retrieve the pointer to the SSL of the connection currently treated
 * and the application specific data stored into the SSL object.
 */
 
 X509_NAME_oneline(X509_get_subject_name(cert), buf, 256);
 
 /*
 * Catch a too long certificate chain. The depth limit set using
 * SSL_CTX_set_verify_depth() is by purpose set to "limit+1" so
 * that whenever the "depth>verify_depth" condition is met, we
 * have violated the limit and want to log this error condition.
 * We must do it here, because the CHAIN_TOO_LONG error would not
 * be found explicitly; only errors introduced by cutting off the
 * additional certificates would be logged.
 */
 
 ssl = X509_STORE_CTX_get_ex_data(ctx, SSL_get_ex_data_X509_STORE_CTX_idx());
 myData = SSL_get_ex_data(ssl, myDataIndex);
 
 
 if (depth > atoi(https_cert_check_depth))
     {
     preverify_ok = 0;
     err = X509_V_ERR_CERT_CHAIN_TOO_LONG;
     X509_STORE_CTX_set_error(ctx, err);
     }
 if (sameString(https_cert_check_verbose, "on"))
     {
     fprintf(stderr,"depth=%d:%s\n", depth, buf);
     }
 if (!preverify_ok) 
     {
     if (SCRIPT_NAME)  // CGI mode
 	{
 	fprintf(stderr, "verify error:num=%d:%s:depth=%d:%s hostName=%s CGI=%s\n", err,
 	    X509_verify_cert_error_string(err), depth, buf, myData->hostName, SCRIPT_NAME);
 	}
     if (!sameString(https_cert_check, "log"))
 	{
 	char *cn = strstr(buf, "/CN=");
 	if (cn) cn+=4;  // strlen /CN=
 	if (sameString(cn, myData->hostName))
 	    warn("%s on %s", X509_verify_cert_error_string(err), cn);
 	else
 	    warn("%s on %s (%s)", X509_verify_cert_error_string(err), cn, myData->hostName);
 	}
     }
 /* err contains the last verification error.  */
 if (!preverify_ok && (err == X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT))
     {
     X509_NAME_oneline(X509_get_issuer_name(cert), buf, 256);
     fprintf(stderr, "issuer= %s\n", buf);
     }
 if (sameString(https_cert_check, "warn") || sameString(https_cert_check, "log"))
     return 1;
 else
     return preverify_ok;
 }
 
 struct hash *domainWhiteList = NULL;
 
 void initDomainWhiteListHash()
 /* Initialize once, has all the old existing domains 
  * for which cert checking is skipped since they are not compatible (yet) with openssl.*/
 {
 
 domainWhiteList = hashNew(8);
 
 // whitelisted domain exceptions set in hg.conf
 // space separated list.
 char *dmwl = cloneString(https_cert_check_domain_exceptions);
 int wordCount = chopByWhite(dmwl, NULL, 0);
 if (wordCount > 0)
     {
     char **words;
     AllocArray(words, wordCount);
     chopByWhite(dmwl, words, wordCount);
     int w;
     for(w=0; w < wordCount; w++)
 	{
 	hashStoreName(domainWhiteList, words[w]);
 	}
     freeMem(words);
     }
 freez(&dmwl);
 
 // useful for testing, turns off hardwired whitelist exceptions
 if (!hashLookup(domainWhiteList, "noHardwiredExceptions"))  
     {
     // Hardwired exceptions whitelist
     // whitelist domains used in URLs given as IPv4 or IPv6 addresses
     hashStoreName(domainWhiteList, "119.17.138.121");
     hashStoreName(domainWhiteList, "11plusprepschool.com");
     hashStoreName(domainWhiteList, "132.198.67.10");
     hashStoreName(domainWhiteList, "133.9.148.160");
     hashStoreName(domainWhiteList, "141.80.181.46");
     hashStoreName(domainWhiteList, "143.225.99.51");
     hashStoreName(domainWhiteList, "147.139.138.179");
     hashStoreName(domainWhiteList, "149.129.235.214");
     hashStoreName(domainWhiteList, "152.42.204.21");
     hashStoreName(domainWhiteList, "161.116.70.109");
     hashStoreName(domainWhiteList, "193.166.24.115");
     hashStoreName(domainWhiteList, "207.148.96.144");
     hashStoreName(domainWhiteList, "*.altius.org");
     hashStoreName(domainWhiteList, "*.apps.wistar.org");
     hashStoreName(domainWhiteList, "*.bio.ed.ac.uk");
     hashStoreName(domainWhiteList, "*.cbu.uib.no");
     hashStoreName(domainWhiteList, "*.clinic.cat");
     hashStoreName(domainWhiteList, "*.crg.eu");
     hashStoreName(domainWhiteList, "*.dwf.go.th");
     hashStoreName(domainWhiteList, "*.ezproxy.u-pec.fr");
     hashStoreName(domainWhiteList, "*.genebook.com.cn");
     hashStoreName(domainWhiteList, "*.jncasr.ac.in");
     hashStoreName(domainWhiteList, "*.sund.ku.dk");
     hashStoreName(domainWhiteList, "*.wistar.upenn.edu");
     hashStoreName(domainWhiteList, "2527521.line.6szsl57h1gngdxghwngytbxk2b83ws.burpcollaborator.net");
     hashStoreName(domainWhiteList, "2594771.line.6szsl57h1gngdxghwngytbxk2b83ws.burpcollaborator.net");
     hashStoreName(domainWhiteList, "2594771.pizza.b2egnpn4tjfqghmkrucr2ahtmksbg14q.oastify.com");
     hashStoreName(domainWhiteList, "35.80.111.76");
     hashStoreName(domainWhiteList, "4324332.pizza.b2egnpn4tjfqghmkrucr2ahtmksbg14q.oastify.com");
     hashStoreName(domainWhiteList, "50.16.251.170");
     hashStoreName(domainWhiteList, "52128.bham.ac.uk");
     hashStoreName(domainWhiteList, "54.175.59.127");
     hashStoreName(domainWhiteList, "66.154.14.49");
     hashStoreName(domainWhiteList, "Etlhp-Inspektorat.Sultengprov.Go.id");
     hashStoreName(domainWhiteList, "Hakeemacademy.jo");
     hashStoreName(domainWhiteList, "Panen33pro.co");
     hashStoreName(domainWhiteList, "Users");
     hashStoreName(domainWhiteList, "andrew.seq1.s3.amazonaws.com");
     hashStoreName(domainWhiteList, "annotation.dbi.udel.edu");
     hashStoreName(domainWhiteList, "api.wenglab.org");
     hashStoreName(domainWhiteList, "apprisws.bioinfo.cnio.es");
     hashStoreName(domainWhiteList, "arn.ugr.es");
     hashStoreName(domainWhiteList, "b2b.hci.utah.edu");
     hashStoreName(domainWhiteList, "bacpac.chori.org");
     hashStoreName(domainWhiteList, "beagrie00.bmrc.ox.ac.uk");
     hashStoreName(domainWhiteList, "bibliopam.ec-lyon.fr");
     hashStoreName(domainWhiteList, "bic2.ibi.upenn.edu");
     hashStoreName(domainWhiteList, "bifx-core3.bio.ed.ac.uk");
     hashStoreName(domainWhiteList, "biodb.kaist.ac.kr");
     hashStoreName(domainWhiteList, "bioinf.eva.mpg.de");
     hashStoreName(domainWhiteList, "bioinfo.gwdg.de");
     hashStoreName(domainWhiteList, "bioinfo5.ugr.es");
     hashStoreName(domainWhiteList, "bioinformaticspa.com");
     hashStoreName(domainWhiteList, "biorepo.epfl.ch");
     hashStoreName(domainWhiteList, "bioshare.genomecenter.ucdavis.edu");
     hashStoreName(domainWhiteList, "bricweb.sund.ku.dk");
     hashStoreName(domainWhiteList, "browser.rhesusbase.com");
     hashStoreName(domainWhiteList, "bsaa.edu.ru");
     hashStoreName(domainWhiteList, "bx.bio.jhu.edu");
     hashStoreName(domainWhiteList, "candy.seq.s3.amazonaws.com");
     hashStoreName(domainWhiteList, "cbio.ensmp.fr");
     hashStoreName(domainWhiteList, "cell-innovation.nig.ac.jp");
     hashStoreName(domainWhiteList, "centrodeatencionalusuario.uniempresarial.edu.co");
     hashStoreName(domainWhiteList, "chopchop.cbu.uib.no.");
     hashStoreName(domainWhiteList, "clip.korea.ac.kr");
     hashStoreName(domainWhiteList, "cll-lab.yalepages.org.s3.amazonaws.com");
     hashStoreName(domainWhiteList, "cloud.baker.edu.au");
     hashStoreName(domainWhiteList, "cloud.brc.hu");
     hashStoreName(domainWhiteList, "compbio.uta.fi");
     hashStoreName(domainWhiteList, "coppolalab.ucla.edu");
     hashStoreName(domainWhiteList, "costalab.ukaachen.de");
     hashStoreName(domainWhiteList, "cotneylab.cam.uchc.edu");
     hashStoreName(domainWhiteList, "cs.xuxingdianzikeji.com");
     hashStoreName(domainWhiteList, "cvmfs-hubs.vhost38.genap.ca");
     hashStoreName(domainWhiteList, "darned.ucc.ie");
     hashStoreName(domainWhiteList, "datahub-7ak6xof0.udes.genap.ca");
     hashStoreName(domainWhiteList, "datahub-7mu6z13t.udes.genap.ca");
     hashStoreName(domainWhiteList, "datahub-bx3mvzla.udes.genap.ca");
     hashStoreName(domainWhiteList, "datahub-gvhsc2p7.udes.genap.ca");
     hashStoreName(domainWhiteList, "datahub-i8kms5wt.udes.genap.ca");
     hashStoreName(domainWhiteList, "datahub-kazb7g4u.udes.genap.ca");
     hashStoreName(domainWhiteList, "datahub-nyt53rix.udes.genap.ca");
     hashStoreName(domainWhiteList, "datahub-ruigbdoq.udes.genap.ca");
     hashStoreName(domainWhiteList, "dbrip.org");
     hashStoreName(domainWhiteList, "debass.ga");
     hashStoreName(domainWhiteList, "dev.stanford.edu");
     hashStoreName(domainWhiteList, "dice-green.liai.org");
     hashStoreName(domainWhiteList, "dinglab.rimuhc.ca");
     hashStoreName(domainWhiteList, "dip.mbi.ucla.edu");
     hashStoreName(domainWhiteList, "diskasu.pku.edu.cn");
     hashStoreName(domainWhiteList, "dr-earray.chem.agilent.com");
     hashStoreName(domainWhiteList, "dropbox.ogic.ca");
     hashStoreName(domainWhiteList, "e1-lugh2.science.psu.edu");
     hashStoreName(domainWhiteList, "edbc.org");
     hashStoreName(domainWhiteList, "edn.som.umaryland.edu");
     hashStoreName(domainWhiteList, "endoquad.chenzxlab.cn");
     hashStoreName(domainWhiteList, "epigenomegateway.wustl.edu");
     hashStoreName(domainWhiteList, "eric.seq.s3.amazonaws.com");
     hashStoreName(domainWhiteList, "euL1db.unice.fr");
     hashStoreName(domainWhiteList, "eurexpress.org");
     hashStoreName(domainWhiteList, "fame.edbc.org");
     hashStoreName(domainWhiteList, "fhlife.nojo.kr");
     hashStoreName(domainWhiteList, "flamingo.psychiatry.uiowa.edu");
     hashStoreName(domainWhiteList, "flash.biohpc.swmed.edu");
     hashStoreName(domainWhiteList, "flu-infection.vhost38.genap.ca");
     hashStoreName(domainWhiteList, "fluinfection.vhost38.genap.ca");
     hashStoreName(domainWhiteList, "free.smokys.com");
     hashStoreName(domainWhiteList, "frigg.uio.no");
     hashStoreName(domainWhiteList, "ftp.fidelitypensionmanagers.com");
     hashStoreName(domainWhiteList, "ftp.stowers.org");
     hashStoreName(domainWhiteList, "g-5fb8fc.a0115.5898.data.globus.org");
     hashStoreName(domainWhiteList, "g-77af68.06752.75bc.data.globus.org");
     hashStoreName(domainWhiteList, "g-ee0263.a0115.5898.data.globus.org");
     hashStoreName(domainWhiteList, "galaxy.anunna.wur.nl");
     hashStoreName(domainWhiteList, "galaxy.genome.uab.edu");
     hashStoreName(domainWhiteList, "galaxy.med.uvm.edu");
     hashStoreName(domainWhiteList, "gb.faryabilab.com");
     hashStoreName(domainWhiteList, "gcp.wenglab.org");
     hashStoreName(domainWhiteList, "genemo.ucsd.edu");
     hashStoreName(domainWhiteList, "general.curtis-lab-analysis.cloud.edu.au");
     hashStoreName(domainWhiteList, "genie.weizmann.ac.il");
     hashStoreName(domainWhiteList, "genome");
     hashStoreName(domainWhiteList, "genome-asia.ucsc.edu");
     hashStoreName(domainWhiteList, "genome-test.cse");
     hashStoreName(domainWhiteList, "genome-tracks.ngs.omrf.in");
     hashStoreName(domainWhiteList, "genome.compbio.cs.cmu.edu");
     hashStoreName(domainWhiteList, "genome.senckenberg.de");
     hashStoreName(domainWhiteList, "genomics.virus.kyoto-u.ac.jp");
     hashStoreName(domainWhiteList, "genomicsdata.cs.ucl.ac.uk");
     hashStoreName(domainWhiteList, "gi.ucsc.edu");
     hashStoreName(domainWhiteList, "giannina.seq.s3.amazonaws.com");
     hashStoreName(domainWhiteList, "greennetwork.us.es");
     hashStoreName(domainWhiteList, "gvarianti.oasi.en.it");
     hashStoreName(domainWhiteList, "gwdu100.gwdg.de");
     hashStoreName(domainWhiteList, "hcampanha.dyndns.org");
     hashStoreName(domainWhiteList, "hci-bio-app.hci.utah.edu");
     hashStoreName(domainWhiteList, "hilbert.bio.ifi.lmu.de");
     hashStoreName(domainWhiteList, "hiview.case.edu");
     hashStoreName(domainWhiteList, "hiview10.gene.cwru.edu");
     hashStoreName(domainWhiteList, "hkgateway.med.umich.edu");
     hashStoreName(domainWhiteList, "hpc.bmrn.com");
     hashStoreName(domainWhiteList, "hprc-browser.ucsc.edu");
     hashStoreName(domainWhiteList, "hub.igh.cnrs.fr");
     hashStoreName(domainWhiteList, "human.genome.dnadigest.org");
     hashStoreName(domainWhiteList, "hyeshik.qbio.io");
     hashStoreName(domainWhiteList, "iamelf.com");
     hashStoreName(domainWhiteList, "icbi.at");
     hashStoreName(domainWhiteList, "ihec-dev.vhost38.genap.ca");
     hashStoreName(domainWhiteList, "irods-webdav.cyverse.at");
     hashStoreName(domainWhiteList, "itmat.data.s3.amazonaws.com");
     hashStoreName(domainWhiteList, "jadhavserver.usc.edu");
     hashStoreName(domainWhiteList, "jianglab.yalespace.org.s3.amazonaws.com");
     hashStoreName(domainWhiteList, "kbm7.genomebrowser.cemm.at");
     hashStoreName(domainWhiteList, "key2hair.com");
     hashStoreName(domainWhiteList, "ki-data.mit.edu");
     hashStoreName(domainWhiteList, "kiddlabshare.med.umich.edu");
     hashStoreName(domainWhiteList, "lapti.ucc.ie");
     hashStoreName(domainWhiteList, "launs.ru");
     hashStoreName(domainWhiteList, "lili.seq.s3.amazonaws.com");
     hashStoreName(domainWhiteList, "localhost");
     hashStoreName(domainWhiteList, "login.bases-doc.univ-lorraine.fr");
     hashStoreName(domainWhiteList, "longlab.uchicago.edu");
     hashStoreName(domainWhiteList, "lugh.bmb.psu.edu");
     hashStoreName(domainWhiteList, "lvgsrv1.epfl.ch");
     hashStoreName(domainWhiteList, "lyncoffee.cafe24.com");
     hashStoreName(domainWhiteList, "mahonylab.science.psu.edu");
     hashStoreName(domainWhiteList, "mbdata.upc.edu");
     hashStoreName(domainWhiteList, "mcahematopoiesis.bioinfo.cnio.es");
     hashStoreName(domainWhiteList, "medinfo.hebeu.edu.cn");
     hashStoreName(domainWhiteList, "members.cbio.mines-paristech.fr");
     hashStoreName(domainWhiteList, "microb215.med.upenn.edu");
     hashStoreName(domainWhiteList, "mitranscriptome.org");
     hashStoreName(domainWhiteList, "mmerono.carrerasresearch.org");
     hashStoreName(domainWhiteList, "mydrive.unilim.fr");
     hashStoreName(domainWhiteList, "nextcloud.ibv.csic.es");
     hashStoreName(domainWhiteList, "nextcloud.rhi.hi.is");
     hashStoreName(domainWhiteList, "nextgen.izkf.rwth-aachen.de");
     hashStoreName(domainWhiteList, "nucleome.dcmb.med.umich.edu");
     hashStoreName(domainWhiteList, "nucleus.ics.hut.fi");
     hashStoreName(domainWhiteList, "numbzone.com");
     hashStoreName(domainWhiteList, "omics.bioch.ox.ac.uk");
     hashStoreName(domainWhiteList, "onesgateway.med.umich.edu");
     hashStoreName(domainWhiteList, "orig-pintolab04.mssm.edu");
     hashStoreName(domainWhiteList, "orio.niehs.nih.gov");
     hashStoreName(domainWhiteList, "owww.molgen.npg.de");
     hashStoreName(domainWhiteList, "people.ucsc.edu");
     hashStoreName(domainWhiteList, "personal.utdallas.edu");
     hashStoreName(domainWhiteList, "pgv19.virol.ucl.ac.uk");
     hashStoreName(domainWhiteList, "pricenas.biochem.uiowa.edu");
     hashStoreName(domainWhiteList, "psangle.co.kr");
     hashStoreName(domainWhiteList, "pub.taejoonlab.org");
     hashStoreName(domainWhiteList, "public.scg.stanford.edu");
     hashStoreName(domainWhiteList, "q10marketing.com");
     hashStoreName(domainWhiteList, "rafalab.jhsph.edu");
     hashStoreName(domainWhiteList, "regmedsrv1.wustl.edu");
     hashStoreName(domainWhiteList, "rewrite.bcgsc.ca");
     hashStoreName(domainWhiteList, "rloop.hamadalab.com");
     hashStoreName(domainWhiteList, "rnaseqhub.brain.mpg.de");
     hashStoreName(domainWhiteList, "roneill-ucsc.neocent.s3.us-east-2.amazonaws.com");
     hashStoreName(domainWhiteList, "rsousaluis.co.uk");
     hashStoreName(domainWhiteList, "ruoho.uta.fi");
     hashStoreName(domainWhiteList, "sbwdev.stanford.edu");
     hashStoreName(domainWhiteList, "schatzlabucscdata.yalespace.org.s3.amazonaws.com");
     hashStoreName(domainWhiteList, "seanryderlab.org");
     hashStoreName(domainWhiteList, "sendfiles.salk.edu");
     hashStoreName(domainWhiteList, "share.ics.aalto.fi");
     hashStoreName(domainWhiteList, "shavitlab.org");
     hashStoreName(domainWhiteList, "sheba-cancer.org.il");
     hashStoreName(domainWhiteList, "shop.vbc.ac.at");
     hashStoreName(domainWhiteList, "si-ru.kr");
     hashStoreName(domainWhiteList, "silo.bioinf.uni-leipzig.de");
     hashStoreName(domainWhiteList, "singlecell.broadinstitute.org");
     hashStoreName(domainWhiteList, "spades.cgi.bch.uconn.edu");
     hashStoreName(domainWhiteList, "spinup-00218c-cllucas.lab.s3.amazonaws.com");
     hashStoreName(domainWhiteList, "sprite.ba.itb.cnr.it");
     hashStoreName(domainWhiteList, "ssglanders.fan");
     hashStoreName(domainWhiteList, "stockcenter.vdrc.at");
     hashStoreName(domainWhiteList, "sustainableadx.com");
     hashStoreName(domainWhiteList, "swaruplab.bio.uci.edu");
     hashStoreName(domainWhiteList, "t2t.gi.ucsc.edu");
     hashStoreName(domainWhiteList, "tale-nt.cac.cornell.edu");
     hashStoreName(domainWhiteList, "test.phenogen.org");
     hashStoreName(domainWhiteList, "thebasicsofit.com");
     hashStoreName(domainWhiteList, "theparkerlab.med.umich.edu");
     hashStoreName(domainWhiteList, "trackhub.facebase.org");
     hashStoreName(domainWhiteList, "trackhub2.genereg.net");
     hashStoreName(domainWhiteList, "tracks.stowers.org");
     hashStoreName(domainWhiteList, "trna.ucsc.edu");
     hashStoreName(domainWhiteList, "ucsc-track-hubs.scicore.unibas.ch");
     hashStoreName(domainWhiteList, "usevision.org");
     hashStoreName(domainWhiteList, "v91rc2.master.demo.encodedcc.org");
     hashStoreName(domainWhiteList, "v91rc3.master.demo.encodedcc.org");
     hashStoreName(domainWhiteList, "v94.rc2.demo.encodedcc.org");
     hashStoreName(domainWhiteList, "varbank.ccg.uni-koeln.de");
     hashStoreName(domainWhiteList, "virtlehre.informatik.uni-leipzig.de");
     hashStoreName(domainWhiteList, "vm-galaxy-prod.toulouse.inra.fr");
     hashStoreName(domainWhiteList, "vm10-dn4.qub.ac.uk");
     hashStoreName(domainWhiteList, "waxmanlabvm.bu.edu");
     hashStoreName(domainWhiteList, "webdisk.rsousaluis.co.uk");
     hashStoreName(domainWhiteList, "webserver-schilder-ukdri.dsi.ic.ac.uk");
     hashStoreName(domainWhiteList, "wiench.ngs.data.s3.amazonaws.com");
     hashStoreName(domainWhiteList, "wilsonlab.org");
     hashStoreName(domainWhiteList, "www-ncbi-nlm-nih-gov.bases-doc.univ-lorraine.fr");
     hashStoreName(domainWhiteList, "www.51766.net");
     hashStoreName(domainWhiteList, "www.affymetrix.com");
     hashStoreName(domainWhiteList, "www.akiko.caltech.edu");
     hashStoreName(domainWhiteList, "www.bio.ifi.lmu.de");
     hashStoreName(domainWhiteList, "www.crustcorporate.com");
     hashStoreName(domainWhiteList, "www.datadepot.rcac.purdue.edu");
     hashStoreName(domainWhiteList, "www.edbc.org");
     hashStoreName(domainWhiteList, "www.epigenomes.ca");
     hashStoreName(domainWhiteList, "www.genenetwork.org");
     hashStoreName(domainWhiteList, "www.healthstoriesonline.com");
     hashStoreName(domainWhiteList, "www.morgridge.net");
     hashStoreName(domainWhiteList, "www.morgridge.us");
     hashStoreName(domainWhiteList, "www.nitrofish.de");
     hashStoreName(domainWhiteList, "www.ogic.ca");
     hashStoreName(domainWhiteList, "www.owww.molgen.npg.de");
     hashStoreName(domainWhiteList, "www.polyweb.fr");
     hashStoreName(domainWhiteList, "www.proshoetech.com");
     hashStoreName(domainWhiteList, "www.soe.ucsc.edu");
     hashStoreName(domainWhiteList, "www.to.infn.it");
     hashStoreName(domainWhiteList, "www.v93rc2.demo.encodedcc.org");
     hashStoreName(domainWhiteList, "xyz.com");
     hashStoreName(domainWhiteList, "yakuba.uchicago.edu");
     hashStoreName(domainWhiteList, "yama-arashi.info");
     hashStoreName(domainWhiteList, "yardsacres.com");
     hashStoreName(domainWhiteList, "yoda.ust.hk");
     hashStoreName(domainWhiteList, "yui.seq.s3.amazonaws.com");
     hashStoreName(domainWhiteList, "zdzlab.einsteinmed.edu");
     hashStoreName(domainWhiteList, "zhaohua.urmc.rochester.edu");
     hashStoreName(domainWhiteList, "zhoulab.whu.edu.cn");
     hashStoreName(domainWhiteList, "zlab-annotations.umassmed.edu");
     hashStoreName(domainWhiteList, "zlab-trackhub.umassmed.edu");
     hashStoreName(domainWhiteList, "zlab-trackhub.wenglab.org");
     hashStoreName(domainWhiteList, "zlab.umassmed.edu");
     }
 
 }
 
 struct hashEl *checkIfInHashWithWildCard(char *hostName)
 /* check if in hash, and if in hash with lowest-level domain set to "*" wildcard */
 {
 struct hashEl *result = hashLookup(domainWhiteList, hostName);
 if (!result)
     {
     char *dot = strchr(hostName, '.');
     if (dot && (dot - hostName) >= 1)
 	{
         int length=strlen(hostName)+1;
 	char wildHost[length];
 	safef(wildHost, sizeof wildHost, "*%s", dot);
 	result = hashLookup(domainWhiteList, wildHost);
 	}
     }
 return result;
 }
 
 int netConnectHttps(char *hostName, int port, boolean noProxy, char *httpProtocol)
 /* Return socket for https connection with server or -1 if error.
  * httpProtocol is HTTP/1.0 or HTTP/1.1.  
  * 1.1 may only be used for non-persistent connections. Chunked encoding also not supported yet. */
 {
 
 int fd=0;
 
 // https_cert_check env var can be abort warn or none.
 
 char *connectHost;
 int connectPort;
 
 BIO *fbio=NULL;  // file descriptor bio
 BIO *sbio=NULL;  // ssl bio
 SSL_CTX *ctx;
 SSL *ssl;
 
 openSslInit();   // call early since it initializes vars from env vars in a thread-safe way.
 
 char *proxyUrl = https_proxy;
 
 if (noProxy)
     proxyUrl = NULL;
 
 
 #if OPENSSL_VERSION_NUMBER < 0x10100000L   // # 1.1
 ctx = SSL_CTX_new(TLSv1_2_client_method()); // OLD SSLv23_client_method());
 #else
 ctx = SSL_CTX_new(TLS_client_method());
 SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION);
 SSL_CTX_set_max_proto_version(ctx, TLS1_3_VERSION);
 #endif
 
 fd_set readfds;
 fd_set writefds;
 int err;
 struct timeval tv;
 
 struct myData myData;
 boolean doSetMyData = FALSE;
 
 X509_VERIFY_PARAM *param = NULL;
 
 if (!sameString(https_cert_check, "none"))
     {
     if (checkIfInHashWithWildCard(hostName))
 	{
 	// old existing domains which are not (yet) compatible with openssl.
 	if (SCRIPT_NAME)  // CGI mode
 	    {
 	    fprintf(stderr, "domain %s cert check skipped because it is white-listed as an exception.\n", hostName);
 	    }
 	}
     else
 	{
 
 	/* Enable automatic hostname checks */
 	param = SSL_CTX_get0_param(ctx);
 	X509_VERIFY_PARAM_set_hostflags(param, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS);
 	if (!X509_VERIFY_PARAM_set1_host(param, hostName, 0))   // some had strlen(hostName)
 	    {
 	    warn("SSL hostName for verify failed");
 	    return 0;
 	    }
 
 	// verify peer cert of the server.
 	// Set TRUSTED_FIRST for openssl 1.0
 	// Fixes common issue openssl 1.0 had with with LetsEncrypt certs in the Fall of 2021.
 	X509_STORE_set_flags(SSL_CTX_get_cert_store(ctx), X509_V_FLAG_TRUSTED_FIRST);
 
         // This flag causes intermediate certificates in the trust store to be treated as trust-anchors, in the same way as the self-signed root CA certificates. 
         // This makes it possible to trust certificates issued by an intermediate CA without having to trust its ancestor root CA.
         // GNU-TLS uses it, and openssl probably will do it in the future. 
         // Currently this does not fix any of our known issues with users servers certs.
 	// X509_STORE_set_flags(SSL_CTX_get_cert_store(ctx), X509_V_FLAG_PARTIAL_CHAIN);
 
 	// verify_callback gets called once per certificate returned by the server.
 	SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, verify_callback);
 
 	/*
 	 * Let the verify_callback catch the verify_depth error so that we get
 	 * an appropriate error in the logfile.
 	 */
 	SSL_CTX_set_verify_depth(ctx, atoi(https_cert_check_depth) + 1);
 
 	// VITAL FOR PROPER VERIFICATION OF CERTS
         if (fileExists("/etc/pki/tls/cert.pem"))
 	    {
 	    if (!SSL_CTX_load_verify_locations(ctx, "/etc/pki/tls/cert.pem", NULL))
 		{
 		warn("SSL set load_verify_file /etc/pki/tls/cert.pem failed");
 		}
 	    }
         else if (fileExists("/etc/ssl/certs"))
 	    {
 	    if (!SSL_CTX_load_verify_locations(ctx, NULL, "/etc/ssl/certs"))
 		{
 		warn("SSL set load_verify_dir /etc/ssl/certs failed");
 		}
 	    }
         else if (!SSL_CTX_set_default_verify_paths(ctx)) 
 	    {
 	    warn("SSL set default verify paths failed");
 	    }
 
 	// add the hostName to the structure and set it here, making it available during callback.
 	myData.hostName = hostName;
 	doSetMyData = TRUE;
 
 	} 
     }
 
 // Don't want any retries since we are non-blocking bio now
 // This is available on newer versions of openssl
 //SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);  // this has become the default, but only matters for blocking mode which we are not using.
 
 // Support for Http Proxy
 struct netParsedUrl pxy;
 if (proxyUrl)
     {
     netParseUrl(proxyUrl, &pxy);
     if (!sameString(pxy.protocol, "http"))
 	{
 	warn("Unknown proxy protocol %s in %s. Should be http.", pxy.protocol, proxyUrl);
 	goto cleanup2;
 	}
     connectHost = pxy.host;
     connectPort = atoi(pxy.port);
     }
 else
     {
     connectHost = hostName;
     connectPort = port;
     }
 fd = netConnect(connectHost,connectPort);
 if (fd == -1)
     {
     warn("netConnect() failed");
     goto cleanup2;
     }
 
 if (proxyUrl)
     {
     if (sameOk(log_proxy,"on"))
 	verbose(1, "CONNECT %s:%d %s via %s:%d\n", hostName, port, httpProtocol, connectHost,connectPort);
     struct dyString *dy = dyStringNew(512);
     dyStringPrintf(dy, "CONNECT %s:%d %s\r\n", hostName, port, httpProtocol);
     setAuthorization(pxy, "Proxy-Authorization", dy);
     dyStringAppend(dy, "\r\n");
     mustWriteFd(fd, dy->string, dy->stringSize);
     dyStringFree(&dy);
     // verify response
     char *newUrl = NULL;
     boolean success = netSkipHttpHeaderLinesWithRedirect(fd, proxyUrl, &newUrl);
     if (!success) 
 	{
 	warn("proxy server response failed");
 	goto cleanup2;
 	}
     if (newUrl) /* no redirects */
 	{
 	warn("proxy server response should not be a redirect");
 	goto cleanup2;
 	}
     }
 
 fbio=BIO_new_socket(fd,BIO_NOCLOSE);  
 // BIO_NOCLOSE because we handle closing fd ourselves.
 if (fbio == NULL)
     {
     warn("BIO_new_socket() failed");
     goto cleanup2;
     }
 sbio = BIO_new_ssl(ctx, 1);
 if (sbio == NULL) 
     {
     warn("BIO_new_ssl() failed");
     goto cleanup2;
     }
 sbio = BIO_push(sbio, fbio);
 BIO_get_ssl(sbio, &ssl);
 if(!ssl) 
     {
     warn("Can't locate SSL pointer");
     goto cleanup2;
     }
 
 if (doSetMyData)
     SSL_set_ex_data(ssl, myDataIndex, &myData);
 
 /* 
 Server Name Indication (SNI)
 Required to complete tls ssl negotiation for systems which house multiple domains. (SNI)
 This is common when serving HTTPS requests with a wildcard certificate (*.domain.tld).
 This line will allow the ssl connection to send the hostname at tls negotiation time.
 It tells the remote server which hostname the client is connecting to.
 The hostname must not be an IP address.
 */ 
 if (!isIpv4Address(hostName) && !isIpv6Address(hostName))
     SSL_set_tlsext_host_name(ssl,hostName);
 
 BIO_set_nbio(sbio, 1);     /* non-blocking mode */
 
 while (1) 
     {
     if (BIO_do_handshake(sbio) == 1) 
 	{
 	break;  /* Connected */
 	}
     if (! BIO_should_retry(sbio)) 
 	{
 	//BIO_do_handshake() failed
 	warn("SSL error: %s", ERR_reason_error_string(ERR_get_error()));
 	goto cleanup2;
 	}
 
     fd = BIO_get_fd(sbio, NULL);
     if (fd == -1) 
 	{
 	warn("unable to get BIO descriptor");
 	goto cleanup2;
 	}
     FD_ZERO(&readfds);
     FD_ZERO(&writefds);
     if (BIO_should_read(sbio)) 
 	{
 	FD_SET(fd, &readfds);
 	}
     else if (BIO_should_write(sbio)) 
 	{
 	FD_SET(fd, &writefds);
 	}
     else 
 	{  /* BIO_should_io_special() */
 	FD_SET(fd, &readfds);
 	FD_SET(fd, &writefds);
 	}
 
     tv.tv_sec = (long) (DEFAULTCONNECTTIMEOUTMSEC/1000);  // timeout default 10 seconds
     tv.tv_usec = (long) (((DEFAULTCONNECTTIMEOUTMSEC/1000)-tv.tv_sec)*1000000);
 
     err = select(fd + 1, &readfds, &writefds, NULL, &tv);
     if (err < 0) 
 	{
 	warn("select() error");
 	goto cleanup2;
 	}
 
     if (err == 0) 
 	{
 	warn("connection timeout to %s", hostName);
 	goto cleanup2;
 	}
     }
 
 struct netConnectHttpsParams *params;
 AllocVar(params);
 params->sbio = sbio;
 
 socketpair(AF_UNIX, SOCK_STREAM, 0, params->sv);
 
 netBlockBrokenPipes();
 // we had a version that was more sophisticated about blocking only the current thread,
 // but it only worked for Linux, and fixing it for MacOS would futher increase complexity with little benefit.
 // SIGPIPE is often more of a hassle than a help in may cases, so we can just ignore it.
 
 int rc;
 rc = pthread_create(&params->thread, NULL, netConnectHttpsThread, (void *)params);
 if (rc)
     {
     errAbort("Unexpected error %d from pthread_create(): %s",rc,strerror(rc));
     }
 
 return params->sv[0];
 
 /* parent */
 
 cleanup2:
 
 if (sbio)
     BIO_free_all(sbio);  // will free entire chain of bios
 if (fd != -1)
     close(fd);  // Needed because we use BIO_NOCLOSE above. Someday might want to re-use a connection.
 
 return -1;
 
 }