233372100fe94b4265cc557eb80b17aef2f7720a
galt
  Sat Jun 3 17:38:52 2023 -0700
openssl does not verify host name by default, so we need to set a flag and pass the host name. This also means that urls with IP addresses instead of hostname will now give an error, but the whitelist can now just have IP addresses and it all works out. I double checked the history and there never more than 2 IP addresses in the whitelist, and these have been restored from those and also the recent full logs scan. We can watch the logs for any new IP addresses, but not expecting much to turn up. Also LetsEncrypt and other providers do not allow the IP address in the cert.

diff --git src/lib/https.c src/lib/https.c
index cd919b2..5547feb 100644
--- src/lib/https.c
+++ src/lib/https.c
@@ -1,23 +1,26 @@
 /* Connect via https. */
 
 /* Copyright (C) 2012 The Regents of the University of California 
  * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */
 
 #include "openssl/ssl.h"
 #include "openssl/err.h"
 
+#include "openssl/x509v3.h"
+#include "openssl/x509_vfy.h"
+
 #include <sys/socket.h>
 #include <unistd.h>
 #include <pthread.h>
 #include <signal.h>
 
 #include "common.h"
 #include "internet.h"
 #include "errAbort.h"
 #include "hash.h"
 #include "net.h"
 
 char *https_cert_check = "log";                 // DEFAULT certificate check is log.
 char *https_cert_check_depth = "9";             // DEFAULT depth check level is 9.
 char *https_cert_check_verbose = "off";         // DEFAULT verbose is off.
 char *https_cert_check_domain_exceptions = "";  // DEFAULT space separated list is empty string.
@@ -370,31 +373,41 @@
     AllocArray(words, wordCount);
     chopByWhite(dmwl, words, wordCount);
     int w;
     for(w=0; w < wordCount; w++)
 	{
 	hashStoreName(domainWhiteList, words[w]);
 	}
     freeMem(words);
     }
 freez(&dmwl);
 
 // useful for testing, turns off hardwired whitelist exceptions
 if (!hashLookup(domainWhiteList, "noHardwiredExceptions"))  
     {
     // Hardwired exceptions whitelist
-    // openssl automatically whitelists domains which are given as IPv4 or IPv6 addresses
+    // whitelist domains used in URLs given as IPv4 or IPv6 addresses
+    hashStoreName(domainWhiteList, "141.80.181.46");
+    hashStoreName(domainWhiteList, "119.17.138.121");
+    hashStoreName(domainWhiteList, "132.198.67.10");
+    hashStoreName(domainWhiteList, "133.9.148.160");
+    hashStoreName(domainWhiteList, "143.225.99.51");
+    hashStoreName(domainWhiteList, "147.139.138.179");
+    hashStoreName(domainWhiteList, "149.129.235.214");
+    hashStoreName(domainWhiteList, "161.116.70.109");
+    hashStoreName(domainWhiteList, "193.166.24.115");
+    hashStoreName(domainWhiteList, "66.154.14.49");
     hashStoreName(domainWhiteList, "*.altius.org");
     hashStoreName(domainWhiteList, "*.apps.wistar.org");
     hashStoreName(domainWhiteList, "*.bio.ed.ac.uk");
     hashStoreName(domainWhiteList, "*.cbu.uib.no");
     hashStoreName(domainWhiteList, "*.clinic.cat");
     hashStoreName(domainWhiteList, "*.crg.eu");
     hashStoreName(domainWhiteList, "*.dwf.go.th");
     hashStoreName(domainWhiteList, "*.ezproxy.u-pec.fr");
     hashStoreName(domainWhiteList, "*.genebook.com.cn");
     hashStoreName(domainWhiteList, "*.jncasr.ac.in");
     hashStoreName(domainWhiteList, "*.sund.ku.dk");
     hashStoreName(domainWhiteList, "*.wistar.upenn.edu");
     hashStoreName(domainWhiteList, "11plusprepschool.com");
     hashStoreName(domainWhiteList, "52128.bham.ac.uk");
     hashStoreName(domainWhiteList, "Hakeemacademy.jo");
@@ -584,43 +597,54 @@
 char *proxyUrl = https_proxy;
 
 if (noProxy)
     proxyUrl = NULL;
 
 ctx = SSL_CTX_new(SSLv23_client_method());
 
 fd_set readfds;
 fd_set writefds;
 int err;
 struct timeval tv;
 
 struct myData myData;
 boolean doSetMyData = FALSE;
 
+X509_VERIFY_PARAM *param = NULL;
+
 if (!sameString(https_cert_check, "none"))
     {
     if (checkIfInHashWithWildCard(hostName))
 	{
 	// old existing domains which are not (yet) compatible with openssl.
 	if (SCRIPT_NAME)  // CGI mode
 	    {
 	    fprintf(stderr, "domain %s cert check skipped because it is white-listed as an exception.\n", hostName);
 	    }
 	}
     else
 	{
 
+	/* Enable automatic hostname checks */
+	param = SSL_CTX_get0_param(ctx);
+	X509_VERIFY_PARAM_set_hostflags(param, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS);
+	if (!X509_VERIFY_PARAM_set1_host(param, hostName, 0))   // some had strlen(hostName)
+	    {
+	    warn("SSL hostName for verify failed");
+	    return 0;
+	    }
+
 	// verify peer cert of the server.
 	// Set TRUSTED_FIRST for openssl 1.0
 	// Fixes common issue openssl 1.0 had with with LetsEncrypt certs in the Fall of 2021.
 	X509_STORE_set_flags(SSL_CTX_get_cert_store(ctx), X509_V_FLAG_TRUSTED_FIRST);
 
         // This flag causes intermediate certificates in the trust store to be treated as trust-anchors, in the same way as the self-signed root CA certificates. 
         // This makes it possible to trust certificates issued by an intermediate CA without having to trust its ancestor root CA.
         // GNU-TLS uses it, and openssl probably will do it in the future. 
         // Currently this does not fix any of our known issues with users servers certs.
 	// X509_STORE_set_flags(SSL_CTX_get_cert_store(ctx), X509_V_FLAG_PARTIAL_CHAIN);
 
 	// verify_callback gets called once per certificate returned by the server.
 	SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, verify_callback);
 
 	/*