5359edc160de518d8e43fdd3448365c15b912c3c galt Mon Jul 22 11:48:10 2019 -0700 Added ipv6 support. Listening processes us hybrid dual stack feature of OS to simplify implementation and use a single listening socket. Works with both TCP and UDP. Parasol working. geoIp also updated and ready for IPv6. Should be invisible to most users, while providing connections via ipv6 where available. Supports both ipv4 and ipv6. diff --git src/lib/net.c src/lib/net.c index 60689c4..8471eca 100644 --- src/lib/net.c +++ src/lib/net.c @@ -10,37 +10,38 @@ #include <sys/time.h> #include <pthread.h> #include "errAbort.h" #include "hash.h" #include "net.h" #include "linefile.h" #include "base64.h" #include "cheapcgi.h" #include "https.h" #include "sqlNum.h" #include "obscure.h" /* Brought errno in to get more useful error messages */ extern int errno; -static int netStreamSocket() -/* Create a TCP/IP streaming socket. Complain and return something - * negative if can't */ +static int netStreamSocket6(struct addrinfo *address) +/* Create a socket from addrinfo structure. + * Complain and return something negative if can't. */ { -int sd = socket(AF_INET, SOCK_STREAM, 0); +int sd = socket(address->ai_family, address->ai_socktype, address->ai_protocol); if (sd < 0) - warn("Couldn't make AF_INET socket."); + warn("Couldn't make %s socket.", familyToString(address->ai_family)); + return sd; } static int setSocketNonBlocking(int sd, boolean set) /* Use socket control flags to set O_NONBLOCK if set==TRUE, * or clear it if set==FALSE. * Return -1 if there are any errors, 0 if successful. */ { long fcntlFlags; // Set or clear non-blocking if ((fcntlFlags = fcntl(sd, F_GETFL, NULL)) < 0) { warn("Error fcntl(..., F_GETFL) (%s)", strerror(errno)); return -1; } @@ -90,53 +91,55 @@ a.tv_usec += 1000000; a.tv_sec--; } a.tv_usec -= b.tv_usec; a.tv_sec -= b.tv_sec; return a; } static int netConnectWithTimeout(char *hostName, int port, long msTimeout) /* In order to avoid a very long default timeout (several minutes) for hosts that will * not answer the port, we are forced to connect non-blocking. * After the connection has been established, we return to blocking mode. * Also closes sd if error. */ { int sd; -struct sockaddr_in sai; /* Some system socket info. */ +struct addrinfo *address=NULL; int res; fd_set mySet; +char portStr[8]; +safef(portStr, sizeof portStr, "%d", port); if (hostName == NULL) { warn("NULL hostName in netConnect"); return -1; } -if (!internetFillInAddress(hostName, port, &sai)) +if (!internetGetAddrInfo6n4(hostName, portStr, &address)) return -1; -if ((sd = netStreamSocket()) < 0) +if ((sd = netStreamSocket6(address)) < 0) return sd; // Set socket to nonblocking so we can manage our own timeout time. if (setSocketNonBlocking(sd, TRUE) < 0) { close(sd); return -1; } // Trying to connect with timeout -res = connect(sd, (struct sockaddr*) &sai, sizeof(sai)); +res = connect(sd, address->ai_addr, address->ai_addrlen); if (res < 0) { if (errno == EINPROGRESS) { struct timeval startTime; gettimeofday(&startTime, NULL); struct timeval remainingTime; remainingTime.tv_sec = (long) (msTimeout/1000); remainingTime.tv_usec = (long) (((msTimeout/1000)-remainingTime.tv_sec)*1000000); while (1) { FD_ZERO(&mySet); FD_SET(sd, &mySet); // use tempTime (instead of using remainingTime directly) because on some platforms select() may modify the time val. struct timeval tempTime = remainingTime; @@ -203,121 +206,135 @@ } // Set to blocking mode again if (setSocketNonBlocking(sd, FALSE) < 0) { close(sd); return -1; } if (setReadWriteTimeouts(sd, DEFAULTREADWRITETTIMEOUTSEC) < 0) { close(sd); return -1; } +freeaddrinfo(address); return sd; } int netConnect(char *hostName, int port) /* Start connection with a server. */ { return netConnectWithTimeout(hostName, port, DEFAULTCONNECTTIMEOUTMSEC); // 10 seconds connect timeout } int netMustConnect(char *hostName, int port) /* Start connection with server or die. */ { int sd = netConnect(hostName, port); if (sd < 0) noWarnAbort(); return sd; } int netMustConnectTo(char *hostName, char *portName) /* Start connection with a server and a port that needs to be converted to integer */ { if (!isdigit(portName[0])) errAbort("netConnectTo: ports must be numerical, not %s", portName); return netMustConnect(hostName, atoi(portName)); } -int netAcceptingSocketFrom(int port, int queueSize, char *host) -/* Create a socket that can accept connections from a - * IP address on the current machine if the current machine - * has multiple IP addresses. */ +int netAcceptingSocket(int port, int queueSize) +/* Create an IPV6 socket that can accept connections from + * both IPV4 and IPV6 clients on the current machine. */ { -struct sockaddr_in sai; +struct sockaddr_in6 serverAddr; int sd; -int flag = 1; netBlockBrokenPipes(); -if ((sd = netStreamSocket()) < 0) - return sd; -if (!internetFillInAddress(host, port, &sai)) - return -1; -if (setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(int))) - return -1; -if (bind(sd, (struct sockaddr*)&sai, sizeof(sai)) == -1) + +// Hybrid dual stack ipv6 listening socket accepts ipv6 and ipv4 mapped connections. +if ((sd = socket(AF_INET6, SOCK_STREAM, 0)) < 0) { - warn("Couldn't bind socket to %d: %s", port, strerror(errno)); - close(sd); - return -1; + errAbort("socket() failed"); } -listen(sd, queueSize); -return sd; + +// Allow local address reuse when server is restarted without waiting. +int on = -1; +if (setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, (char *)&on,sizeof(on)) < 0) + { + errAbort("setsockopt(SO_REUSEADDR) failed"); } -int netAcceptingSocket(int port, int queueSize) -/* Create a socket that can accept connections from - * anywhere. */ +// Explicitly turn off IPV6_V6ONLY which is needed on non-Linux platforms like NetBSD and Darwin. +// This means we allow ipv4 socket connections that can have ipv4-mapped ipv6 IPs. +int off = 0; +if (setsockopt(sd, IPPROTO_IPV6, IPV6_V6ONLY, (void *)&off, sizeof(off)) < 0) + { + errAbort("setsockopt IPV6_V6ONLY off failed."); + } + +ZeroVar(&serverAddr); +serverAddr.sin6_family = AF_INET6; +serverAddr.sin6_port = htons(port); +serverAddr.sin6_addr = in6addr_any; +if (bind(sd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { -return netAcceptingSocketFrom(port, queueSize, NULL); + errAbort("Couldn't bind socket to %d: %s", port, strerror(errno)); + } + +if (listen(sd, queueSize) < 0) + { + errAbort("listen() failed"); + } + +return sd; } int netAccept(int sd) /* Accept incoming connection from socket descriptor. */ { socklen_t fromLen; return accept(sd, NULL, &fromLen); } int netAcceptFrom(int acceptor, struct cidr *subnet) /* Wait for incoming connection from socket descriptor * from IP address in subnet. Subnet is something - * returned from netParseSubnet or internetParseDottedQuad. + * returned from internetParseSubnetCidr. * Subnet may be NULL. */ { -struct sockaddr_in sai; /* Some system socket info. */ -ZeroVar(&sai); -sai.sin_family = AF_INET; +struct sockaddr_storage theirAddr; +ZeroVar(&theirAddr); for (;;) { - socklen_t addrSize = sizeof(sai); - int sd = accept(acceptor, (struct sockaddr *)&sai, &addrSize); + int sd = accept(acceptor, NULL, NULL); if (sd >= 0) { if (subnet == NULL) return sd; else { - unsigned char unpacked[4]; - internetUnpackIp(ntohl(sai.sin_addr.s_addr), unpacked); - if (internetIpInSubnetCidr(unpacked, subnet)) + struct sockaddr_in6 clientAddr; + unsigned int addrLen=sizeof(clientAddr); + getpeername(sd, (struct sockaddr *)&clientAddr, &addrLen); + if (internetIpInSubnetCidr(&clientAddr.sin6_addr, subnet)) { return sd; } else { close(sd); } } } } } FILE *netFileFromSocket(int socket) /* Wrap a FILE around socket. This should be fclose'd * and separately the socket close'd. */ @@ -393,38 +410,52 @@ char *z=strchr(y, '-'); if (z) { ++z; if (terminateAtByteRange) *x = 0; *rangeStart = atoll(y); if (z[0] != '\0') *rangeEnd = atoll(z); } } } } +void netHandleHostForIpv6(struct netParsedUrl *npu, struct dyString *dy) +/* if needed, add brackets around the host name, + * for raw ipv6 address so it is not confused with colon port that may follow. */ +{ +boolean hostIsIpv6 = FALSE; +if (strchr(npu->host, ':')) // Is the host really an IPV6 address? + hostIsIpv6 = TRUE; +if (hostIsIpv6) // surround ipv6 host with brakets [] + dyStringAppendC(dy, '['); +dyStringAppend(dy, npu->host); +if (hostIsIpv6) // surround ipv6 host with brakets [] + dyStringAppendC(dy, ']'); +} + void netParseUrl(char *url, struct netParsedUrl *parsed) /* Parse a URL into components. A full URL is made up as so: * http://user:password@hostName:port/file;byterange=0-499 * User and password may be cgi-encoded. * This is set up so that the http:// and the port are optional. */ { -char *s, *t, *u, *v, *w; +char *s, *t, *u, *v, *w, *br, *bl; char buf[MAXURLSIZE]; /* Make local copy of URL. */ if (strlen(url) >= sizeof(buf)) errAbort("Url too long: '%s'", url); strcpy(buf, url); url = buf; /* Find out protocol - default to http. */ s = trimSpaces(url); s = stringIn("://", url); if (s == NULL) { strcpy(parsed->protocol, "http"); s = url; @@ -496,75 +527,120 @@ strcpy(parsed->password, ""); } else { *w = 0; safecpy(parsed->user, sizeof(parsed->user), s); safecpy(parsed->password, sizeof(parsed->password), w+1); } cgiDecode(parsed->user,parsed->user,strlen(parsed->user)); cgiDecode(parsed->password,parsed->password,strlen(parsed->password)); s = v+1; } +// Whenever IPv6 address : port are provided, +// the address MUST be surrounded by square brackets like [IPv6-address]:port +// because without the square brackets, we cannot tell if the trailing bit +// is end end of an IPv6 address, or port number. + +int blCount = countChars(s, '['); +int brCount = countChars(s, ']'); + +// double-check any stray brackets +if ((brCount != blCount) || (brCount > 1)) + errAbort("badly formed url, stray square brackets in IPv6 address"); + /* Save port if it's there. If not default to 80. */ -t = strchr(s, ':'); -if (t == NULL) - { - if (sameWord(parsed->protocol,"http")) - strcpy(parsed->port, "80"); - if (sameWord(parsed->protocol,"https")) - strcpy(parsed->port, "443"); - if (sameWord(parsed->protocol,"ftp")) - strcpy(parsed->port, "21"); +bl = strchr(s, '['); // IPV6 address in url surrounded by brackets [] +br = strrchr(s, ']'); // IPV6 address in url surrounded by brackets [] + +if (!br != !bl) // logical XOR + errAbort("badly formed url, unbalanced square brackets around IPv6 address."); + +if (!br && isIpv6Address(s)) // host looks like IPv6 address but no brackets. + errAbort("badly formed url, should be protocol://[IPv6-address]:port/. Put square brackets around literal IPv6 address."); + +// trim off the brackets around the ipv6 host name +if (br) + { + // expecting *s == [ + if (*s != '[') + errAbort("badly formed url %s, expected [ at start of ipv6 address", s); + ++s; // skip [ + *br = 0; // erase ] + t = br+1; + char c = *t; + if (c == 0) + t = NULL; + else if (c != ':') + errAbort("badly formed url %s, stray characters after ] at end of ipv6 address", s); } else { + t = strrchr(s, ':'); + } + +if (br && !isIpv6Address(s)) // host has brackets but does not look like IPv6 address. + errAbort("badly formed url, brackets found, but not valid literal IPv6 address."); + +if (t) // the port was explicitly provided + { *t++ = 0; if (!isdigit(t[0])) errAbort("Non-numeric port name %s", t); safecpy(parsed->port, sizeof(parsed->port), t); } +else // get default port for each protocol + { + if (sameWord(parsed->protocol,"http")) + strcpy(parsed->port, "80"); + if (sameWord(parsed->protocol,"https")) + strcpy(parsed->port, "443"); + if (sameWord(parsed->protocol,"ftp")) + strcpy(parsed->port, "21"); + } /* What's left is the host. */ safecpy(parsed->host, sizeof(parsed->host), s); } char *urlFromNetParsedUrl(struct netParsedUrl *npu) /* Build URL from netParsedUrl structure */ { struct dyString *dy = newDyString(512); dyStringAppend(dy, npu->protocol); dyStringAppend(dy, "://"); if (npu->user[0] != 0) { char *encUser = cgiEncode(npu->user); dyStringAppend(dy, encUser); freeMem(encUser); if (npu->password[0] != 0) { dyStringAppend(dy, ":"); char *encPassword = cgiEncode(npu->password); dyStringAppend(dy, encPassword); freeMem(encPassword); } dyStringAppend(dy, "@"); } -dyStringAppend(dy, npu->host); + +netHandleHostForIpv6(npu, dy); + /* do not include port if it is the default */ if (!( (sameString(npu->protocol, "ftp" ) && sameString("21", npu->port)) || (sameString(npu->protocol, "http" ) && sameString("80", npu->port)) || (sameString(npu->protocol, "https") && sameString("443",npu->port)) )) { dyStringAppend(dy, ":"); dyStringAppend(dy, npu->port); } dyStringAppend(dy, npu->file); if (npu->byteRangeStart != -1) { dyStringPrintf(dy, ";byterange=%lld-", (long long)npu->byteRangeStart); if (npu->byteRangeEnd != -1) @@ -1144,36 +1220,47 @@ return -1; /* Ask remote server for a file. */ char *urlForProxy = NULL; if (proxyUrl) { /* trim off the byterange part at the end of url because proxy does not understand it. */ urlForProxy = cloneString(url); char *x = strrchr(urlForProxy, ';'); if (x && startsWith(";byterange=", x)) *x = 0; } dyStringPrintf(dy, "%s %s %s\r\n", method, proxyUrl ? urlForProxy : npu.file, protocol); freeMem(urlForProxy); dyStringPrintf(dy, "User-Agent: %s\r\n", agent); + +dyStringPrintf(dy, "Host: "); +netHandleHostForIpv6(&npu, dy); + +boolean portIsDefault = FALSE; /* do not need the 80 since it is the default */ -if ((sameString(npu.protocol, "http" ) && sameString("80", npu.port)) || - (sameString(npu.protocol, "https") && sameString("443",npu.port))) - dyStringPrintf(dy, "Host: %s\r\n", npu.host); -else - dyStringPrintf(dy, "Host: %s:%s\r\n", npu.host, npu.port); +if (sameString(npu.protocol, "http" ) && sameString("80", npu.port)) + portIsDefault = TRUE; +if (sameString(npu.protocol, "https" ) && sameString("443", npu.port)) + portIsDefault = TRUE; +if (!portIsDefault) + { + dyStringAppendC(dy, ':'); + dyStringAppend(dy, npu.port); + } +dyStringPrintf(dy, "\r\n"); + setAuthorization(npu, "Authorization", dy); if (proxyUrl) setAuthorization(pxy, "Proxy-Authorization", dy); dyStringAppend(dy, "Accept: */*\r\n"); if (npu.byteRangeStart != -1) { if (npu.byteRangeEnd != -1) dyStringPrintf(dy, "Range: bytes=%lld-%lld\r\n" , (long long)npu.byteRangeStart , (long long)npu.byteRangeEnd); else dyStringPrintf(dy, "Range: bytes=%lld-\r\n" , (long long)npu.byteRangeStart); } @@ -1950,31 +2037,37 @@ /* Return handle. */ lf = lineFileAttach(url, TRUE, sd); return lf; } /* netHttpLineFileMayOpen */ void netHttpGet(struct lineFile *lf, struct netParsedUrl *npu, boolean keepAlive) /* Send a GET request, possibly with Keep-Alive. */ { struct dyString *dy = newDyString(512); /* Ask remote server for the file/query. */ dyStringPrintf(dy, "GET %s HTTP/1.1\r\n", npu->file); dyStringPrintf(dy, "User-Agent: genome.ucsc.edu/net.c\r\n"); -dyStringPrintf(dy, "Host: %s:%s\r\n", npu->host, npu->port); + +dyStringPrintf(dy, "Host: "); +netHandleHostForIpv6(npu, dy); +dyStringAppendC(dy, ':'); +dyStringAppend(dy, npu->port); +dyStringPrintf(dy, "\r\n"); + if (!sameString(npu->user,"")) { char up[256]; char *b64up = NULL; safef(up,sizeof(up), "%s:%s", npu->user, npu->password); b64up = base64Encode(up, strlen(up)); dyStringPrintf(dy, "Authorization: Basic %s\r\n", b64up); freez(&b64up); } dyStringAppend(dy, "Accept: */*\r\n"); if (keepAlive) { dyStringAppend(dy, "Connection: Keep-Alive\r\n"); dyStringAppend(dy, "Connection: Persist\r\n"); }