08b1be37724c83f2dab17f71b5494963f34180b7 braney Fri Apr 24 08:16:21 2020 -0700 expand the size of a buffer in net.c to 8k instead of 2000 diff --git src/lib/net.c src/lib/net.c index 4f8a5a5..9766bd0 100644 --- src/lib/net.c +++ src/lib/net.c @@ -1,2282 +1,2282 @@ /* net.c some stuff to wrap around net communications. * * This file is copyright 2002 Jim Kent, but license is hereby * granted for all use - public, private or commercial. */ #include "common.h" #include <signal.h> #include <errno.h> #include <string.h> #include <sys/time.h> #include <pthread.h> #include "errAbort.h" #include "hash.h" #include "net.h" #include "linefile.h" #include "base64.h" #include "cheapcgi.h" #include "https.h" #include "sqlNum.h" #include "obscure.h" #include "errCatch.h" /* Brought errno in to get more useful error messages */ extern int errno; static int netStreamSocketFromAddrInfo(struct addrinfo *address) /* Create a socket from addrinfo structure. * Complain and return something negative if can't. */ { int sd = socket(address->ai_family, address->ai_socktype, address->ai_protocol); if (sd < 0) warn("Couldn't make %s socket.", familyToString(address->ai_family)); return sd; } static int setSocketNonBlocking(int sd, boolean set) /* Use socket control flags to set O_NONBLOCK if set==TRUE, * or clear it if set==FALSE. * Return -1 if there are any errors, 0 if successful. */ { long fcntlFlags; // Set or clear non-blocking if ((fcntlFlags = fcntl(sd, F_GETFL, NULL)) < 0) { warn("Error fcntl(..., F_GETFL) (%s)", strerror(errno)); return -1; } if (set) fcntlFlags |= O_NONBLOCK; else fcntlFlags &= (~O_NONBLOCK); if (fcntl(sd, F_SETFL, fcntlFlags) < 0) { warn("Error fcntl(..., F_SETFL) (%s)", strerror(errno)); return -1; } return 0; } int setReadWriteTimeouts(int sd, int seconds) /* Set read and write timeouts on socket sd * Return -1 if there are any errors, 0 if successful. */ { struct timeval timeout; timeout.tv_sec = seconds; timeout.tv_usec = 0; if (setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(timeout)) < 0) { warn("setsockopt failed setting socket receive timeout\n"); return -1; } if (setsockopt(sd, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(timeout)) < 0) { warn("setsockopt failed setting socket send timeout\n"); return -1; } return 0; } static struct timeval tvMinus(struct timeval a, struct timeval b) /* Return the result of a - b; this handles wrapping of milliseconds. * result.tv_usec will always be positive. * result.tv_sec will be negative if b > a. */ { // subtract b from a. if (a.tv_usec < b.tv_usec) { a.tv_usec += 1000000; a.tv_sec--; } a.tv_usec -= b.tv_usec; a.tv_sec -= b.tv_sec; return a; } int netConnectWithTimeoutOneAddr(int sd, struct addrinfo *address, long msTimeout, char *hostName, int port, struct dyString *dy) /* Try to connect one address with timeout or return success == 0, failure == -1 */ { // Set socket to nonblocking so we can manage our own timeout time. if (setSocketNonBlocking(sd, TRUE) < 0) { return -1; } // Trying to connect with timeout int res; res = connect(sd, address->ai_addr, address->ai_addrlen); char ipStr[NI_MAXHOST]; getAddrAsString6n4((struct sockaddr_storage *)address->ai_addr, ipStr, sizeof ipStr); if (res < 0) { if (errno == EINPROGRESS) { struct timeval startTime; gettimeofday(&startTime, NULL); struct timeval remainingTime; remainingTime.tv_sec = (long) (msTimeout/1000); remainingTime.tv_usec = (long) (((msTimeout/1000)-remainingTime.tv_sec)*1000000); while (1) { fd_set mySet; FD_ZERO(&mySet); FD_SET(sd, &mySet); // use tempTime (instead of using remainingTime directly) because on some platforms select() may modify the time val. struct timeval tempTime = remainingTime; res = select(sd+1, NULL, &mySet, &mySet, &tempTime); if (res < 0) { if (errno == EINTR) // Ignore the interrupt { // Subtract the elapsed time from remaining time since some platforms need this. struct timeval newTime; gettimeofday(&newTime, NULL); struct timeval elapsedTime = tvMinus(newTime, startTime); remainingTime = tvMinus(remainingTime, elapsedTime); if (remainingTime.tv_sec < 0) // means our timeout has more than expired { remainingTime.tv_sec = 0; remainingTime.tv_usec = 0; } startTime = newTime; } else { dyStringPrintf(dy, "Error in select() during TCP non-blocking connect %d - %s\n", errno, strerror(errno)); return -1; } } else if (res > 0) { // Socket selected for write when it is ready int valOpt; socklen_t lon; // But check the socket for any errors lon = sizeof(valOpt); if (getsockopt(sd, SOL_SOCKET, SO_ERROR, (void*) (&valOpt), &lon) < 0) { warn("Error in getsockopt() %d - %s", errno, strerror(errno)); return -1; } // Check the value returned... if (valOpt) { dyStringPrintf(dy, "Error in TCP non-blocking connect() %d - %s. Host %s IP %s port %d.\n", valOpt, strerror(valOpt), hostName, ipStr, port); return -1; } break; // OK } else { dyStringPrintf(dy, "TCP non-blocking connect() to %s IP %s timed-out in select() after %ld milliseconds - Cancelling!", hostName, ipStr, msTimeout); return -1; } } } else { dyStringPrintf(dy, "TCP non-blocking connect() error %d - %s", errno, strerror(errno)); return -1; } } return 0; // OK } static int netConnectWithTimeout(char *hostName, int port, long msTimeout) /* In order to avoid a very long default timeout (several minutes) for hosts that will * not answer the port, we are forced to connect non-blocking. * After the connection has been established, we return to blocking mode. * Also closes sd if error. */ { int sd; struct addrinfo *addressList=NULL, *address; char portStr[8]; safef(portStr, sizeof portStr, "%d", port); if (hostName == NULL) { warn("NULL hostName in netConnect"); return -1; } if (!internetGetAddrInfo6n4(hostName, portStr, &addressList)) return -1; struct dyString *errMsg = newDyString(256); for (address = addressList; address; address = address->ai_next) { if ((sd = netStreamSocketFromAddrInfo(address)) < 0) continue; if (netConnectWithTimeoutOneAddr(sd, address, msTimeout, hostName, port, errMsg) == 0) break; close(sd); } boolean connected = (address != NULL); // one of the addresses connected successfully freeaddrinfo(addressList); if (!connected) { if (!sameString(errMsg->string, "")) { warn("%s", errMsg->string); } } dyStringFree(&errMsg); if (!connected) return -1; // Set to blocking mode again if (setSocketNonBlocking(sd, FALSE) < 0) { close(sd); return -1; } if (setReadWriteTimeouts(sd, DEFAULTREADWRITETTIMEOUTSEC) < 0) { close(sd); return -1; } return sd; } int netConnect(char *hostName, int port) /* Start connection with a server. */ { return netConnectWithTimeout(hostName, port, DEFAULTCONNECTTIMEOUTMSEC); // 10 seconds connect timeout } int netMustConnect(char *hostName, int port) /* Start connection with server or die. */ { int sd = netConnect(hostName, port); if (sd < 0) noWarnAbort(); return sd; } int netMustConnectTo(char *hostName, char *portName) /* Start connection with a server and a port that needs to be converted to integer */ { if (!isdigit(portName[0])) errAbort("netConnectTo: ports must be numerical, not %s", portName); return netMustConnect(hostName, atoi(portName)); } int netAcceptingSocket4Only(int port, int queueSize) /* Create an IPV4 socket that can accept connections from * only IPV4 clients on the current machine. Useful for systems with ipv6 disabled. */ { struct sockaddr_in serverAddr; int sd; netBlockBrokenPipes(); // ipv4 listening socket accepts ipv4 connections. if ((sd = socket(AF_INET, SOCK_STREAM, 0)) < 0) { errAbort("socket() failed"); } // Allow local address reuse when server is restarted without waiting. int on = -1; if (setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, (char *)&on,sizeof(on)) < 0) { errAbort("setsockopt(SO_REUSEADDR) failed"); } ZeroVar(&serverAddr); serverAddr.sin_family = AF_INET; serverAddr.sin_port = htons(port); serverAddr.sin_addr.s_addr = INADDR_ANY; if (bind(sd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { errAbort("Couldn't bind socket to %d: %s", port, strerror(errno)); } if (listen(sd, queueSize) < 0) { errAbort("listen() failed"); } return sd; } int netAcceptingSocket6n4(int port, int queueSize) /* Create an IPV6 socket that can accept connections from * both IPV4 and IPV6 clients on the current machine. */ { struct sockaddr_in6 serverAddr; int sd; netBlockBrokenPipes(); // Hybrid dual stack ipv6 listening socket accepts ipv6 and ipv4 mapped connections. if ((sd = socket(AF_INET6, SOCK_STREAM, 0)) < 0) { errAbort("socket() failed"); } // Allow local address reuse when server is restarted without waiting. int on = -1; if (setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, (char *)&on,sizeof(on)) < 0) { errAbort("setsockopt(SO_REUSEADDR) failed"); } // Explicitly turn off IPV6_V6ONLY which is needed on non-Linux platforms like NetBSD and Darwin. // This means we allow ipv4 socket connections that can have ipv4-mapped ipv6 IPs. int off = 0; if (setsockopt(sd, IPPROTO_IPV6, IPV6_V6ONLY, (void *)&off, sizeof(off)) < 0) { errAbort("setsockopt IPV6_V6ONLY off failed."); } ZeroVar(&serverAddr); serverAddr.sin6_family = AF_INET6; serverAddr.sin6_port = htons(port); serverAddr.sin6_addr = in6addr_any; if (bind(sd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { errAbort("Couldn't bind socket to %d: %s", port, strerror(errno)); } if (listen(sd, queueSize) < 0) { errAbort("listen() failed"); } return sd; } int netAcceptingSocket(int port, int queueSize) /* Create an IPV6 socket that can accept connections from * both IPV4 and IPV6 clients on the current machine. * OR Failover to making IPv4 Only socket if IPv6 is disabled. */ { int sd = -1; struct errCatch *errCatch = errCatchNew(); if (errCatchStart(errCatch)) { sd = netAcceptingSocket6n4(port, queueSize); } errCatchEnd(errCatch); if (errCatch->gotError) { // if ipv6 is disabled, fall back to trying ipv4 only listen socket warn("%s", errCatch->message->string); warn("Retrying listen socket using ipv4 only."); sd = netAcceptingSocket4Only(port, queueSize); } errCatchFree(&errCatch); if (sd == -1) errAbort("unable to open listening socket"); return sd; } int netAccept(int sd) /* Accept incoming connection from socket descriptor. */ { socklen_t fromLen; return accept(sd, NULL, &fromLen); } int netAcceptFrom(int acceptor, struct cidr *subnet) /* Wait for incoming connection from socket descriptor * from IP address in subnet. Subnet is something * returned from internetParseSubnetCidr. * Subnet may be NULL. */ { for (;;) { int sd = accept(acceptor, NULL, NULL); if (sd >= 0) { if (subnet == NULL) return sd; else { socklen_t len; struct sockaddr_storage addr; char ipStr[INET6_ADDRSTRLEN]; len = sizeof addr; getpeername(sd, (struct sockaddr*)&addr, &len); getAddrAsString6n4(&addr, ipStr, sizeof ipStr); if (!strchr(ipStr, ':')) // convert ipv4 to ipv6-mapped { // prepend "::ffff:" to ipStr char temp[INET6_ADDRSTRLEN]; safef(temp, sizeof temp, "::ffff:%s", ipStr); safecpy(ipStr, sizeof ipStr, temp); } // convert back to ipv6 address. struct sockaddr_in6 clientAddr; internetIpStringToIp6(ipStr, &clientAddr.sin6_addr); // see if it is in the allowed subnet if (internetIpInSubnetCidr(&clientAddr.sin6_addr, subnet)) { return sd; } else { close(sd); } } } } } FILE *netFileFromSocket(int socket) /* Wrap a FILE around socket. This should be fclose'd * and separately the socket close'd. */ { FILE *f; if ((socket = dup(socket)) < 0) errnoAbort("Couldn't dupe socket in netFileFromSocket"); f = fdopen(socket, "r+"); if (f == NULL) errnoAbort("Couldn't fdopen socket in netFileFromSocket"); return f; } static boolean plumberInstalled = FALSE; void netBlockBrokenPipes() /* Make it so a broken pipe doesn't kill us. */ { if (!plumberInstalled) { signal(SIGPIPE, SIG_IGN); /* Block broken pipe signals. */ plumberInstalled = TRUE; } } ssize_t netReadAll(int sd, void *vBuf, ssize_t size) /* Read given number of bytes into buffer. * Don't give up on first read! */ { char *buf = vBuf; ssize_t totalRead = 0; int oneRead; if (!plumberInstalled) netBlockBrokenPipes(); while (totalRead < size) { oneRead = read(sd, buf + totalRead, size - totalRead); if (oneRead < 0) return oneRead; if (oneRead == 0) break; totalRead += oneRead; } return totalRead; } ssize_t netMustReadAll(int sd, void *vBuf, ssize_t size) /* Read given number of bytes into buffer or die. * Don't give up if first read is short! */ { ssize_t ret = netReadAll(sd, vBuf, size); if (ret < 0) errnoAbort("Couldn't finish netReadAll"); return ret; } static void parseByteRange(char *url, ssize_t *rangeStart, ssize_t *rangeEnd, boolean terminateAtByteRange) /* parse the byte range information from url */ { char *x; /* default to no byte range specified */ *rangeStart = -1; *rangeEnd = -1; x = strrchr(url, ';'); if (x) { if (startsWith(";byterange=", x)) { char *y=strchr(x, '='); ++y; char *z=strchr(y, '-'); if (z) { ++z; if (terminateAtByteRange) *x = 0; *rangeStart = atoll(y); if (z[0] != '\0') *rangeEnd = atoll(z); } } } } void netHandleHostForIpv6(struct netParsedUrl *npu, struct dyString *dy) /* if needed, add brackets around the host name, * for raw ipv6 address so it is not confused with colon port that may follow. */ { boolean hostIsIpv6 = FALSE; if (strchr(npu->host, ':')) // Is the host really an IPV6 address? hostIsIpv6 = TRUE; if (hostIsIpv6) // surround ipv6 host with brakets [] dyStringAppendC(dy, '['); dyStringAppend(dy, npu->host); if (hostIsIpv6) // surround ipv6 host with brakets [] dyStringAppendC(dy, ']'); } void netParseUrl(char *url, struct netParsedUrl *parsed) /* Parse a URL into components. A full URL is made up as so: * http://user:password@hostName:port/file;byterange=0-499 * User and password may be cgi-encoded. * This is set up so that the http:// and the port are optional. */ { char *s, *t, *u, *v, *w, *br, *bl; char buf[MAXURLSIZE]; /* Make local copy of URL. */ if (strlen(url) >= sizeof(buf)) errAbort("Url too long: '%s'", url); strcpy(buf, url); url = buf; /* Find out protocol - default to http. */ s = trimSpaces(url); s = stringIn("://", url); if (s == NULL) { strcpy(parsed->protocol, "http"); s = url; } else { *s = 0; tolowers(url); safecpy(parsed->protocol, sizeof(parsed->protocol), url); s += 3; } /* Split off file part. */ parsed->byteRangeStart = -1; /* default to no byte range specified */ parsed->byteRangeEnd = -1; u = strchr(s, '/'); if (u == NULL) strcpy(parsed->file, "/"); else { parseByteRange(u, &parsed->byteRangeStart, &parsed->byteRangeEnd, TRUE); if (sameWord(parsed->protocol,"http") || sameWord(parsed->protocol,"https")) { // http servers expect the URL request to be URL-encoded already. /* need to encode spaces, but not ! other characters */ char *t=replaceChars(u," ","%20"); safecpy(parsed->file, sizeof(parsed->file), t); freeMem(t); } *u = 0; // terminate the host:port string if (sameWord(parsed->protocol,"ftp")) { ++u; // that first slash is not considered part of the ftp path // decode now because the FTP server does NOT expect URL-encoding. cgiDecodeFull(u,parsed->file,strlen(u)); // decodes %FF but not + } } /* Split off user part */ v = strchr(s, '@'); if (v == NULL) { if (sameWord(parsed->protocol,"http") || sameWord(parsed->protocol,"https")) { strcpy(parsed->user, ""); strcpy(parsed->password, ""); } if (sameWord(parsed->protocol,"ftp")) { strcpy(parsed->user, "anonymous"); strcpy(parsed->password, "x@genome.ucsc.edu"); } } else { *v = 0; /* split off password part */ w = strchr(s, ':'); if (w == NULL) { safecpy(parsed->user, sizeof(parsed->user), s); strcpy(parsed->password, ""); } else { *w = 0; safecpy(parsed->user, sizeof(parsed->user), s); safecpy(parsed->password, sizeof(parsed->password), w+1); } cgiDecode(parsed->user,parsed->user,strlen(parsed->user)); cgiDecode(parsed->password,parsed->password,strlen(parsed->password)); s = v+1; } // Whenever IPv6 address : port are provided, // the address MUST be surrounded by square brackets like [IPv6-address]:port // because without the square brackets, we cannot tell if the trailing bit // is end end of an IPv6 address, or port number. int blCount = countChars(s, '['); int brCount = countChars(s, ']'); // double-check any stray brackets if ((brCount != blCount) || (brCount > 1)) errAbort("badly formed url, stray square brackets in IPv6 address"); /* Save port if it's there. If not default to 80. */ bl = strchr(s, '['); // IPV6 address in url surrounded by brackets [] br = strrchr(s, ']'); // IPV6 address in url surrounded by brackets [] if (!br != !bl) // logical XOR errAbort("badly formed url, unbalanced square brackets around IPv6 address."); if (!br && isIpv6Address(s)) // host looks like IPv6 address but no brackets. errAbort("badly formed url, should be protocol://[IPv6-address]:port/. Put square brackets around literal IPv6 address."); // trim off the brackets around the ipv6 host name if (br) { // expecting *s == [ if (*s != '[') errAbort("badly formed url %s, expected [ at start of ipv6 address", s); ++s; // skip [ *br = 0; // erase ] t = br+1; char c = *t; if (c == 0) t = NULL; else if (c != ':') errAbort("badly formed url %s, stray characters after ] at end of ipv6 address", s); } else { t = strrchr(s, ':'); } if (br && !isIpv6Address(s)) // host has brackets but does not look like IPv6 address. errAbort("badly formed url, brackets found, but not valid literal IPv6 address."); if (t) // the port was explicitly provided { *t++ = 0; if (!isdigit(t[0])) errAbort("Non-numeric port name %s", t); safecpy(parsed->port, sizeof(parsed->port), t); } else // get default port for each protocol { if (sameWord(parsed->protocol,"http")) strcpy(parsed->port, "80"); if (sameWord(parsed->protocol,"https")) strcpy(parsed->port, "443"); if (sameWord(parsed->protocol,"ftp")) strcpy(parsed->port, "21"); } /* What's left is the host. */ safecpy(parsed->host, sizeof(parsed->host), s); } char *urlFromNetParsedUrl(struct netParsedUrl *npu) /* Build URL from netParsedUrl structure */ { struct dyString *dy = newDyString(512); dyStringAppend(dy, npu->protocol); dyStringAppend(dy, "://"); if (npu->user[0] != 0) { char *encUser = cgiEncode(npu->user); dyStringAppend(dy, encUser); freeMem(encUser); if (npu->password[0] != 0) { dyStringAppend(dy, ":"); char *encPassword = cgiEncode(npu->password); dyStringAppend(dy, encPassword); freeMem(encPassword); } dyStringAppend(dy, "@"); } netHandleHostForIpv6(npu, dy); /* do not include port if it is the default */ if (!( (sameString(npu->protocol, "ftp" ) && sameString("21", npu->port)) || (sameString(npu->protocol, "http" ) && sameString("80", npu->port)) || (sameString(npu->protocol, "https") && sameString("443",npu->port)) )) { dyStringAppend(dy, ":"); dyStringAppend(dy, npu->port); } dyStringAppend(dy, npu->file); if (npu->byteRangeStart != -1) { dyStringPrintf(dy, ";byterange=%lld-", (long long)npu->byteRangeStart); if (npu->byteRangeEnd != -1) dyStringPrintf(dy, "%lld", (long long)npu->byteRangeEnd); } /* Clean up and return handle. */ return dyStringCannibalize(&dy); } int netWaitForData(int sd, int microseconds) /* Wait for descriptor to have some data to read, up to given number of * microseconds. Returns amount of data there or zero if timed out. */ { struct timeval tv; fd_set set; int readyCount; for (;;) { if (microseconds >= 1000000) { tv.tv_sec = microseconds/1000000; tv.tv_usec = microseconds%1000000; } else { tv.tv_sec = 0; tv.tv_usec = microseconds; } FD_ZERO(&set); FD_SET(sd, &set); readyCount = select(sd+1, &set, NULL, NULL, &tv); if (readyCount < 0) { if (errno == EINTR) /* Select interrupted, not timed out. */ continue; else warn("select failure %s", strerror(errno)); } else { return readyCount; /* Zero readyCount indicates time out */ } } } static boolean readReadyWait(int sd, int microseconds) /* Wait for descriptor to have some data to read, up to given number of * number of microseconds. Returns true if there is data, false if timed out. */ { int readyCount = netWaitForData(sd, microseconds); return readyCount > 0; /* Zero readyCount indicates time out */ } static void sendFtpCommandOnly(int sd, char *cmd) /* send command to ftp server */ { mustWriteFd(sd, cmd, strlen(cmd)); } #define NET_FTP_TIMEOUT 1000000 static boolean receiveFtpReply(int sd, char *cmd, struct dyString **retReply, int *retCode) /* send command to ftp server and check resulting reply code, * warn and return FALSE if not desired reply. If retReply is non-NULL, store reply text there. */ { char *startLastLine = NULL; struct dyString *rs = newDyString(4*1024); while (1) { int readSize = 0; while (1) { char buf[4*1024]; if (!readReadyWait(sd, NET_FTP_TIMEOUT)) { warn("ftp server response timed out > %d microsec", NET_FTP_TIMEOUT); return FALSE; } if ((readSize = read(sd, buf, sizeof(buf))) == 0) break; dyStringAppendN(rs, buf, readSize); if (endsWith(rs->string,"\n")) break; } /* find the start of the last line in the buffer */ startLastLine = rs->string+strlen(rs->string)-1; if (startLastLine >= rs->string) if (*startLastLine == '\n') --startLastLine; while ((startLastLine >= rs->string) && (*startLastLine != '\n')) --startLastLine; ++startLastLine; if (strlen(startLastLine)>4) if ( isdigit(startLastLine[0]) && isdigit(startLastLine[1]) && isdigit(startLastLine[2]) && startLastLine[3]==' ') break; if (readSize == 0) break; // EOF /* must be some text info we can't use, ignore it till we get status code */ } int reply = atoi(startLastLine); if ((reply < 200) || (reply > 399)) { warn("ftp server error on cmd=[%s] response=[%s]",cmd,rs->string); return FALSE; } if (retReply) *retReply = rs; else dyStringFree(&rs); if (retCode) *retCode = reply; return TRUE; } static boolean sendFtpCommand(int sd, char *cmd, struct dyString **retReply, int *retCode) /* send command to ftp server and check resulting reply code, * warn and return FALSE if not desired reply. If retReply is non-NULL, store reply text there. */ { sendFtpCommandOnly(sd, cmd); return receiveFtpReply(sd, cmd, retReply, retCode); } static int parsePasvPort(char *rs) /* parse PASV reply to get the port and return it */ { char *words[7]; int wordCount; char *rsStart = strchr(rs,'('); char *rsEnd = strchr(rs,')'); int result = 0; rsStart++; *rsEnd=0; wordCount = chopString(rsStart, ",", words, ArraySize(words)); if (wordCount != 6) errAbort("PASV reply does not parse correctly"); result = atoi(words[4])*256+atoi(words[5]); return result; } static long long parseFtpSIZE(char *rs) /* parse reply to SIZE and return it */ { char *words[3]; int wordCount; char *rsStart = rs; long long result = 0; wordCount = chopString(rsStart, " ", words, ArraySize(words)); if (wordCount != 2) errAbort("SIZE reply does not parse correctly"); result = atoll(words[1]); return result; } static time_t parseFtpMDTM(char *rs) /* parse reply to MDTM and return it * 200 YYYYMMDDhhmmss */ { char *words[3]; int wordCount = chopString(rs, " ", words, ArraySize(words)); if (wordCount != 2) errAbort("MDTM reply should have 2 words but has %d", wordCount); #define EXPECTED_MDTM_FORMAT "YYYYmmddHHMMSS" if (strlen(words[1]) < strlen(EXPECTED_MDTM_FORMAT)) errAbort("MDTM reply word [%s] shorter than " EXPECTED_MDTM_FORMAT, words[1]); // man strptime -> // "There should be whitespace or other alphanumeric characters between any two field descriptors." // There are no separators in the MDTM timestamp, so make a spread-out version for strptime: char spread[] = "YYYY mm dd HH MM SS"; char *from = words[1]; char *to = spread; *to++ = *from++; *to++ = *from++; *to++ = *from++; *to++ = *from++; *to++ = '-'; *to++ = *from++; *to++ = *from++; *to++ = '-'; *to++ = *from++; *to++ = *from++; *to++ = ' '; *to++ = *from++; *to++ = *from++; *to++ = ':'; *to++ = *from++; *to++ = *from++; *to++ = ':'; *to++ = *from++; *to++ = *from++; *to++ = 0; struct tm tm; if (strptime(spread, "%Y-%m-%d %H:%M:%S", &tm) == NULL) errAbort("unable to parse MDTM string [%s]", spread); // Not set by strptime(); tells mktime() to determine whether daylight saving time is in effect: tm.tm_isdst = -1; time_t t = mktime(&tm); if (t == -1) errAbort("mktime failed while parsing strptime'd MDTM string [%s]", words[1]); return t; } static int openFtpControlSocket(char *host, int port, char *user, char *password) /* Open a socket to host,port; authenticate anonymous ftp; set type to I; * return socket desc or -1 if there was an error. */ { int sd = netConnect(host, port); if (sd < 0) return -1; /* First read the welcome msg */ if (readReadyWait(sd, NET_FTP_TIMEOUT)) sendFtpCommand(sd, "", NULL, NULL); char cmd[256]; int retCode = 0; safef(cmd,sizeof(cmd),"USER %s\r\n", user); if (!sendFtpCommand(sd, cmd, NULL, &retCode)) { close(sd); return -1; } if (retCode == 331) { safef(cmd,sizeof(cmd),"PASS %s\r\n", password); if (!sendFtpCommand(sd, cmd, NULL, NULL)) { close(sd); return -1; } } if (!sendFtpCommand(sd, "TYPE I\r\n", NULL, NULL)) { close(sd); return -1; } /* 200 Type set to I */ /* (send the data as binary, so can support compressed files) */ return sd; } boolean netGetFtpInfo(char *url, long long *retSize, time_t *retTime) /* Return date in UTC and size of ftp url file */ { /* Parse the URL and connect. */ struct netParsedUrl npu; netParseUrl(url, &npu); if (!sameString(npu.protocol, "ftp")) errAbort("netGetFtpInfo: url (%s) is not for ftp.", url); if (sameString(npu.file,"/")) { *retSize = 0; *retTime = time(NULL); return TRUE; } int sd = openFtpControlSocket(npu.host, atoi(npu.port), npu.user, npu.password); if (sd < 0) return FALSE; char cmd[256]; safef(cmd,sizeof(cmd),"SIZE %s\r\n", npu.file); struct dyString *rs = NULL; if (!sendFtpCommand(sd, cmd, &rs, NULL)) { close(sd); return FALSE; } *retSize = parseFtpSIZE(rs->string); /* 200 12345 */ dyStringFree(&rs); safef(cmd,sizeof(cmd),"MDTM %s\r\n", npu.file); if (!sendFtpCommand(sd, cmd, &rs, NULL)) { close(sd); return FALSE; } *retTime = parseFtpMDTM(rs->string); /* 200 YYYYMMDDhhmmss */ dyStringFree(&rs); close(sd); return TRUE; } struct netConnectFtpParams /* params to pass to thread */ { pthread_t thread; int pipefd[2]; int sd; int sdata; struct netParsedUrl npu; }; static void *sendFtpDataToPipeThread(void *threadParams) /* This is to be executed by the child process after the fork in netGetOpenFtpSockets. * It keeps the ftp control socket alive while reading from the ftp data socket * and writing to the pipe to the parent process, which closes the ftp sockets * and reads from the pipe. */ { struct netConnectFtpParams *params = threadParams; pthread_detach(params->thread); // this thread will never join back with it's progenitor char buf[32768]; int rd = 0; long long dataPos = 0; if (params->npu.byteRangeStart != -1) dataPos = params->npu.byteRangeStart; while((rd = read(params->sdata, buf, 32768)) > 0) { if (params->npu.byteRangeEnd != -1 && (dataPos + rd) > params->npu.byteRangeEnd) rd = params->npu.byteRangeEnd - dataPos + 1; int wt = write(params->pipefd[1], buf, rd); if (wt == -1 && params->npu.byteRangeEnd != -1) { // errAbort in child process is messy; let reader complain if // trouble. If byterange was open-ended, we will hit this point // when the parent stops reading and closes the pipe. errnoWarn("error writing ftp data to pipe"); break; } dataPos += rd; if (params->npu.byteRangeEnd != -1 && dataPos >= params->npu.byteRangeEnd) break; } if (rd == -1) // Again, avoid abort in child process. errnoWarn("error reading ftp socket"); close(params->pipefd[1]); /* we are done with it */ close(params->sd); close(params->sdata); return NULL; } static int netGetOpenFtpSockets(char *url, int *retCtrlSd) /* Return a socket descriptor for url data (url can end in ";byterange:start-end)", * or -1 if error. * If retCtrlSd is non-null, keep the control socket alive and set *retCtrlSd. * Otherwise, create a pipe and fork to keep control socket alive in the child * process until we are done fetching data. */ { char cmd[256]; /* Parse the URL and connect. */ struct netParsedUrl npu; struct netParsedUrl pxy; netParseUrl(url, &npu); if (!sameString(npu.protocol, "ftp")) errAbort("netGetOpenFtpSockets: url (%s) is not for ftp.", url); boolean noProxy = checkNoProxy(npu.host); char *proxyUrl = getenv("ftp_proxy"); if (noProxy) proxyUrl = NULL; int sd = -1; if (proxyUrl) { netParseUrl(proxyUrl, &pxy); if (!sameString(pxy.protocol, "ftp")) errAbort("Unknown proxy protocol %s in %s. Should be ftp.", pxy.protocol, proxyUrl); char proxyUser[4096]; safef(proxyUser, sizeof proxyUser, "%s@%s:%s", npu.user, npu.host, npu.port); sd = openFtpControlSocket(pxy.host, atoi(pxy.port), proxyUser, npu.password); char *logProxy = getenv("log_proxy"); if (sameOk(logProxy,"on")) verbose(1, "%s as %s via proxy %s\n", url, proxyUser, proxyUrl); } else { sd = openFtpControlSocket(npu.host, atoi(npu.port), npu.user, npu.password); } if (sd == -1) return -1; struct dyString *rs = NULL; if (!sendFtpCommand(sd, "PASV\r\n", &rs, NULL)) { close(sd); return -1; } /* 227 Entering Passive Mode (128,231,210,81,222,250) */ if (npu.byteRangeStart != -1) { safef(cmd,sizeof(cmd),"REST %lld\r\n", (long long) npu.byteRangeStart); if (!sendFtpCommand(sd, cmd, NULL, NULL)) { close(sd); return -1; } } /* RETR for files, LIST for directories ending in / */ safef(cmd,sizeof(cmd),"%s %s\r\n",((npu.file[strlen(npu.file)-1]) == '/') ? "LIST" : "RETR", npu.file); sendFtpCommandOnly(sd, cmd); int sdata = -1; if (proxyUrl) sdata = netConnect(pxy.host, parsePasvPort(rs->string)); else sdata = netConnect(npu.host, parsePasvPort(rs->string)); dyStringFree(&rs); if (sdata < 0) { close(sd); return -1; } int secondsWaited = 0; while (TRUE) { if (secondsWaited >= 10) { warn("ftp server error on cmd=[%s] timed-out waiting for data or error", cmd); close(sd); close(sdata); return -1; } if (readReadyWait(sdata, NET_FTP_TIMEOUT)) break; // we have some data if (readReadyWait(sd, 0)) /* wait in microsec */ { // this can see an error like bad filename if (!receiveFtpReply(sd, cmd, NULL, NULL)) { close(sd); close(sdata); return -1; } } ++secondsWaited; } if (retCtrlSd != NULL) { *retCtrlSd = sd; return sdata; } else { /* Because some FTP servers will kill the data connection * as soon as the control connection closes, * we have to develop a workaround using a partner thread. */ fflush(stdin); fflush(stdout); fflush(stderr); struct netConnectFtpParams *params; AllocVar(params); params->sd = sd; params->sdata = sdata; params->npu = npu; /* make a pipe (fds go in pipefd[0] and pipefd[1]) */ if (pipe(params->pipefd) != 0) errAbort("netGetOpenFtpSockets: failed to create pipe: %s", strerror(errno)); int rc; rc = pthread_create(¶ms->thread, NULL, sendFtpDataToPipeThread, (void *)params); if (rc) { errAbort("Unexpected error %d from pthread_create(): %s",rc,strerror(rc)); } return params->pipefd[0]; } } int connectNpu(struct netParsedUrl npu, char *url, boolean noProxy) /* Connect using NetParsedUrl. */ { int sd = -1; if (sameString(npu.protocol, "http")) { sd = netConnect(npu.host, atoi(npu.port)); } else if (sameString(npu.protocol, "https")) { sd = netConnectHttps(npu.host, atoi(npu.port), noProxy); } else { errAbort("netHttpConnect: url (%s) is not for http.", url); return -1; /* never gets here, fixes compiler complaint */ } return sd; } void setAuthorization(struct netParsedUrl npu, char *authHeader, struct dyString *dy) /* Set the specified authorization header with BASIC auth base64-encoded user and password */ { if (!sameString(npu.user,"")) { char up[256]; char *b64up = NULL; safef(up, sizeof(up), "%s:%s", npu.user, npu.password); b64up = base64Encode(up, strlen(up)); dyStringPrintf(dy, "%s: Basic %s\r\n", authHeader, b64up); freez(&b64up); } } boolean checkNoProxy(char *host) /* See if host endsWith element on no_proxy list. Elements are comma-separated. */ { char *list = cloneString(getenv("no_proxy")); if (!list) return FALSE; replaceChar(list, ',', ' '); char *word; while((word=nextWord(&list))) { if (endsWith(host, word)) return TRUE; } return FALSE; } int netHttpConnect(char *url, char *method, char *protocol, char *agent, char *optionalHeader) /* Parse URL, connect to associated server on port, and send most of * the request to the server. If specified in the url send user name * and password too. Typically the "method" will be "GET" or "POST" * and the agent will be the name of your program or * library. optionalHeader may be NULL or contain additional header * lines such as cookie info. * Proxy support via hg.conf httpProxy or env var http_proxy * Return data socket, or -1 if error.*/ { struct netParsedUrl npu; struct netParsedUrl pxy; struct dyString *dy = newDyString(512); int sd = -1; /* Parse the URL and connect. */ netParseUrl(url, &npu); boolean noProxy = checkNoProxy(npu.host); char *proxyUrl = getenv("http_proxy"); if (sameString(npu.protocol, "https")) proxyUrl = NULL; if (noProxy) proxyUrl = NULL; if (proxyUrl) { netParseUrl(proxyUrl, &pxy); if (!sameString(pxy.protocol, "http")) errAbort("Unknown proxy protocol %s in %s.", pxy.protocol, proxyUrl); sd = connectNpu(pxy, url, noProxy); char *logProxy = getenv("log_proxy"); if (sameOk(logProxy,"on")) verbose(1, "%s via proxy %s\n", url, proxyUrl); } else { sd = connectNpu(npu, url, noProxy); } if (sd < 0) return -1; /* Ask remote server for a file. */ char *urlForProxy = NULL; if (proxyUrl) { /* trim off the byterange part at the end of url because proxy does not understand it. */ urlForProxy = cloneString(url); char *x = strrchr(urlForProxy, ';'); if (x && startsWith(";byterange=", x)) *x = 0; } dyStringPrintf(dy, "%s %s %s\r\n", method, proxyUrl ? urlForProxy : npu.file, protocol); freeMem(urlForProxy); dyStringPrintf(dy, "User-Agent: %s\r\n", agent); dyStringPrintf(dy, "Host: "); netHandleHostForIpv6(&npu, dy); boolean portIsDefault = FALSE; /* do not need the 80 since it is the default */ if (sameString(npu.protocol, "http" ) && sameString("80", npu.port)) portIsDefault = TRUE; if (sameString(npu.protocol, "https" ) && sameString("443", npu.port)) portIsDefault = TRUE; if (!portIsDefault) { dyStringAppendC(dy, ':'); dyStringAppend(dy, npu.port); } dyStringPrintf(dy, "\r\n"); setAuthorization(npu, "Authorization", dy); if (proxyUrl) setAuthorization(pxy, "Proxy-Authorization", dy); dyStringAppend(dy, "Accept: */*\r\n"); if (npu.byteRangeStart != -1) { if (npu.byteRangeEnd != -1) dyStringPrintf(dy, "Range: bytes=%lld-%lld\r\n" , (long long)npu.byteRangeStart , (long long)npu.byteRangeEnd); else dyStringPrintf(dy, "Range: bytes=%lld-\r\n" , (long long)npu.byteRangeStart); } if (optionalHeader) dyStringAppend(dy, optionalHeader); /* finish off the header with final blank line */ dyStringAppend(dy, "\r\n"); mustWriteFd(sd, dy->string, dy->stringSize); /* Clean up and return handle. */ dyStringFree(&dy); return sd; } int netOpenHttpExt(char *url, char *method, char *optionalHeader) /* Return a file handle that will read the url. optionalHeader * may by NULL or may contain cookies and other info. */ { return netHttpConnect(url, method, "HTTP/1.0", "genome.ucsc.edu/net.c", optionalHeader); } static int netGetOpenHttp(char *url) /* Return a file handle that will read the url. */ { return netOpenHttpExt(url, "GET", NULL); } int netUrlHeadExt(char *url, char *method, struct hash *hash) /* Go get head and return status. Return negative number if * can't get head. If hash is non-null, fill it with header * lines with upper cased keywords for case-insensitive lookup, * including hopefully CONTENT-TYPE: . */ { int sd = netOpenHttpExt(url, method, NULL); int status = EIO; if (sd >= 0) { char *line, *word; struct lineFile *lf = lineFileAttach(url, TRUE, sd); if (lineFileNext(lf, &line, NULL)) { if (startsWith("HTTP/", line)) { word = nextWord(&line); word = nextWord(&line); if (word != NULL && isdigit(word[0])) { status = atoi(word); if (hash != NULL) { while (lineFileNext(lf, &line, NULL)) { word = nextWord(&line); if (word == NULL) break; hashAdd(hash, strUpper(word), cloneString(skipLeadingSpaces(line))); } } } } } lineFileClose(&lf); } else status = errno; return status; } int netUrlHead(char *url, struct hash *hash) /* Go get head and return status. Return negative number if * can't get head. If hash is non-null, fill it with header * lines with upper cased keywords for case-insensitive lookup, * including hopefully CONTENT-TYPE: . */ { return netUrlHeadExt(url, "HEAD", hash); } int netUrlFakeHeadByGet(char *url, struct hash *hash) /* Use GET with byteRange as an alternate method to HEAD. * Return status. */ { char rangeUrl[MAXURLSIZE]; safef(rangeUrl, sizeof(rangeUrl), "%s;byterange=0-0", url); int status = netUrlHeadExt(rangeUrl, "GET", hash); return status; } int netUrlOpenSockets(char *url, int *retCtrlSocket) /* Return socket descriptor (low-level file handle) for read()ing url data, * or -1 if error. * If retCtrlSocket is non-NULL and url is FTP, set *retCtrlSocket * to the FTP control socket which is left open for a persistent connection. * close(result) (and close(*retCtrlSocket) if applicable) when done. * If url is missing :// then it's just treated as a file. */ { if (stringIn("://", url) == NULL) return open(url, O_RDONLY); else { if (startsWith("http://",url) || startsWith("https://",url)) return netGetOpenHttp(url); else if (startsWith("ftp://",url)) return netGetOpenFtpSockets(url, retCtrlSocket); else errAbort("Sorry, can only netUrlOpen http, https and ftp currently, not '%s'", url); } return -1; } int netUrlOpen(char *url) /* Return socket descriptor (low-level file handle) for read()ing url data, * or -1 if error. Just close(result) when done. */ { return netUrlOpenSockets(url, NULL); } struct dyString *netSlurpFile(int sd) /* Slurp file into dynamic string and return. */ { char buf[4*1024]; int readSize; struct dyString *dy = newDyString(4*1024); /* Slurp file into dy and return. */ while ((readSize = read(sd, buf, sizeof(buf))) > 0) dyStringAppendN(dy, buf, readSize); return dy; } struct dyString *netSlurpUrl(char *url) /* Go grab all of URL and return it as dynamic string. */ { int sd = netUrlOpen(url); if (sd < 0) errAbort("netSlurpUrl: failed to open socket for [%s]", url); struct dyString *dy = netSlurpFile(sd); close(sd); return dy; } static void parseContentRange(char *x, ssize_t *rangeStart, ssize_t *rangeEnd) /* parse the content range information from response header value "bytes 763400000-763400112/763400113" */ { /* default to no byte range specified */ *rangeStart = -1; *rangeEnd = -1; if (startsWith("bytes ", x)) { char *y=strchr(x, ' '); ++y; char *z=strchr(y, '-'); if (z) { ++z; *rangeStart = atoll(y); if (z[0] != '\0') *rangeEnd = atoll(z); } } } boolean netSkipHttpHeaderLinesWithRedirect(int sd, char *url, char **redirectedUrl) /* Skip http header lines. Return FALSE if there's a problem. * The input is a standard sd or fd descriptor. * This is meant to be able work even with a re-passable stream handle, * e.g. can pass it to the pipes routines, which means we can't * attach a linefile since filling its buffer reads in more than just the http header. * Handles 300, 301, 302, 303, 307, 308 http redirects by setting *redirectedUrl to * the new location. */ { -char buf[2000]; +char buf[8192]; char *line = buf; int maxbuf = sizeof(buf); int i=0; char c = ' '; int nread = 0; char *sep = NULL; char *headerName = NULL; char *headerVal = NULL; boolean redirect = FALSE; boolean byteRangeUsed = (strstr(url,";byterange=") != NULL); ssize_t byteRangeStart = -1; ssize_t byteRangeEnd = -1; boolean foundContentRange = FALSE; ssize_t contentRangeStart = -1; ssize_t contentRangeEnd = -1; boolean mustUseProxy = FALSE; /* User must use proxy 305 error*/ char *proxyLocation = NULL; boolean mustUseProxyAuth = FALSE; /* User must use proxy authentication 407 error*/ if (byteRangeUsed) { parseByteRange(url, &byteRangeStart, &byteRangeEnd, FALSE); } while(TRUE) { i = 0; while (TRUE) { nread = read(sd, &c, 1); /* one char at a time, but http headers are small */ if (nread != 1) { if (nread == -1) { if (errno == EINTR) continue; warn("Error (%s) reading http header on %s", strerror(errno), url); } else if (nread == 0) warn("Error unexpected end of input reading http header on %s", url); else warn("Error reading http header on %s", url); return FALSE; /* err reading descriptor */ } if (c == 10) break; if (c != 13) buf[i++] = c; if (i >= maxbuf) { warn("http header line too long > %d chars.",maxbuf); return FALSE; } } buf[i] = 0; /* add string terminator */ if (sameString(line,"")) { break; /* End of Header found */ } if (startsWith("HTTP/", line)) { char *code; nextWord(&line); // version code = nextWord(&line); if (code == NULL) { warn("Strange http header on %s", url); return FALSE; } if (sameString(code, "301") || sameString(code, "302") || sameString(code, "307") || sameString(code, "308") ) { redirect = TRUE; } else if (sameString(code, "305")) { mustUseProxy = TRUE; } else if (sameString(code, "407")) { mustUseProxyAuth = TRUE; } else if (byteRangeUsed /* hack for Apache bug 2.2.20 and 2.2.21 2011-10-21 should be OK to remove after one year. */ && !(sameString(code, "200") && byteRangeStart == 0 && byteRangeEnd == -1)) { if (!sameString(code, "206")) { if (sameString(code, "200")) warn("Byte-range request was ignored by server. "); warn("Expected Partial Content 206. %s: %s %s. rangeStart=%lld rangeEnd=%lld", url, code, line, (long long)byteRangeStart, (long long)byteRangeEnd); return FALSE; } } else if (sameString(code, "404")) { warn("404 file not found on %s", url); return FALSE; } else if (!sameString(code, "200")) { warn("Expected 200 %s: %s %s", url, code, line); return FALSE; } line = buf; /* restore it */ } headerName = line; sep = strchr(line,':'); if (sep) { *sep = 0; headerVal = skipLeadingSpaces(++sep); } else { headerVal = NULL; } if (sameWord(headerName,"Location")) { if (redirect) *redirectedUrl = cloneString(headerVal); if (mustUseProxy) proxyLocation = cloneString(headerVal); } if (sameWord(headerName,"Content-Range")) { if (byteRangeUsed) { foundContentRange = TRUE; parseContentRange(headerVal, &contentRangeStart, &contentRangeEnd); if ((contentRangeStart != byteRangeStart) || (byteRangeEnd != -1 && (contentRangeEnd != byteRangeEnd))) { char bre[256]; safef(bre, sizeof bre, "%lld", (long long)byteRangeEnd); if (byteRangeEnd == -1) bre[0] = 0; warn("Found Content-Range: %s. Expected bytes %lld-%s. Improper caching of 206 reponse byte-ranges?", headerVal, (long long) byteRangeStart, bre); return FALSE; } } } } if (mustUseProxy || mustUseProxyAuth) { warn("%s: %s error. Use Proxy%s. Location = %s", url, mustUseProxy ? "" : " Authentication", mustUseProxy ? "305" : "407", proxyLocation ? proxyLocation : "not given"); return FALSE; } if (byteRangeUsed && !foundContentRange && !redirect /* hack for Apache bug 2.2.20 and 2.2.21 2011-10-21 should be OK to remove after one year. */ && !(byteRangeStart == 0 && byteRangeEnd == -1)) { char bre[256]; safef(bre, sizeof bre, "%lld", (long long)byteRangeEnd); if (byteRangeEnd == -1) bre[0] = 0; warn("Expected response header Content-Range: %lld-%s", (long long) byteRangeStart, bre); return FALSE; } return TRUE; } char *transferParamsToRedirectedUrl(char *url, char *newUrl) /* Transfer password, byteRange, and any other parameters from url to newUrl and return result. * freeMem result. */ { struct netParsedUrl npu, newNpu; /* Parse the old URL to make parts available for graft onto the redirected url. */ /* This makes redirection work with byterange urls and user:password@ */ netParseUrl(url, &npu); netParseUrl(newUrl, &newNpu); if (npu.byteRangeStart != -1) { newNpu.byteRangeStart = npu.byteRangeStart; newNpu.byteRangeEnd = npu.byteRangeEnd; } if ((npu.user[0] != 0) && (newNpu.user[0] == 0)) { safecpy(newNpu.user, sizeof newNpu.user, npu.user); safecpy(newNpu.password, sizeof newNpu.password, npu.password); } return urlFromNetParsedUrl(&newNpu); } boolean netSkipHttpHeaderLinesHandlingRedirect(int sd, char *url, int *redirectedSd, char **redirectedUrl) /* Skip http headers lines, returning FALSE if there is a problem. Generally called as * netSkipHttpHeaderLine(sd, url, &sd, &url); * where sd is a socket (file) opened with netUrlOpen(url), and url is in dynamic memory. * If the http header indicates that the file has moved, then it will update the *redirectedSd and * *redirectedUrl with the new socket and URL, first closing sd. * If for some reason you want to detect whether the forwarding has occurred you could * call this as: * char *newUrl = NULL; * int newSd = 0; * netSkipHttpHeaderLine(sd, url, &newSd, &newUrl); * if (newUrl != NULL) * // Update sd with newSd, free url if appropriate and replace it with newUrl, etc. * // free newUrl when finished. * This routine handles up to 5 steps of redirection. * The logic to this routine is also complicated a little to make it work in a pipe, which means we * can't attach a lineFile since filling the lineFile buffer reads in more than just the http header. */ { int redirectCount = 0; while (TRUE) { /* url needed for err msgs, and to return redirect location */ char *newUrl = NULL; boolean success = netSkipHttpHeaderLinesWithRedirect(sd, url, &newUrl); if (success && !newUrl) /* success after 0 to 5 redirects */ { if (redirectCount > 0) { *redirectedSd = sd; *redirectedUrl = url; } else { *redirectedSd = -1; *redirectedUrl = NULL; } return TRUE; } close(sd); if (success) { /* we have a new url to try */ ++redirectCount; if (redirectCount > 5) { warn("code 30x redirects: exceeded limit of 5 redirects, %s", newUrl); success = FALSE; } else if (!startsWith("http://",newUrl) && !startsWith("https://",newUrl)) { warn("redirected to non-http(s): %s", newUrl); success = FALSE; } else { newUrl = transferParamsToRedirectedUrl(url, newUrl); sd = netUrlOpen(newUrl); if (sd < 0) { warn("Couldn't open %s", newUrl); success = FALSE; } } } if (redirectCount > 1) freeMem(url); if (!success) { /* failure after 0 to 5 redirects */ if (redirectCount > 0) freeMem(newUrl); return FALSE; } url = newUrl; } return FALSE; } struct lineFile *netLineFileMayOpen(char *url) /* Return a lineFile attached to url. http skips header. * Supports some compression formats. Prints warning message, but * does not abort, just returning NULL if there's a problem. */ { int sd = netUrlOpen(url); if (sd < 0) { warn("Couldn't open %s", url); return NULL; } else { struct lineFile *lf = NULL; char *newUrl = NULL; int newSd = 0; if (startsWith("http://",url) || startsWith("https://",url)) { if (!netSkipHttpHeaderLinesHandlingRedirect(sd, url, &newSd, &newUrl)) { return NULL; } if (newUrl != NULL) { /* Update sd with newSd, replace it with newUrl, etc. */ sd = newSd; url = newUrl; } } char *urlDecoded = cloneString(url); cgiDecode(url, urlDecoded, strlen(url)); boolean isCompressed = (endsWith(urlDecoded,".gz") || endsWith(urlDecoded,".Z") || endsWith(urlDecoded,".bz2")); freeMem(urlDecoded); if (isCompressed) { lf = lineFileDecompressFd(url, TRUE, sd); /* url needed only for compress type determination */ } else { lf = lineFileAttach(url, TRUE, sd); } if (newUrl) freeMem(newUrl); return lf; } } int netUrlMustOpenPastHeader(char *url) /* Get socket descriptor for URL. Process header, handling any forwarding and * the like. Do errAbort if there's a problem, which includes anything but a 200 * return from http after forwarding. */ { int sd = netUrlOpen(url); if (sd < 0) noWarnAbort(); int newSd = 0; if (startsWith("http://",url) || startsWith("https://",url)) { char *newUrl = NULL; if (!netSkipHttpHeaderLinesHandlingRedirect(sd, url, &newSd, &newUrl)) noWarnAbort(); if (newUrl != NULL) { sd = newSd; freeMem(newUrl); } } return sd; } struct lineFile *netLineFileSilentOpen(char *url) /* Open a lineFile on a URL. Just return NULL without any user * visible warning message if there's a problem. */ { pushSilentWarnHandler(); struct lineFile *lf = netLineFileMayOpen(url); popWarnHandler(); return lf; } char *netReadTextFileIfExists(char *url) /* Read entire URL and return it as a string. URL should be text (embedded zeros will be * interpreted as end of string). If the url doesn't exist or has other problems, * returns NULL. */ { struct lineFile *lf = netLineFileSilentOpen(url); if (lf == NULL) return NULL; char *text = lineFileReadAll(lf); lineFileClose(&lf); return text; } struct lineFile *netLineFileOpen(char *url) /* Return a lineFile attached to url. This one * will skip any headers. Free this with * lineFileClose(). */ { struct lineFile *lf = netLineFileMayOpen(url); if (lf == NULL) noWarnAbort(); return lf; } boolean netSendString(int sd, char *s) /* Send a string down a socket - length byte first. */ { int length = strlen(s); UBYTE len; if (length > 255) errAbort("Trying to send a string longer than 255 bytes (%d bytes)", length); len = length; if (write(sd, &len, 1)<0) { warn("Couldn't send string to socket"); return FALSE; } if (write(sd, s, length)<0) { warn("Couldn't send string to socket"); return FALSE; } return TRUE; } boolean netSendLongString(int sd, char *s) /* Send a long string down socket: two bytes for length. */ { unsigned length = strlen(s); UBYTE b[2]; if (length >= 64*1024) { warn("Trying to send a string longer than 64k bytes (%d bytes)", length); return FALSE; } b[0] = (length>>8); b[1] = (length&0xff); if (write(sd, b, 2) < 0) { warn("Couldn't send long string to socket"); return FALSE; } if (write(sd, s, length)<0) { warn("Couldn't send long string to socket"); return FALSE; } return TRUE; } boolean netSendHugeString(int sd, char *s) /* Send a long string down socket: four bytes for length. */ { unsigned long length = strlen(s); unsigned long l = length; UBYTE b[4]; int i; for (i=3; i>=0; --i) { b[i] = l & 0xff; l >>= 8; } if (write(sd, b, 4) < 0) { warn("Couldn't send huge string to socket"); return FALSE; } if (write(sd, s, length) < 0) { warn("Couldn't send huge string to socket"); return FALSE; } return TRUE; } char *netGetString(int sd, char buf[256]) /* Read string into buf and return it. If buf is NULL * an internal buffer will be used. Print warning message * and return NULL if any problem. */ { static char sbuf[256]; UBYTE len = 0; int length; int sz; if (buf == NULL) buf = sbuf; sz = netReadAll(sd, &len, 1); if (sz == 0) return NULL; if (sz < 0) { warn("Couldn't read string length"); return NULL; } length = len; if (length > 0) if (netReadAll(sd, buf, length) < 0) { warn("Couldn't read string body"); return NULL; } buf[length] = 0; return buf; } char *netGetLongString(int sd) /* Read string and return it. freeMem * the result when done. */ { UBYTE b[2]; char *s = NULL; int length = 0; int sz; b[0] = b[1] = 0; sz = netReadAll(sd, b, 2); if (sz == 0) return NULL; if (sz < 0) { warn("Couldn't read long string length"); return NULL; } length = (b[0]<<8) + b[1]; s = needMem(length+1); if (length > 0) if (netReadAll(sd, s, length) < 0) { warn("Couldn't read long string body"); return NULL; } s[length] = 0; return s; } char *netGetHugeString(int sd) /* Read string and return it. freeMem * the result when done. */ { UBYTE b[4]; char *s = NULL; unsigned long length = 0; int sz, i; sz = netReadAll(sd, b, 4); if (sz == 0) return NULL; if (sz < 4) { warn("Couldn't read huge string length"); return NULL; } for (i=0; i<4; ++i) { length <<= 8; length += b[i]; } s = needMem(length+1); if (length > 0) { if (netReadAll(sd, s, length) < 0) { warn("Couldn't read huge string body"); return NULL; } } s[length] = 0; return s; } char *netRecieveString(int sd, char buf[256]) /* Read string into buf and return it. If buf is NULL * an internal buffer will be used. Abort if any problem. */ { char *s = netGetString(sd, buf); if (s == NULL) noWarnAbort(); return s; } char *netRecieveLongString(int sd) /* Read string and return it. freeMem * the result when done. Abort if any problem*/ { char *s = netGetLongString(sd); if (s == NULL) noWarnAbort(); return s; } char *netRecieveHugeString(int sd) /* Read string and return it. freeMem * the result when done. Abort if any problem*/ { char *s = netGetHugeString(sd); if (s == NULL) noWarnAbort(); return s; } struct lineFile *netHttpLineFileMayOpen(char *url, struct netParsedUrl **npu) /* Parse URL and open an HTTP socket for it but don't send a request yet. */ { int sd; struct lineFile *lf; /* Parse the URL and try to connect. */ AllocVar(*npu); netParseUrl(url, *npu); if (!sameString((*npu)->protocol, "http")) errAbort("netHttpLineFileMayOpen: url (%s) is not for http.", url); sd = netConnect((*npu)->host, atoi((*npu)->port)); if (sd < 0) return NULL; /* Return handle. */ lf = lineFileAttach(url, TRUE, sd); return lf; } /* netHttpLineFileMayOpen */ void netHttpGet(struct lineFile *lf, struct netParsedUrl *npu, boolean keepAlive) /* Send a GET request, possibly with Keep-Alive. */ { struct dyString *dy = newDyString(512); /* Ask remote server for the file/query. */ dyStringPrintf(dy, "GET %s HTTP/1.1\r\n", npu->file); dyStringPrintf(dy, "User-Agent: genome.ucsc.edu/net.c\r\n"); dyStringPrintf(dy, "Host: "); netHandleHostForIpv6(npu, dy); dyStringAppendC(dy, ':'); dyStringAppend(dy, npu->port); dyStringPrintf(dy, "\r\n"); if (!sameString(npu->user,"")) { char up[256]; char *b64up = NULL; safef(up,sizeof(up), "%s:%s", npu->user, npu->password); b64up = base64Encode(up, strlen(up)); dyStringPrintf(dy, "Authorization: Basic %s\r\n", b64up); freez(&b64up); } dyStringAppend(dy, "Accept: */*\r\n"); if (keepAlive) { dyStringAppend(dy, "Connection: Keep-Alive\r\n"); dyStringAppend(dy, "Connection: Persist\r\n"); } else dyStringAppend(dy, "Connection: close\r\n"); dyStringAppend(dy, "\r\n"); mustWriteFd(lf->fd, dy->string, dy->stringSize); /* Clean up. */ dyStringFree(&dy); } /* netHttpGet */ int netHttpGetMultiple(char *url, struct slName *queries, void *userData, void (*responseCB)(void *userData, char *req, char *hdr, struct dyString *body)) /* Given an URL which is the base of all requests to be made, and a * linked list of queries to be appended to that base and sent in as * requests, send the requests as a batch and read the HTTP response * headers and bodies. If not all the requests get responses (i.e. if * the server is ignoring Keep-Alive or is imposing a limit), try again * until we can't connect or until all requests have been served. * For each HTTP response, do a callback. */ { struct slName *qStart; struct slName *qPtr; struct lineFile *lf; struct netParsedUrl *npu; struct dyString *dyQ = newDyString(512); struct dyString *body; char *base; char *hdr; int qCount; int qTotal; int numParseFailures; int contentLength; boolean chunked; boolean done; boolean keepAlive; /* Find out how many queries we'll need to do so we know how many times * it's OK to run into end of file in case server ignores Keep-Alive. */ qTotal = 0; for (qPtr = queries; qPtr != NULL; qPtr = qPtr->next) { qTotal++; } done = FALSE; qCount = 0; numParseFailures = 0; qStart = queries; while ((! done) && (qStart != NULL)) { lf = netHttpLineFileMayOpen(url, &npu); if (lf == NULL) { done = TRUE; break; } base = cloneString(npu->file); /* Send all remaining requests with keep-alive. */ for (qPtr = qStart; qPtr != NULL; qPtr = qPtr->next) { dyStringClear(dyQ); dyStringAppend(dyQ, base); dyStringAppend(dyQ, qPtr->name); strcpy(npu->file, dyQ->string); keepAlive = (qPtr->next == NULL) ? FALSE : TRUE; netHttpGet(lf, npu, keepAlive); } /* Get as many responses as we can; call responseCB() and * advance qStart for each. */ for (qPtr = qStart; qPtr != NULL; qPtr = qPtr->next) { if (lineFileParseHttpHeader(lf, &hdr, &chunked, &contentLength)) { body = lineFileSlurpHttpBody(lf, chunked, contentLength); dyStringClear(dyQ); dyStringAppend(dyQ, base); dyStringAppend(dyQ, qPtr->name); responseCB(userData, dyQ->string, hdr, body); qStart = qStart->next; qCount++; } else { if (numParseFailures++ > qTotal) { done = TRUE; } break; } } } return qCount; } /* netHttpMultipleQueries */ boolean hasProtocol(char *urlOrPath) /* Return TRUE if it looks like it has http://, ftp:// etc. */ { return stringIn("://", urlOrPath) != NULL; }