4413ec298eaf5ef8adce9df9b0b7c1d9a405f145 galt Tue May 17 14:49:58 2011 -0700 Check the Content-Range for 206 response to confirm that it matches the byteRange requested - this can detect certain kinds of caching errors by servers; also removed unneeded newlines in warn and errAbort calls. diff --git src/lib/net.c src/lib/net.c index a57d9c6..bf99f67 100644 --- src/lib/net.c +++ src/lib/net.c @@ -49,113 +49,113 @@ long fcntlFlags; if (hostName == NULL) { warn("NULL hostName in netConnect"); return -1; } if (!internetFillInAddress(hostName, port, &sai)) return -1; if ((sd = netStreamSocket()) < 0) return sd; // Set non-blocking if ((fcntlFlags = fcntl(sd, F_GETFL, NULL)) < 0) { - warn("Error fcntl(..., F_GETFL) (%s)\n", strerror(errno)); + warn("Error fcntl(..., F_GETFL) (%s)", strerror(errno)); close(sd); return -1; } fcntlFlags |= O_NONBLOCK; if (fcntl(sd, F_SETFL, fcntlFlags) < 0) { - warn("Error fcntl(..., F_SETFL) (%s)\n", strerror(errno)); + warn("Error fcntl(..., F_SETFL) (%s)", strerror(errno)); close(sd); return -1; } // Trying to connect with timeout res = connect(sd, (struct sockaddr*) &sai, sizeof(sai)); if (res < 0) { if (errno == EINPROGRESS) { while (1) { lTime.tv_sec = (long) (msTimeout/1000); lTime.tv_usec = (long) (((msTimeout/1000)-lTime.tv_sec)*1000000); FD_ZERO(&mySet); FD_SET(sd, &mySet); res = select(sd+1, NULL, &mySet, &mySet, &lTime); if (res < 0) { if (errno != EINTR) { - warn("Error in select() during TCP non-blocking connect %d - %s\n", errno, strerror(errno)); + warn("Error in select() during TCP non-blocking connect %d - %s", errno, strerror(errno)); close(sd); return -1; } } else if (res > 0) { // Socket selected for write when it is ready int valOpt; socklen_t lon; // But check the socket for any errors lon = sizeof(valOpt); if (getsockopt(sd, SOL_SOCKET, SO_ERROR, (void*) (&valOpt), &lon) < 0) { - warn("Error in getsockopt() %d - %s\n", errno, strerror(errno)); + warn("Error in getsockopt() %d - %s", errno, strerror(errno)); close(sd); return -1; } // Check the value returned... if (valOpt) { - warn("Error in TCP non-blocking connect() %d - %s\n", valOpt, strerror(valOpt)); + warn("Error in TCP non-blocking connect() %d - %s", valOpt, strerror(valOpt)); close(sd); return -1; } break; } else { - warn("TCP non-blocking connect() timed-out in select() after %ld milliseconds - Cancelling!\n", msTimeout); + warn("TCP non-blocking connect() timed-out in select() after %ld milliseconds - Cancelling!", msTimeout); close(sd); return -1; } } } else { - warn("TCP non-blocking connect() error %d - %s\n", errno, strerror(errno)); + warn("TCP non-blocking connect() error %d - %s", errno, strerror(errno)); close(sd); return -1; } } // Set to blocking mode again if ((fcntlFlags = fcntl(sd, F_GETFL, NULL)) < 0) { - warn("Error fcntl(..., F_GETFL) (%s)\n", strerror(errno)); + warn("Error fcntl(..., F_GETFL) (%s)", strerror(errno)); close(sd); return -1; } fcntlFlags &= (~O_NONBLOCK); if (fcntl(sd, F_SETFL, fcntlFlags) < 0) { - warn("Error fcntl(..., F_SETFL) (%s)\n", strerror(errno)); + warn("Error fcntl(..., F_SETFL) (%s)", strerror(errno)); close(sd); return -1; } return sd; } int netConnect(char *hostName, int port) /* Start connection with a server. */ { return netConnectWithTimeout(hostName, port, 10000); // 10 seconds connect timeout } @@ -300,69 +300,99 @@ int netMustReadAll(int sd, void *vBuf, size_t size) /* Read given number of bytes into buffer or die. * Don't give up if first read is short! */ { int ret = netReadAll(sd, vBuf, size); if (ret < 0) errnoAbort("Couldn't finish netReadAll"); return ret; } static void notGoodSubnet(char *sns) /* Complain about subnet format. */ { errAbort("'%s' is not a properly formatted subnet. Subnets must consist of\n" - "one to three dot-separated numbers between 0 and 255\n", sns); + "one to three dot-separated numbers between 0 and 255", sns); } void netParseSubnet(char *in, unsigned char out[4]) /* Parse subnet, which is a prefix of a normal dotted quad form. * Out will contain 255's for the don't care bits. */ { out[0] = out[1] = out[2] = out[3] = 255; if (in != NULL) { char *snsCopy = strdup(in); char *words[5]; int wordCount, i; wordCount = chopString(snsCopy, ".", words, ArraySize(words)); if (wordCount > 3 || wordCount < 1) notGoodSubnet(in); for (i=0; i<wordCount; ++i) { char *s = words[i]; int x; if (!isdigit(s[0])) notGoodSubnet(in); x = atoi(s); if (x > 255) notGoodSubnet(in); out[i] = x; } freez(&snsCopy); } } +static void parseByteRange(char *url, ssize_t *rangeStart, ssize_t *rangeEnd, boolean terminateAtByteRange) +/* parse the byte range information from url */ +{ +char *x; +/* default to no byte range specified */ +*rangeStart = -1; +*rangeEnd = -1; +x = strrchr(url, ';'); +if (x) + { + if (startsWith(";byterange=", x)) + { + char *y=strchr(x, '='); + ++y; + char *z=strchr(y, '-'); + if (z) + { + ++z; + if (terminateAtByteRange) + *x = 0; + // TODO: use something better than atoll() ? + *rangeStart = atoll(y); + if (z[0] != '\0') + *rangeEnd = atoll(z); + } + } + } + +} + void netParseUrl(char *url, struct netParsedUrl *parsed) /* Parse a URL into components. A full URL is made up as so: * http://user:password@hostName:port/file;byterange=0-499 * User and password may be cgi-encoded. * This is set up so that the http:// and the port are optional. */ { -char *s, *t, *u, *v, *w, *x; +char *s, *t, *u, *v, *w; char buf[1024]; /* Make local copy of URL. */ if (strlen(url) >= sizeof(buf)) errAbort("Url too long: '%s'", url); strcpy(buf, url); url = buf; /* Find out protocol - default to http. */ s = trimSpaces(url); s = stringIn("://", url); if (s == NULL) { strcpy(parsed->protocol, "http"); s = url; @@ -371,49 +401,31 @@ { *s = 0; tolowers(url); safecpy(parsed->protocol, sizeof(parsed->protocol), url); s += 3; } /* Split off file part. */ parsed->byteRangeStart = -1; /* default to no byte range specified */ parsed->byteRangeEnd = -1; u = strchr(s, '/'); if (u == NULL) strcpy(parsed->file, "/"); else { - x = strrchr(u, ';'); - if (x) - { - if (startsWith(";byterange=", x)) - { - char *y=strchr(x, '='); - ++y; - char *z=strchr(y, '-'); - if (z) - { - ++z; - *x = 0; - // TODO: use something better than atol() ? - parsed->byteRangeStart = atoll(y); - if (z[0] != '\0') - parsed->byteRangeEnd = atoll(z); - } - } - } + parseByteRange(u, &parsed->byteRangeStart, &parsed->byteRangeEnd, TRUE); /* need to encode spaces, but not ! other characters */ char *t=replaceChars(u," ","%20"); safecpy(parsed->file, sizeof(parsed->file), t); freeMem(t); *u = 0; } /* Split off user part */ v = strchr(s, '@'); if (v == NULL) { if (sameWord(parsed->protocol,"http") || sameWord(parsed->protocol,"https")) { @@ -557,31 +569,31 @@ if ( isdigit(startLastLine[0]) && isdigit(startLastLine[1]) && isdigit(startLastLine[2]) && startLastLine[3]==' ') break; if (readSize == 0) break; // EOF /* must be some text info we can't use, ignore it till we get status code */ } int reply = atoi(startLastLine); if ((reply < 200) || (reply > 399)) { - warn("ftp server error on cmd=[%s] response=[%s]\n",cmd,rs->string); + warn("ftp server error on cmd=[%s] response=[%s]",cmd,rs->string); return FALSE; } if (retReply) *retReply = rs; else dyStringFree(&rs); if (retCode) *retCode = reply; return TRUE; } static boolean sendFtpCommand(int sd, char *cmd, struct dyString **retReply, int *retCode) /* send command to ftp server and check resulting reply code, * warn and return FALSE if not desired reply. If retReply is non-NULL, store reply text there. */ @@ -844,31 +856,31 @@ sendFtpCommandOnly(sd, cmd); int sdata = netConnect(npu.host, parsePasvPort(rs->string)); dyStringFree(&rs); if (sdata < 0) { close(sd); return -1; } int secondsWaited = 0; while (TRUE) { if (secondsWaited >= 10) { - warn("ftp server error on cmd=[%s] timed-out waiting for data or error\n", cmd); + warn("ftp server error on cmd=[%s] timed-out waiting for data or error", cmd); close(sd); close(sdata); return -1; } if (readReadyWait(sdata, NET_FTP_TIMEOUT)) break; // we have some data if (readReadyWait(sd, 0)) /* wait in microsec */ { // this can see an error like bad filename if (!receiveFtpReply(sd, cmd, NULL, NULL)) { close(sd); close(sdata); return -1; } @@ -1160,155 +1172,219 @@ dyStringAppendN(dy, buf, readSize); return dy; } struct dyString *netSlurpUrl(char *url) /* Go grab all of URL and return it as dynamic string. */ { int sd = netUrlOpen(url); if (sd < 0) errAbort("netSlurpUrl: failed to open socket for [%s]", url); struct dyString *dy = netSlurpFile(sd); close(sd); return dy; } +static void parseContentRange(char *x, ssize_t *rangeStart, ssize_t *rangeEnd) +/* parse the content range information from response header value + "bytes 763400000-763400112/763400113" + */ +{ +/* default to no byte range specified */ +*rangeStart = -1; +*rangeEnd = -1; +if (startsWith("bytes ", x)) + { + char *y=strchr(x, ' '); + ++y; + char *z=strchr(y, '-'); + if (z) + { + ++z; + // TODO: use something better than atoll() ? + *rangeStart = atoll(y); + if (z[0] != '\0') + *rangeEnd = atoll(z); + } + } + +} + boolean netSkipHttpHeaderLinesWithRedirect(int sd, char *url, char **redirectedUrl) /* Skip http header lines. Return FALSE if there's a problem. * The input is a standard sd or fd descriptor. * This is meant to be able work even with a re-passable stream handle, * e.g. can pass it to the pipes routines, which means we can't * attach a linefile since filling its buffer reads in more than just the http header. * Handles 300, 301, 302, 303, 307 http redirects by setting *redirectedUrl to * the new location. */ { char buf[2000]; char *line = buf; int maxbuf = sizeof(buf); int i=0; char c = ' '; int nread = 0; char *sep = NULL; char *headerName = NULL; char *headerVal = NULL; boolean redirect = FALSE; boolean byteRangeUsed = (strstr(url,";byterange=") != NULL); +ssize_t byteRangeStart = -1; +ssize_t byteRangeEnd = -1; +boolean foundContentRange = FALSE; +ssize_t contentRangeStart = -1; +ssize_t contentRangeEnd = -1; boolean mustUseProxy = FALSE; /* User must use proxy 305 error*/ char *proxyLocation = NULL; boolean mustUseProxyAuth = FALSE; /* User must use proxy authentication 407 error*/ +if (byteRangeUsed) + { + parseByteRange(url, &byteRangeStart, &byteRangeEnd, FALSE); + } + while(TRUE) { i = 0; while (TRUE) { nread = read(sd, &c, 1); /* one char at a time, but http headers are small */ if (nread != 1) { if (nread == -1) - warn("Error (%s) reading http header on %s\n", strerror(errno), url); + warn("Error (%s) reading http header on %s", strerror(errno), url); else if (nread == 0) - warn("Error unexpected end of input reading http header on %s\n", url); + warn("Error unexpected end of input reading http header on %s", url); else - warn("Error reading http header on %s\n", url); + warn("Error reading http header on %s", url); return FALSE; /* err reading descriptor */ } if (c == 10) break; if (c != 13) buf[i++] = c; if (i >= maxbuf) { warn("http header line too long > %d chars.",maxbuf); return FALSE; } } buf[i] = 0; /* add string terminator */ if (sameString(line,"")) { break; /* End of Header found */ } if (startsWith("HTTP/", line)) { char *version, *code; version = nextWord(&line); code = nextWord(&line); if (code == NULL) { - warn("Strange http header on %s\n", url); + warn("Strange http header on %s", url); return FALSE; } if (startsWith("30", code) && isdigit(code[2]) && ((code[2] >= '0' && code[2] <= '3') || code[2] == '7') && code[3] == 0) { redirect = TRUE; } else if (sameString(code, "305")) { mustUseProxy = TRUE; } else if (sameString(code, "407")) { mustUseProxyAuth = TRUE; } else if (byteRangeUsed) { if (!sameString(code, "206")) { if (sameString(code, "200")) warn("Byte-range request was ignored by server. "); - warn("Expected Partial Content 206. %s: %s %s\n", url, code, line); + warn("Expected Partial Content 206. %s: %s %s", url, code, line); return FALSE; } } else if (!sameString(code, "200")) { - warn("Expected 200 %s: %s %s\n", url, code, line); + warn("Expected 200 %s: %s %s", url, code, line); return FALSE; } line = buf; /* restore it */ } headerName = line; sep = strchr(line,':'); if (sep) { *sep = 0; headerVal = skipLeadingSpaces(++sep); } else { headerVal = NULL; } if (sameWord(headerName,"Location")) { if (redirect) *redirectedUrl = cloneString(headerVal); if (mustUseProxy) proxyLocation = cloneString(headerVal); } + if (sameWord(headerName,"Content-Range")) + { + if (byteRangeUsed) + { + foundContentRange = TRUE; + parseContentRange(headerVal, &contentRangeStart, &contentRangeEnd); + if ((contentRangeStart != byteRangeStart) || + (byteRangeEnd != -1 && (contentRangeEnd != byteRangeEnd))) + { + char bre[256]; + safef(bre, sizeof bre, "%lld", (long long)byteRangeEnd); + if (byteRangeEnd == -1) + bre[0] = 0; + warn("Found Content-Range: %s. Expected bytes %lld-%s. Improper caching of 206 reponse byte-ranges?", + headerVal, (long long) byteRangeStart, bre); + return FALSE; + } + } + } } if (mustUseProxy || mustUseProxyAuth) { - warn("%s: %s error. Use Proxy%s. Location = %s\n", url, + warn("%s: %s error. Use Proxy%s. Location = %s", url, mustUseProxy ? "" : " Authentication", mustUseProxy ? "305" : "407", proxyLocation ? proxyLocation : "not given"); return FALSE; } +if (byteRangeUsed && !foundContentRange) + { + char bre[256]; + safef(bre, sizeof bre, "%lld", (long long)byteRangeEnd); + if (byteRangeEnd == -1) + bre[0] = 0; + warn("Expected response header Content-Range: %lld-%s", (long long) byteRangeStart, bre); + return FALSE; + } + return TRUE; } boolean netSkipHttpHeaderLinesHandlingRedirect(int sd, char *url, int *redirectedSd, char **redirectedUrl) /* Skip http headers lines, returning FALSE if there is a problem. Generally called as * netSkipHttpHeaderLine(sd, url, &sd, &url); * where sd is a socket (file) opened with netUrlOpen(url), and url is in dynamic memory. * If the http header indicates that the file has moved, then it will update the *redirectedSd and * *redirectedUrl with the new socket and URL, first closing sd. * If for some reason you want to detect whether the forwarding has occurred you could * call this as: * char *newUrl = NULL; * int newSd = 0; * netSkipHttpHeaderLine(sd, url, &newSd, &newUrl);