4413ec298eaf5ef8adce9df9b0b7c1d9a405f145
galt
  Tue May 17 14:49:58 2011 -0700
Check the Content-Range for 206 response to confirm that it matches the byteRange requested - this can detect certain kinds of caching errors by servers; also removed unneeded newlines in warn and errAbort calls.
diff --git src/lib/net.c src/lib/net.c
index a57d9c6..bf99f67 100644
--- src/lib/net.c
+++ src/lib/net.c
@@ -49,113 +49,113 @@
 long fcntlFlags;
 
 if (hostName == NULL)
     {
     warn("NULL hostName in netConnect");
     return -1;
     }
 if (!internetFillInAddress(hostName, port, &sai))
     return -1;
 if ((sd = netStreamSocket()) < 0)
     return sd;
 
 // Set non-blocking
 if ((fcntlFlags = fcntl(sd, F_GETFL, NULL)) < 0) 
     {
-    warn("Error fcntl(..., F_GETFL) (%s)\n", strerror(errno));
+    warn("Error fcntl(..., F_GETFL) (%s)", strerror(errno));
     close(sd);
     return -1;
     }
 fcntlFlags |= O_NONBLOCK;
 if (fcntl(sd, F_SETFL, fcntlFlags) < 0) 
     {
-    warn("Error fcntl(..., F_SETFL) (%s)\n", strerror(errno));
+    warn("Error fcntl(..., F_SETFL) (%s)", strerror(errno));
     close(sd);
     return -1;
     }
 
 // Trying to connect with timeout
 res = connect(sd, (struct sockaddr*) &sai, sizeof(sai));
 if (res < 0)
     {
     if (errno == EINPROGRESS)
 	{
 	while (1) 
 	    {
 	    lTime.tv_sec = (long) (msTimeout/1000);
 	    lTime.tv_usec = (long) (((msTimeout/1000)-lTime.tv_sec)*1000000);
 	    FD_ZERO(&mySet);
 	    FD_SET(sd, &mySet);
 	    res = select(sd+1, NULL, &mySet, &mySet, &lTime);
 	    if (res < 0) 
 		{
 		if (errno != EINTR) 
 		    {
-		    warn("Error in select() during TCP non-blocking connect %d - %s\n", errno, strerror(errno));
+		    warn("Error in select() during TCP non-blocking connect %d - %s", errno, strerror(errno));
 		    close(sd);
 		    return -1;
 		    }
 		}
 	    else if (res > 0)
 		{
 		// Socket selected for write when it is ready
 		int valOpt;
 		socklen_t lon;
                 // But check the socket for any errors
                 lon = sizeof(valOpt);
                 if (getsockopt(sd, SOL_SOCKET, SO_ERROR, (void*) (&valOpt), &lon) < 0)
                     {
-                    warn("Error in getsockopt() %d - %s\n", errno, strerror(errno));
+                    warn("Error in getsockopt() %d - %s", errno, strerror(errno));
                     close(sd);
                     return -1;
                     }
                 // Check the value returned...
                 if (valOpt)
                     {
-                    warn("Error in TCP non-blocking connect() %d - %s\n", valOpt, strerror(valOpt));
+                    warn("Error in TCP non-blocking connect() %d - %s", valOpt, strerror(valOpt));
                     close(sd);
                     return -1;
                     }
 		break;
 		}
 	    else
 		{
-		warn("TCP non-blocking connect() timed-out in select() after %ld milliseconds - Cancelling!\n", msTimeout);
+		warn("TCP non-blocking connect() timed-out in select() after %ld milliseconds - Cancelling!", msTimeout);
 		close(sd);
 		return -1;
 		}
 	    }
 	}
     else
 	{
-	warn("TCP non-blocking connect() error %d - %s\n", errno, strerror(errno));
+	warn("TCP non-blocking connect() error %d - %s", errno, strerror(errno));
 	close(sd);
 	return -1;
 	}
     }
 
 // Set to blocking mode again
 if ((fcntlFlags = fcntl(sd, F_GETFL, NULL)) < 0)
     {
-    warn("Error fcntl(..., F_GETFL) (%s)\n", strerror(errno));
+    warn("Error fcntl(..., F_GETFL) (%s)", strerror(errno));
     close(sd);
     return -1;
     }
 fcntlFlags &= (~O_NONBLOCK);
 if (fcntl(sd, F_SETFL, fcntlFlags) < 0)
     {
-    warn("Error fcntl(..., F_SETFL) (%s)\n", strerror(errno));
+    warn("Error fcntl(..., F_SETFL) (%s)", strerror(errno));
     close(sd);
     return -1;
     }
 
 return sd;
 
 }
 
 
 int netConnect(char *hostName, int port)
 /* Start connection with a server. */
 {
 return netConnectWithTimeout(hostName, port, 10000); // 10 seconds connect timeout
 }
 
@@ -300,69 +300,99 @@
 
 int netMustReadAll(int sd, void *vBuf, size_t size)
 /* Read given number of bytes into buffer or die.
  * Don't give up if first read is short! */
 {
 int ret = netReadAll(sd, vBuf, size);
 if (ret < 0)
     errnoAbort("Couldn't finish netReadAll");
 return ret;
 }
 
 static void notGoodSubnet(char *sns)
 /* Complain about subnet format. */
 {
 errAbort("'%s' is not a properly formatted subnet.  Subnets must consist of\n"
-         "one to three dot-separated numbers between 0 and 255\n", sns);
+         "one to three dot-separated numbers between 0 and 255", sns);
 }
 
 void netParseSubnet(char *in, unsigned char out[4])
 /* Parse subnet, which is a prefix of a normal dotted quad form.
  * Out will contain 255's for the don't care bits. */
 {
 out[0] = out[1] = out[2] = out[3] = 255;
 if (in != NULL)
     {
     char *snsCopy = strdup(in);
     char *words[5];
     int wordCount, i;
     wordCount = chopString(snsCopy, ".", words, ArraySize(words));
     if (wordCount > 3 || wordCount < 1)
         notGoodSubnet(in);
     for (i=0; i<wordCount; ++i)
 	{
 	char *s = words[i];
 	int x;
 	if (!isdigit(s[0]))
 	    notGoodSubnet(in);
 	x = atoi(s);
 	if (x > 255)
 	    notGoodSubnet(in);
 	out[i] = x;
 	}
     freez(&snsCopy);
     }
 }
 
+static void parseByteRange(char *url, ssize_t *rangeStart, ssize_t *rangeEnd, boolean terminateAtByteRange)
+/* parse the byte range information from url */
+{
+char *x;
+/* default to no byte range specified */
+*rangeStart = -1;
+*rangeEnd = -1;
+x = strrchr(url, ';');
+if (x)
+    {
+    if (startsWith(";byterange=", x))
+	{
+	char *y=strchr(x, '=');
+	++y;
+	char *z=strchr(y, '-');
+	if (z)
+	    {
+	    ++z;
+	    if (terminateAtByteRange)
+		*x = 0;
+	    // TODO: use something better than atoll() ?
+	    *rangeStart = atoll(y); 
+	    if (z[0] != '\0')
+		*rangeEnd = atoll(z);
+	    }    
+	}
+    }
+
+}
+
 void netParseUrl(char *url, struct netParsedUrl *parsed)
 /* Parse a URL into components.   A full URL is made up as so:
  *   http://user:password@hostName:port/file;byterange=0-499
  * User and password may be cgi-encoded.
  * This is set up so that the http:// and the port are optional. 
  */
 {
-char *s, *t, *u, *v, *w, *x;
+char *s, *t, *u, *v, *w;
 char buf[1024];
 
 /* Make local copy of URL. */
 if (strlen(url) >= sizeof(buf))
     errAbort("Url too long: '%s'", url);
 strcpy(buf, url);
 url = buf;
 
 /* Find out protocol - default to http. */
 s = trimSpaces(url);
 s = stringIn("://", url);
 if (s == NULL)
     {
     strcpy(parsed->protocol, "http");
     s = url;
@@ -371,49 +401,31 @@
     {
     *s = 0;
     tolowers(url);
     safecpy(parsed->protocol, sizeof(parsed->protocol), url);
     s += 3;
     }
 
 /* Split off file part. */
 parsed->byteRangeStart = -1;  /* default to no byte range specified */
 parsed->byteRangeEnd = -1;
 u = strchr(s, '/');
 if (u == NULL)
     strcpy(parsed->file, "/");
 else
     {
-    x = strrchr(u, ';');
-    if (x)
-	{
-	if (startsWith(";byterange=", x))
-	    {
-	    char *y=strchr(x, '=');
-	    ++y;
-	    char *z=strchr(y, '-');
-	    if (z)
-		{
-    		++z;
-		*x = 0;
-		// TODO: use something better than atol() ?
-		parsed->byteRangeStart = atoll(y); 
-		if (z[0] != '\0')
-		    parsed->byteRangeEnd = atoll(z);
-	    	}    
-	    }
-	}
+    parseByteRange(u, &parsed->byteRangeStart, &parsed->byteRangeEnd, TRUE);
 
     /* need to encode spaces, but not ! other characters */
     char *t=replaceChars(u," ","%20");
     safecpy(parsed->file, sizeof(parsed->file), t);
     freeMem(t);
     *u = 0;
     }
 
 /* Split off user part */
 v = strchr(s, '@');
 if (v == NULL)
     {
     if (sameWord(parsed->protocol,"http") ||
         sameWord(parsed->protocol,"https"))
 	{
@@ -557,31 +569,31 @@
       if (
 	isdigit(startLastLine[0]) &&
 	isdigit(startLastLine[1]) &&
 	isdigit(startLastLine[2]) &&
 	startLastLine[3]==' ')
 	break;
 	
     if (readSize == 0)
 	break;  // EOF
     /* must be some text info we can't use, ignore it till we get status code */
     }
 
 int reply = atoi(startLastLine);
 if ((reply < 200) || (reply > 399))
     {
-    warn("ftp server error on cmd=[%s] response=[%s]\n",cmd,rs->string);
+    warn("ftp server error on cmd=[%s] response=[%s]",cmd,rs->string);
     return FALSE;
     }
     
 if (retReply)
     *retReply = rs;
 else
     dyStringFree(&rs);
 if (retCode)
     *retCode = reply;
 return TRUE;
 }
 
 static boolean sendFtpCommand(int sd, char *cmd, struct dyString **retReply, int *retCode)
 /* send command to ftp server and check resulting reply code, 
  * warn and return FALSE if not desired reply.  If retReply is non-NULL, store reply text there. */
@@ -844,31 +856,31 @@
 sendFtpCommandOnly(sd, cmd);
 
 int sdata = netConnect(npu.host, parsePasvPort(rs->string));
 dyStringFree(&rs);
 if (sdata < 0)
     {
     close(sd);
     return -1;
     }
 
 int secondsWaited = 0;
 while (TRUE)
     {
     if (secondsWaited >= 10)
 	{
-	warn("ftp server error on cmd=[%s] timed-out waiting for data or error\n", cmd);
+	warn("ftp server error on cmd=[%s] timed-out waiting for data or error", cmd);
 	close(sd);
 	close(sdata);
 	return -1;
 	}
     if (readReadyWait(sdata, NET_FTP_TIMEOUT))
 	break;   // we have some data
     if (readReadyWait(sd, 0)) /* wait in microsec */
 	{
 	// this can see an error like bad filename
 	if (!receiveFtpReply(sd, cmd, NULL, NULL))
 	    {
 	    close(sd);
 	    close(sdata);
 	    return -1;
 	    }
@@ -1160,155 +1172,219 @@
     dyStringAppendN(dy, buf, readSize);
 return dy;
 }
 
 struct dyString *netSlurpUrl(char *url)
 /* Go grab all of URL and return it as dynamic string. */
 {
 int sd = netUrlOpen(url);
 if (sd < 0)
     errAbort("netSlurpUrl: failed to open socket for [%s]", url);
 struct dyString *dy = netSlurpFile(sd);
 close(sd);
 return dy;
 }
 
+static void parseContentRange(char *x, ssize_t *rangeStart, ssize_t *rangeEnd)
+/* parse the content range information from response header value 
+	"bytes 763400000-763400112/763400113"
+ */
+{
+/* default to no byte range specified */
+*rangeStart = -1;
+*rangeEnd = -1;
+if (startsWith("bytes ", x))
+    {
+    char *y=strchr(x, ' ');
+    ++y;
+    char *z=strchr(y, '-');
+    if (z)
+	{
+	++z;
+	// TODO: use something better than atoll() ?
+	*rangeStart = atoll(y); 
+	if (z[0] != '\0')
+	    *rangeEnd = atoll(z);
+	}    
+    }
+
+}
+
 
 boolean netSkipHttpHeaderLinesWithRedirect(int sd, char *url, char **redirectedUrl)
 /* Skip http header lines. Return FALSE if there's a problem.
  * The input is a standard sd or fd descriptor.
  * This is meant to be able work even with a re-passable stream handle,
  * e.g. can pass it to the pipes routines, which means we can't
  * attach a linefile since filling its buffer reads in more than just the http header.
  * Handles 300, 301, 302, 303, 307 http redirects by setting *redirectedUrl to
  * the new location. */
 {
 char buf[2000];
 char *line = buf;
 int maxbuf = sizeof(buf);
 int i=0;
 char c = ' ';
 int nread = 0;
 char *sep = NULL;
 char *headerName = NULL;
 char *headerVal = NULL;
 boolean redirect = FALSE;
 boolean byteRangeUsed = (strstr(url,";byterange=") != NULL);
+ssize_t byteRangeStart = -1;
+ssize_t byteRangeEnd = -1;
+boolean foundContentRange = FALSE;
+ssize_t contentRangeStart = -1;
+ssize_t contentRangeEnd = -1;
 
 boolean mustUseProxy = FALSE;  /* User must use proxy 305 error*/
 char *proxyLocation = NULL;
 boolean mustUseProxyAuth = FALSE;  /* User must use proxy authentication 407 error*/
 
+if (byteRangeUsed)
+    {
+    parseByteRange(url, &byteRangeStart, &byteRangeEnd, FALSE);
+    }
+
 while(TRUE)
     {
     i = 0;
     while (TRUE)
 	{
 	nread = read(sd, &c, 1);  /* one char at a time, but http headers are small */
 	if (nread != 1)
 	    {
 	    if (nread == -1)
-    		warn("Error (%s) reading http header on %s\n", strerror(errno), url);
+    		warn("Error (%s) reading http header on %s", strerror(errno), url);
 	    else if (nread == 0)
-    		warn("Error unexpected end of input reading http header on %s\n", url);
+    		warn("Error unexpected end of input reading http header on %s", url);
 	    else
-    		warn("Error reading http header on %s\n", url);
+    		warn("Error reading http header on %s", url);
 	    return FALSE;  /* err reading descriptor */
 	    }
 	if (c == 10)
 	    break;
 	if (c != 13)
     	    buf[i++] = c;
 	if (i >= maxbuf)
 	    {
 	    warn("http header line too long > %d chars.",maxbuf);
 	    return FALSE;
 	    }
 	}
     buf[i] = 0;  /* add string terminator */
 
     if (sameString(line,""))
 	{
 	break; /* End of Header found */
 	}
     if (startsWith("HTTP/", line))
         {
 	char *version, *code;
 	version = nextWord(&line);
 	code = nextWord(&line);
 	if (code == NULL)
 	    {
-	    warn("Strange http header on %s\n", url);
+	    warn("Strange http header on %s", url);
 	    return FALSE;
 	    }
 	if (startsWith("30", code) && isdigit(code[2])
 	    && ((code[2] >= '0' && code[2] <= '3') || code[2] == '7') && code[3] == 0)
 	    {
 	    redirect = TRUE;
 	    }
 	else if (sameString(code, "305"))
 	    {
 	    mustUseProxy = TRUE;
 	    }
 	else if (sameString(code, "407"))
 	    {
 	    mustUseProxyAuth = TRUE;
 	    }
 	else if (byteRangeUsed)
 	    {
 	    if (!sameString(code, "206"))
 		{
 		if (sameString(code, "200"))
 		    warn("Byte-range request was ignored by server. ");
-		warn("Expected Partial Content 206. %s: %s %s\n", url, code, line);
+		warn("Expected Partial Content 206. %s: %s %s", url, code, line);
 		return FALSE;
 		}
 	    }
 	else if (!sameString(code, "200"))
 	    {
-	    warn("Expected 200 %s: %s %s\n", url, code, line);
+	    warn("Expected 200 %s: %s %s", url, code, line);
 	    return FALSE;
 	    }
 	line = buf;  /* restore it */
 	}
     headerName = line;
     sep = strchr(line,':');
     if (sep)
 	{
 	*sep = 0;
 	headerVal = skipLeadingSpaces(++sep);
 	}
     else
 	{
 	headerVal = NULL;
 	}
     if (sameWord(headerName,"Location"))
 	{
 	if (redirect)
 	    *redirectedUrl = cloneString(headerVal);
 	if (mustUseProxy)
 	    proxyLocation = cloneString(headerVal);
 	}
+    if (sameWord(headerName,"Content-Range"))
+	{
+	if (byteRangeUsed)
+	    {
+	    foundContentRange = TRUE;
+	    parseContentRange(headerVal, &contentRangeStart, &contentRangeEnd);	
+    	    if ((contentRangeStart != byteRangeStart) ||
+		(byteRangeEnd != -1 && (contentRangeEnd != byteRangeEnd)))
+		{
+		char bre[256];
+		safef(bre, sizeof bre, "%lld", (long long)byteRangeEnd);
+		if (byteRangeEnd == -1)
+		    bre[0] = 0;
+		warn("Found Content-Range: %s. Expected bytes %lld-%s. Improper caching of 206 reponse byte-ranges?",
+		    headerVal, (long long) byteRangeStart, bre);
+    		return FALSE;
+		}
+	    }
+	}
     }
 if (mustUseProxy ||  mustUseProxyAuth)
     {
-    warn("%s: %s error. Use Proxy%s. Location = %s\n", url, 
+    warn("%s: %s error. Use Proxy%s. Location = %s", url, 
 	mustUseProxy ? "" : " Authentication", 
 	mustUseProxy ? "305" : "407", 
 	proxyLocation ? proxyLocation : "not given");
     return FALSE;
     }
+if (byteRangeUsed && !foundContentRange)
+    {
+    char bre[256];
+    safef(bre, sizeof bre, "%lld", (long long)byteRangeEnd);
+    if (byteRangeEnd == -1)
+	bre[0] = 0;
+    warn("Expected response header Content-Range: %lld-%s", (long long) byteRangeStart, bre);
+    return FALSE;
+    }
+
 return TRUE;
 }
 
 
 boolean netSkipHttpHeaderLinesHandlingRedirect(int sd, char *url, int *redirectedSd, char **redirectedUrl)
 /* Skip http headers lines, returning FALSE if there is a problem.  Generally called as
  *    netSkipHttpHeaderLine(sd, url, &sd, &url);
  * where sd is a socket (file) opened with netUrlOpen(url), and url is in dynamic memory.
  * If the http header indicates that the file has moved, then it will update the *redirectedSd and
  * *redirectedUrl with the new socket and URL, first closing sd.
  * If for some reason you want to detect whether the forwarding has occurred you could
  * call this as:
  *    char *newUrl = NULL;
  *    int newSd = 0;
  *    netSkipHttpHeaderLine(sd, url, &newSd, &newUrl);