a89c648b36bf7ae11587c2d83f375cb8431afb0a
galt
  Thu Mar 31 01:54:40 2011 -0700
adding progress bar showing filesize, progress, total time, and avg download rate in MB/sec.
diff --git src/lib/net.c src/lib/net.c
index e169195..1149afa 100644
--- src/lib/net.c
+++ src/lib/net.c
@@ -6,30 +6,31 @@
 #include "common.h"
 #include <signal.h>
 #include <errno.h>
 #include <string.h>
 #include <sys/time.h>
 #include "internet.h"
 #include "errabort.h"
 #include "hash.h"
 #include "net.h"
 #include "linefile.h"
 #include "base64.h"
 #include "cheapcgi.h"
 #include "https.h"
 #include "sqlNum.h"
 #include <utime.h>
+#include "obscure.h"
 
 static char const rcsid[] = "$Id: net.c,v 1.80 2010/04/14 07:42:06 galt Exp $";
 
 /* Brought errno in to get more useful error messages */
 
 extern int errno;
 
 static int netStreamSocket()
 /* Create a TCP/IP streaming socket.  Complain and return something
  * negative if can't */
 {
 int sd = socket(AF_INET, SOCK_STREAM, 0);
 if (sd < 0)
     warn("Couldn't make AF_INET socket.");
 return sd;
@@ -1554,42 +1555,45 @@
     unlink(outStat);
     return FALSE;
     }
 
 *pPcList = pcList;
 *pUrl = url;
 *pFileSize = fileSize;
 *pDateString = dateString;
 *pTotalDownloaded = totalDownloaded;
 
 return TRUE;
 
 }
 
 
-boolean parallelFetch(char *url, char *outPath, int numConnections, int numRetries, boolean newer)
+boolean parallelFetch(char *url, char *outPath, int numConnections, int numRetries, boolean newer, boolean progress)
 /* Open multiple parallel connections to URL to speed downloading */
 {
 char *origPath = outPath;
 char outTemp[1024];
 safef(outTemp, sizeof(outTemp), "%s.paraFetch", outPath);
 outPath = outTemp;
 /* get the size of the file to be downloaded */
 off_t fileSize = 0;
 off_t totalDownloaded = 0;
 ssize_t sinceLastStatus = 0;
 char *dateString = "";
+int star = 1;  
+int starMax = 20;  
+int starStep = 1;
 // TODO handle case-sensitivity of protocols input
 if (startsWith("http://",url) || startsWith("https://",url))
     {
     struct hash *hash = newHash(0);
     int status = netUrlHead(url, hash);
     if (status != 200) // && status != 302 && status != 301)
 	{
 	warn("Error code: %d, expected 200 for %s, can't proceed, sorry", status, url);
 	return FALSE;
 	}
     char *sizeString = hashFindValUpperCase(hash, "Content-Length:");
     if (sizeString)
 	{
 	fileSize = atoll(sizeString);
 	}
@@ -1619,30 +1623,31 @@
     struct tm  *ts;
     char ftpTime[80];
  
     /* Format the time "Tue, 15 Jun 2010 06:45:08 GMT" */
     ts = localtime(&t);
     strftime(ftpTime, sizeof(ftpTime), "%a, %d %b %Y %H:%M:%S %Z", ts);
     dateString = cloneString(ftpTime);
 
     }
 else
     {
     warn("unrecognized protocol: %s", url);
     return FALSE;
     }
 
+
 verbose(2,"fileSize=%lld\n", (long long) fileSize);
 
 if (fileSize < 65536)    /* special case small file */
     numConnections = 1;
 
 if (numConnections > 50)    /* ignore high values for numConnections */
     {
     warn("Currently maximum number of connections is 50. You requested %d. Will proceed with 50 on %s", numConnections, url);
     numConnections = 50;
     }
 
 verbose(2,"numConnections=%d\n", numConnections); //debug
 
 if (numConnections < 1)
     {
@@ -1733,60 +1738,81 @@
 	{
 	AllocVar(pc);
 	pc->next = NULL;
 	pc->rangeStart = base;
 	base += partSize;
 	pc->partSize = partSize;
 	if (fileSize != -1 && pc->rangeStart+pc->partSize >= fileSize)
 	    pc->partSize = fileSize - pc->rangeStart;
 	pc->received = 0;
 	pc->sd = -4;  /* no connection tried yet */
 	slAddHead(&pcList, pc);
 	}
     slReverse(&pcList);
     }
 
+if (progress)
+    {
+    char nicenumber[1024]="";
+    sprintWithGreekByte(nicenumber, sizeof(nicenumber), fileSize);
+    printf("downloading %s ", nicenumber); fflush(stdout);
+    starStep = fileSize/starMax;
+    if (starStep < 1)
+	starStep = 1;
+    }
+
 int out = open(outPath, O_CREAT|O_WRONLY, 0664);
 if (out < 0)
     {
     warn("Unable to open %s for write while downloading %s, can't proceed, sorry", url, outPath);
     return FALSE;
     }
 
 
 /* descriptors for select() */
 fd_set rfds;
 struct timeval tv;
 int retval;
 
 ssize_t readCount = 0;
 #define BUFSIZE 65536 * 4
 char buf[BUFSIZE];
 
 /* create paraFetchStatus right away for monitoring programs */
 writeParaFetchStatus(origPath, pcList, url, fileSize, dateString, FALSE);
 sinceLastStatus = 0;
 
 int retryCount = 0;
 
+time_t startTime = time(NULL);
+
 #define SELTIMEOUT 5
 #define RETRYSLEEPTIME 30    
 while (TRUE)
     {
 
     verbose(2,"Top of big loop\n");
 
+    if (progress)
+	{
+	while (totalDownloaded >= star * starStep)
+	    {
+	    printf("*");fflush(stdout);
+	    ++star;
+	    }
+	}
+
     /* are we done? */
     if (connOpen == 0)
 	{
 	boolean done = TRUE;
 	for(pc = pcList; pc; pc = pc->next)
 	    if (pc->sd != -1)
 		done = FALSE;
 	if (done) break;
 	}
 
     /* See if we need to open any connections, either new or retries */
     for(pc = pcList; pc; pc = pc->next)
 	{
 	if ((pc->sd == -4)  /* never even tried to open yet */
 	 || ((reOpen>0)      /* some connections have been released */
@@ -2013,30 +2039,49 @@
 	    ut.actime = mystat.st_atime;
 	    ut.modtime = t;
 	    if (utime(outTemp, &ut)==-1)
 		{
 		char errMsg[256];
                 safef(errMsg, sizeof(errMsg), "paraFetch: error setting modification time of %s to %s\n", outTemp, dateString);
 		perror(errMsg);
 		}
 	    }
 	}
     }
 
 /* rename the successful download to the original name */
 rename(outTemp, origPath);
 
+
+
+if (progress)
+    {
+    while (star <= starMax)
+	{
+	printf("*");fflush(stdout);
+	++star;
+	}
+    long timeDiff = (long)(time(NULL) - startTime);
+    if (timeDiff > 0)
+	{
+	printf(" %ld seconds", timeDiff);
+	float mbpersec =  ((totalDownloaded - restartTotalDownloaded)/1000000) / timeDiff;
+	printf(" %0.1f MB/sec", mbpersec);
+	}
+    printf("\n");fflush(stdout);
+    }
+
 if (fileSize != -1 && totalDownloaded != fileSize)
     {
     warn("Unexpected result: Total downloaded bytes %lld is not equal to fileSize %lld"
 	, (long long) totalDownloaded
 	, (long long) fileSize);
     return FALSE;
     }
 return TRUE;
 }
 
 
 struct lineFile *netLineFileOpen(char *url)
 /* Return a lineFile attached to url.  This one
  * will skip any headers.   Free this with
  * lineFileClose(). */