a89c648b36bf7ae11587c2d83f375cb8431afb0a galt Thu Mar 31 01:54:40 2011 -0700 adding progress bar showing filesize, progress, total time, and avg download rate in MB/sec. diff --git src/lib/net.c src/lib/net.c index e169195..1149afa 100644 --- src/lib/net.c +++ src/lib/net.c @@ -6,30 +6,31 @@ #include "common.h" #include #include #include #include #include "internet.h" #include "errabort.h" #include "hash.h" #include "net.h" #include "linefile.h" #include "base64.h" #include "cheapcgi.h" #include "https.h" #include "sqlNum.h" #include +#include "obscure.h" static char const rcsid[] = "$Id: net.c,v 1.80 2010/04/14 07:42:06 galt Exp $"; /* Brought errno in to get more useful error messages */ extern int errno; static int netStreamSocket() /* Create a TCP/IP streaming socket. Complain and return something * negative if can't */ { int sd = socket(AF_INET, SOCK_STREAM, 0); if (sd < 0) warn("Couldn't make AF_INET socket."); return sd; @@ -1554,42 +1555,45 @@ unlink(outStat); return FALSE; } *pPcList = pcList; *pUrl = url; *pFileSize = fileSize; *pDateString = dateString; *pTotalDownloaded = totalDownloaded; return TRUE; } -boolean parallelFetch(char *url, char *outPath, int numConnections, int numRetries, boolean newer) +boolean parallelFetch(char *url, char *outPath, int numConnections, int numRetries, boolean newer, boolean progress) /* Open multiple parallel connections to URL to speed downloading */ { char *origPath = outPath; char outTemp[1024]; safef(outTemp, sizeof(outTemp), "%s.paraFetch", outPath); outPath = outTemp; /* get the size of the file to be downloaded */ off_t fileSize = 0; off_t totalDownloaded = 0; ssize_t sinceLastStatus = 0; char *dateString = ""; +int star = 1; +int starMax = 20; +int starStep = 1; // TODO handle case-sensitivity of protocols input if (startsWith("http://",url) || startsWith("https://",url)) { struct hash *hash = newHash(0); int status = netUrlHead(url, hash); if (status != 200) // && status != 302 && status != 301) { warn("Error code: %d, expected 200 for %s, can't proceed, sorry", status, url); return FALSE; } char *sizeString = hashFindValUpperCase(hash, "Content-Length:"); if (sizeString) { fileSize = atoll(sizeString); } @@ -1619,30 +1623,31 @@ struct tm *ts; char ftpTime[80]; /* Format the time "Tue, 15 Jun 2010 06:45:08 GMT" */ ts = localtime(&t); strftime(ftpTime, sizeof(ftpTime), "%a, %d %b %Y %H:%M:%S %Z", ts); dateString = cloneString(ftpTime); } else { warn("unrecognized protocol: %s", url); return FALSE; } + verbose(2,"fileSize=%lld\n", (long long) fileSize); if (fileSize < 65536) /* special case small file */ numConnections = 1; if (numConnections > 50) /* ignore high values for numConnections */ { warn("Currently maximum number of connections is 50. You requested %d. Will proceed with 50 on %s", numConnections, url); numConnections = 50; } verbose(2,"numConnections=%d\n", numConnections); //debug if (numConnections < 1) { @@ -1733,60 +1738,81 @@ { AllocVar(pc); pc->next = NULL; pc->rangeStart = base; base += partSize; pc->partSize = partSize; if (fileSize != -1 && pc->rangeStart+pc->partSize >= fileSize) pc->partSize = fileSize - pc->rangeStart; pc->received = 0; pc->sd = -4; /* no connection tried yet */ slAddHead(&pcList, pc); } slReverse(&pcList); } +if (progress) + { + char nicenumber[1024]=""; + sprintWithGreekByte(nicenumber, sizeof(nicenumber), fileSize); + printf("downloading %s ", nicenumber); fflush(stdout); + starStep = fileSize/starMax; + if (starStep < 1) + starStep = 1; + } + int out = open(outPath, O_CREAT|O_WRONLY, 0664); if (out < 0) { warn("Unable to open %s for write while downloading %s, can't proceed, sorry", url, outPath); return FALSE; } /* descriptors for select() */ fd_set rfds; struct timeval tv; int retval; ssize_t readCount = 0; #define BUFSIZE 65536 * 4 char buf[BUFSIZE]; /* create paraFetchStatus right away for monitoring programs */ writeParaFetchStatus(origPath, pcList, url, fileSize, dateString, FALSE); sinceLastStatus = 0; int retryCount = 0; +time_t startTime = time(NULL); + #define SELTIMEOUT 5 #define RETRYSLEEPTIME 30 while (TRUE) { verbose(2,"Top of big loop\n"); + if (progress) + { + while (totalDownloaded >= star * starStep) + { + printf("*");fflush(stdout); + ++star; + } + } + /* are we done? */ if (connOpen == 0) { boolean done = TRUE; for(pc = pcList; pc; pc = pc->next) if (pc->sd != -1) done = FALSE; if (done) break; } /* See if we need to open any connections, either new or retries */ for(pc = pcList; pc; pc = pc->next) { if ((pc->sd == -4) /* never even tried to open yet */ || ((reOpen>0) /* some connections have been released */ @@ -2013,30 +2039,49 @@ ut.actime = mystat.st_atime; ut.modtime = t; if (utime(outTemp, &ut)==-1) { char errMsg[256]; safef(errMsg, sizeof(errMsg), "paraFetch: error setting modification time of %s to %s\n", outTemp, dateString); perror(errMsg); } } } } /* rename the successful download to the original name */ rename(outTemp, origPath); + + +if (progress) + { + while (star <= starMax) + { + printf("*");fflush(stdout); + ++star; + } + long timeDiff = (long)(time(NULL) - startTime); + if (timeDiff > 0) + { + printf(" %ld seconds", timeDiff); + float mbpersec = ((totalDownloaded - restartTotalDownloaded)/1000000) / timeDiff; + printf(" %0.1f MB/sec", mbpersec); + } + printf("\n");fflush(stdout); + } + if (fileSize != -1 && totalDownloaded != fileSize) { warn("Unexpected result: Total downloaded bytes %lld is not equal to fileSize %lld" , (long long) totalDownloaded , (long long) fileSize); return FALSE; } return TRUE; } struct lineFile *netLineFileOpen(char *url) /* Return a lineFile attached to url. This one * will skip any headers. Free this with * lineFileClose(). */