a89c648b36bf7ae11587c2d83f375cb8431afb0a galt Thu Mar 31 01:54:40 2011 -0700 adding progress bar showing filesize, progress, total time, and avg download rate in MB/sec. diff --git src/utils/paraSync/paraSync.c src/utils/paraSync/paraSync.c index 0e3c233..28f4fbe 100644 --- src/utils/paraSync/paraSync.c +++ src/utils/paraSync/paraSync.c @@ -7,44 +7,46 @@ #include "net.h" void usage() /* Explain usage and exit */ { errAbort( "paraSync 1.0\n" "paraSync - uses paraFetch to recursively mirror url to given path\n" "usage:\n" " paraSync {options} N R URL outPath\n" " where N is the number of connections to use\n" " R is the number of retries\n" "options:\n" " -A='ext1,ext2' means accept only files with ext1 or ext2\n" " -newer only download a file if it is newer than the version we already have.\n" + " -progress Show progress of download.\n" ); } static struct optionSpec options[] = { {"A", OPTION_STRING}, {"newer", OPTION_BOOLEAN}, + {"progress", OPTION_BOOLEAN}, {NULL, 0}, }; char *acceptString = NULL; char **acceptExtensions = NULL; int acceptExtensionsCount = 0; -boolean paraSync(int numConnections, int numRetries, struct dyString *url, struct dyString *outPath, boolean newer) +boolean paraSync(int numConnections, int numRetries, struct dyString *url, struct dyString *outPath, boolean newer, boolean progress) /* Fetch given URL, send to stdout. */ { // requirements: // URL must end in / slash // foo must end in / slash if (!endsWith(url->string,"/")) errAbort("URL must end in slash /"); if (!endsWith(outPath->string,"/")) errAbort("outPath must end in slash /"); // create subdir if it does not exist makeDir(outPath->string); struct dyString *dy = netSlurpUrl(url->string); char *p = dy->string; verbose(2,"response=[%s]\n", dy->string); @@ -113,49 +115,49 @@ int saveOutPathSize = outPath->stringSize; dyStringAppend(url, q); dyStringAppend(outPath, q); if (startsWith("ftp:", url->string) && isDirectory) { dyStringAppend(url, "/"); dyStringAppend(outPath, "/"); } // URL found if (isDirectory) { // recursive - if (!paraSync(numConnections, numRetries, url, outPath, newer)) + if (!paraSync(numConnections, numRetries, url, outPath, newer, progress)) result = FALSE; } else // file { // Test accepted extensions if applicable. boolean accepted = (acceptExtensionsCount == 0); int i = 0; for(i=0; istring, outPath->string, numConnections, numRetries, newer)) + if (!parallelFetch(url->string, outPath->string, numConnections, numRetries, newer, progress)) { warn("failed to download %s\n", url->string); // write to a log that this one failed // and try to continue result = FALSE; } } } dyStringResize(url, saveUrlSize); dyStringResize(outPath, saveOutPathSize); } @@ -174,20 +176,20 @@ { acceptExtensionsCount = chopByChar(acceptString, ',', NULL, 0); AllocArray(acceptExtensions, acceptExtensionsCount); chopByChar(acceptString, ',', acceptExtensions, acceptExtensionsCount); verbose(1, "accept-option count: %d\n", acceptExtensionsCount); int i = 0; for(i=0; i