83e926b3090278ebbbd801e0a7e0ef2f98ef0775
galt
  Tue Mar 29 01:55:50 2011 -0700
updating paraFetch and paraSync to support -newer option which only downloads newer files, very useful now that the dates are being preserved
diff --git src/utils/paraSync/paraSync.c src/utils/paraSync/paraSync.c
index f33e9c7..0e3c233 100644
--- src/utils/paraSync/paraSync.c
+++ src/utils/paraSync/paraSync.c
@@ -4,45 +4,47 @@
 #include "dystring.h"
 #include "obscure.h"
 #include "portable.h"
 #include "net.h"
 
 void usage()
 /* Explain usage and exit */
 {
 errAbort(
     "paraSync 1.0\n"
     "paraSync - uses paraFetch to recursively mirror url to given path\n"
     "usage:\n"
     "   paraSync {options} N R URL outPath\n"
     "   where N is the number of connections to use\n"
     "         R is the number of retries\n"
-    "   Options:\n"
+    "options:\n"
     "    -A='ext1,ext2'  means accept only files with ext1 or ext2\n" 
+    "   -newer  only download a file if it is newer than the version we already have.\n"
     );
 }
 
 static struct optionSpec options[] = {
    {"A", OPTION_STRING},
+   {"newer", OPTION_BOOLEAN},
    {NULL, 0},
 };
 
 char *acceptString = NULL;
 char **acceptExtensions = NULL; 
 int acceptExtensionsCount = 0;
 
-boolean paraSync(int numConnections, int numRetries, struct dyString *url, struct dyString *outPath)
+boolean paraSync(int numConnections, int numRetries, struct dyString *url, struct dyString *outPath, boolean newer)
 /* Fetch given URL, send to stdout. */
 {
 // requirements:
 //   URL must end in / slash
 //   foo must end in / slash
 if (!endsWith(url->string,"/"))
     errAbort("URL must end in slash /");
 if (!endsWith(outPath->string,"/"))
     errAbort("outPath must end in slash /");
 // create subdir if it does not exist
 makeDir(outPath->string);
 struct dyString *dy = netSlurpUrl(url->string);
 char *p = dy->string;
 
 verbose(2,"response=[%s]\n", dy->string);
@@ -111,69 +113,55 @@
     int saveOutPathSize = outPath->stringSize;
 
     dyStringAppend(url, q);
     dyStringAppend(outPath, q);
     if (startsWith("ftp:", url->string) && isDirectory)
 	{
 	dyStringAppend(url, "/");
     	dyStringAppend(outPath, "/");
 	}
     
  
     // URL found
     if (isDirectory) 
 	{   
 	// recursive
-	if (!paraSync(numConnections, numRetries, url, outPath))
+	if (!paraSync(numConnections, numRetries, url, outPath, newer))
 	    result = FALSE;
 	}
     else    // file
 	{
 	// Test accepted extensions if applicable.
         boolean accepted = (acceptExtensionsCount == 0);
         int i = 0;
         for(i=0; i<acceptExtensionsCount; ++i) 
 	    {
 	    if (endsWith(q, acceptExtensions[i]))
 		{
 		accepted = TRUE;
 		break;
 		}
 	    }
 	if (accepted)
 	    {
-	    // check to see if it needs download, i.e. file does not exist, or it needs a resume
-	    int restoreSize = outPath->stringSize;
-	    dyStringAppend(outPath, ".paraFetchStatus");
-	    boolean needsDownload = fileExists(outPath->string);
-	    dyStringResize(outPath, restoreSize);
-	    if (!fileExists(outPath->string))
-		needsDownload = TRUE;
-	    if (needsDownload)
-		{
-		if (!parallelFetch(url->string, outPath->string, numConnections, numRetries))
+	    if (!parallelFetch(url->string, outPath->string, numConnections, numRetries, newer))
 		    {
 		    warn("failed to download %s\n", url->string);
 		    // write to a log that this one failed
 		    // and try to continue
 		    result = FALSE;
 		    }
-		else
-		    {
-		    verbose(1,"%s downloaded successfully\n", url->string);
-		    }
-		}
 	    }
 	}
     
     dyStringResize(url, saveUrlSize);
     dyStringResize(outPath, saveOutPathSize);
 
     }
 
 
 return result;
 
 }
 
 int main(int argc, char *argv[])
 /* Process command line. */
@@ -186,20 +174,20 @@
     {
     acceptExtensionsCount = chopByChar(acceptString, ',', NULL, 0);
     AllocArray(acceptExtensions, acceptExtensionsCount);
     chopByChar(acceptString, ',', acceptExtensions, acceptExtensionsCount);
     verbose(1, "accept-option count: %d\n", acceptExtensionsCount);
     int i = 0;
     for(i=0; i<acceptExtensionsCount; ++i) 
 	{
 	verbose(2, "accept-option: %s\n", acceptExtensions[i]);
 	}
     }
 struct dyString *url = dyStringNew(4096);
 struct dyString *outPath = dyStringNew(4096);
 dyStringAppend(url, argv[3]);
 dyStringAppend(outPath, argv[4]);
-if (!paraSync(atoi(argv[1]), atoi(argv[2]), url, outPath))
+if (!paraSync(atoi(argv[1]), atoi(argv[2]), url, outPath, optionExists("newer")))
     exit(1);
 return 0;
 }