8ea58fa13de2de9f113b4596556d3bb197556480 galt Thu Sep 19 14:46:31 2013 -0700 improving timeout and retry handling based in Jims suggestions diff --git src/lib/paraFetch.c src/lib/paraFetch.c index a713676..48a6355 100644 --- src/lib/paraFetch.c +++ src/lib/paraFetch.c @@ -379,32 +379,36 @@ struct timeval tv; int retval; ssize_t readCount = 0; #define BUFSIZE 65536 * 4 char buf[BUFSIZE]; /* create paraFetchStatus right away for monitoring programs */ paraFetchWriteStatus(origPath, pcList, url, fileSize, dateString, FALSE); sinceLastStatus = 0; int retryCount = 0; time_t startTime = time(NULL); -#define SELTIMEOUT 5 +#define SELTIMEOUT 10 #define RETRYSLEEPTIME 30 +int scaledRetries = numRetries; +if (fileSize != -1) + scaledRetries = round(numRetries * (1+fileSize/1e10) ); +verbose(2,"scaledRetries=%d\n", scaledRetries); while (TRUE) { verbose(2,"Top of big loop\n"); if (interrupt != NULL && (*interrupt)(context)) { verbose(1, "Interrupted paraFetch.\n"); return FALSE; } if (progress) { while (totalDownloaded >= star * starStep) { printf("*");fflush(stdout); ++star; @@ -573,31 +577,31 @@ pc->received += readCount; totalDownloaded += readCount; sinceLastStatus += readCount; if (sinceLastStatus >= 10*1024*1024) { paraFetchWriteStatus(origPath, pcList, url, fileSize, dateString, FALSE); sinceLastStatus = 0; } } } } else { warn("No data within %d seconds for %s", SELTIMEOUT, url); /* Retry ? */ - if (retryCount >= numRetries) + if (retryCount >= scaledRetries) { return FALSE; } else { ++retryCount; /* close any open connections */ for(pc = pcList; pc; pc = pc->next) { if (pc->sd >= 0) { close(pc->sd); verbose(2,"closing descriptor: %d\n", pc->sd); } if (pc->sd != -1)