567bb801c1502dd59b886e1ed10e76bcaf72cca4
jcasper
  Wed Dec 18 13:33:36 2019 -0800
Adding most recent validation stats to cdwCheckValidation status, per wrangler request (no ticket)

diff --git src/hg/cirm/cdw/cdwCheckValidation/cdwCheckValidation.c src/hg/cirm/cdw/cdwCheckValidation/cdwCheckValidation.c
index 56752d6..5b5cf37 100644
--- src/hg/cirm/cdw/cdwCheckValidation/cdwCheckValidation.c
+++ src/hg/cirm/cdw/cdwCheckValidation/cdwCheckValidation.c
@@ -199,43 +199,56 @@
     
 void printSubmissionStatistics(FILE *f, struct cdwJob *jobList, char *command)
 /* Take a list of jobs and gather stats. Different stats are printed to file f depending on the
  * command given. */
 {
 if (!jobList)
     uglyAbort("There are no jobs on the joblist"); 
 
 // Keep track of all the possible job types. 
 int finJobs = 0, workingJobs = 0, queuedJobs = 0, failedJobs = 0, validFiles = 0; 
 
 if (startsWith("status", command) && gLong) 
     fprintf(f,"Printing file statistics...\n");
 
 // Loop through all jobs and gather stats. 
+struct hash *latestRuns = hashNew(0);
 struct cdwJob *job;
 for (job = jobList;; job = job->next)
     {
     char *prefix = "cdwQaAgent ";
     assert(startsWith(prefix, job->commandLine)); 
     char *fileIdString = job->commandLine + strlen(prefix);
     long long fileId = sqlLongLong(fileIdString);
     if (job->startTime > 0)
 	{
 	if (job->endTime > 0)
 	// Finished jobs. 
 	    {
 	    ++finJobs;
+
+            // Keep track of the most recent job found for this fileId
+            struct hashEl *fileEl = hashStore(latestRuns, fileIdString);
+            if (fileEl->val == NULL)
+                fileEl->val = job;
+            else
+                {
+                struct cdwJob *prevFileJob = (struct cdwJob*) fileEl->val;
+                if (prevFileJob->endTime < job->endTime)
+                    fileEl->val = job;
+                }
+
 	    if (job->returnCode == 0)
 	    // Jobs that passed validation. 
 		{
 		++validFiles; 
 		if (startsWith("status", command) && gLong) 
 		    printValidFileStats(f, fileIdString);
 		}
 	    else
 	    // Jobs that failed validation. 
 		{
 		++failedJobs;
 		if (startsWith("failed", command))
 		    fprintf(f,"File id: %s | Valid: No | Error: %s |\n", fileIdString, job->stderr);  
 		if (startsWith("status", command) && gLong) 
 		   fprintf(f,"File id: %s | Valid: No | Error: %s |\n", fileIdString, job->stderr);  
@@ -263,43 +276,61 @@
 	if (startsWith("status", command) && gLong) 
 	    fprintf(f,"File id: %s | Valid: queued | \n", fileIdString);  
 	}
     if (!job->next) 
 	break;
     }
 
 // Print out overall submission stats. 
 
 if (startsWith("status", command))
     {
     fprintf(f,"Total files:\t%i\n",slCount(jobList));
     fprintf(f,"Finished validation:\t%i\n", finJobs);
     fprintf(f,"Passed validation:\t\e[1;32m%i\e[0m\n",  validFiles);
     fprintf(f,"Failed validation:\t\e[1;31m%i\e[0m\n",  failedJobs);
+
+    int passedMostRecent = 0, failedMostRecent = 0;
+    struct hashEl *file;
+    struct hashCookie cookie = hashFirst(latestRuns);
+    while ((file = hashNext(&cookie)) != NULL)
+        {
+        struct cdwJob *job = (struct cdwJob*) file->val;
+        if (job->returnCode == 0)
+            passedMostRecent++;
+        else
+            failedMostRecent++;
+        }
+    
+    fprintf(f,"Unique files:\t%d\n", passedMostRecent+failedMostRecent);
+    fprintf(f,"Files that passed their most recent run:\t\e[1;32m%d\e[0m\n", passedMostRecent);
+    fprintf(f,"Files that failed their most recent run:\t\e[1;32m%d\e[0m\n", failedMostRecent);
     fprintf(f,"Jobs in progress:\t%i\n", workingJobs);
     fprintf(f,"Jobs queued:\t%i\n", queuedJobs);
     printTimeStats(f, jobList, finJobs);
     if (slCount(jobList) == finJobs)
 	fprintf(f,"Status:\t\e[1;32mCompleted\e[0m\n"); 
     else
 	fprintf(f,"Status:\tIn progress\n"); 
     
     }
 if (startsWith("retry", command))
     fprintf(f,"Started revalidating %i files.\n", failedJobs); 
 if (startsWith("failed", command))
     fprintf(f,"%i files failed validation.\n", failedJobs); 
+
+hashFree(&latestRuns);
 }
     
 
 void waitLoop(int submitId)
 /* Hang out and wait for all files in the submission to pass through cdwQaAgent. */ 
 {
 for (;;)
     {
     struct sqlConnection *conn = sqlConnect("cdw"); 
 
     // Grab all cdwJob entries with the submitId. 
     char query[1024]; 
     sqlSafef(query, sizeof(query), "select * from cdwJob where submitId = '%i';", submitId);
     struct cdwJob *jobList = cdwJobLoadByQuery(conn, query);
     if (!jobList) uglyAbort("The submission has no entries in cdwJob.\nThere are several "