99cadc77d83b20928b0eb1f8e6bb1d7641b134a8
wong
  Mon Oct 17 16:56:08 2011 -0700
added in solo release functionality, changed how certain things are diaplayed, loose mode for md5sum check
diff --git python/programs/mkChangeNotes/mkChangeNotes python/programs/mkChangeNotes/mkChangeNotes
index d480801..56dd1e6 100755
--- python/programs/mkChangeNotes/mkChangeNotes
+++ python/programs/mkChangeNotes/mkChangeNotes
@@ -156,39 +156,41 @@
     tablelist = list()
     for i in mdbtableset:
         tablelist.append("table_name = '%s'" % i)
     orsep = " OR "
     orstr = orsep.join(tablelist)
 
     cmd = "hgsql %s -e \"SELECT ROUND(data_length/1024/1024,2) total_size_mb, ROUND(index_length/1024/1024,2) total_index_size_mb FROM information_schema.TABLES WHERE table_name = %s\"" % (database, orstr)
     p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
     output = p.stdout.read()
     for i in output.split("\n")[1:]:
         fields = i.split()
         for j in fields:
             tablesize = tablesize + float(j)
     return math.ceil(tablesize)
 
-def checkMd5sums(newfiles, oldfiles):
+def checkMd5sums(newfiles, oldfiles, loose):
     errors = []
     for i in oldfiles:
         if i not in newfiles:
             pass
         elif re.match('wgEncode.*', i):
             if oldfiles[i].md5sum != newfiles[i].md5sum:
                 errors.append("file: %s have changed md5sums between releases. %s vs %s" % (i, oldfiles[i].md5sum, newfiles[i].md5sum))
-
+    if loose:
+        return list()
+    else:
     return errors
 
 def makeFileSizes(c, args, inlist):
     checklist = list()
     for i in inlist:
         checklist.append("%s/%s" % (c.downloadsDirectory + 'release' + args.releaseNew, i))    
     filesizes = 0
     for i in checklist:
         filesizes = filesizes + int(os.path.getsize(i))
     filesizes = filesizes / (1024**2)
         
     return filesizes
 
 def cleanSpecialFiles(inlist):
     specialRemoveList = ['md5sum.history']
@@ -229,62 +231,62 @@
     for i in sorted(set):
         print "%s/%s" % (c.httpDownloadsPath + 'release' + release, i)
         
 def printGbdbPath(set, database):
     for i in sorted(set):
         print "/gbdb/%s/bbi/%s" % (database, i)
 
 def printIter(inlist):
     for i in sorted(inlist):
         print i
 
 def printReport(args, totalFiles, newGbdbSet, newTableSet, additionalList, oldAdditionalList, pushTables, pushFiles, pushGbdbs, c, oldTableSet, oldReleaseFiles, oldGbdbSet, atticSet, revokedFiles, mdb, revokedTableSet, revokedGbdbs, missingFiles, newSupplementalSet, oldSupplementalSet, tableSize):
     #the groups here need to be predefined, I just copied and pasted after working out what they were
     sep = "\n"
     print "mkChangeNotes v2"
-    print "%s %s Release %s" % (args.database, args.composite, args.releaseNew)    
+    print "%s %s Release %s vs Release %s" % (args.database, args.composite, args.releaseNew, args.releaseOld)    
     print "" 
     print "QA Count Summaries for Release %s:" % args.releaseNew
     print "Tables: %d" % int(len(newTableSet))
     print "Files: %d" % int(len(totalFiles - revokedFiles))
     print "Gbdbs: %d" % int(len(newGbdbSet))
     print "Supplemental: %d" % int(len(newSupplementalSet - oldSupplementalSet))
     print "Other: %d" % int(len(additionalList))
     print "\n"
     totalsize = 0;
     print "Sizes of New:"
     tableGb = int(tableSize/1024)
     if tableGb > 1:
         print "Tables: %d MB (%d GB)" % (tableSize, tableGb)
     else:
         print "Tables: %d MB" % tableSize
     totalsize = totalsize + tableSize
     size = int(makeFileSizes(c, args, pushFiles))
     totalsize = totalsize + size
     if int(size/1024) > 1:
         print "Files: %d MB (%d GB)" % (size, int(size/1024))
     else:
         print "Files: %d MB" % size
     size = int(makeFileSizes(c, args, pushGbdbs))
     totalsize = totalsize + size
     if int(size/1024) > 1:
         print "Gbdbs: %d MB (%d GB)" % (size, int(size/1024))
     else:
         print "Gbdbs: %d MB" % size
     size = int(makeFileSizes(c, args, (newSupplementalSet - oldSupplementalSet)))
     totalsize = totalsize + size
-    if int(size/1-24) > 1:
+    if int(size/1024) > 1:
         print "Supplemental: %d MB" % (size, int(size/1024))
     else:
         print "Supplemental: %d MB" % size
     size = int(makeFileSizes(c, args, (additionalList)))
     totalsize = totalsize + size
     if int(size/1024) > 1:
         print "Other: %d MB" % size
     else:
         print "Other: %d MB" % size
     if int(totalsize/1024) > 1:
         print "Total: %d MB (%d GB)" % (totalsize, int(totalsize/1024))
     else:
         print "Total: %d MB" % totalsize
     print "\n"
     print "TABLES:"
@@ -322,135 +324,148 @@
         print "Revoked/Replaced/Renamed Download Files (%s):" % len(revokedFiles)    
         printWithPath(revokedFiles, c, args.releaseNew)
     print "\n"
     print "GBDBS:"
     print "New: %s" % len(pushGbdbs)
     print "Persisting: %s" % len((newGbdbSet & oldGbdbSet) - revokedGbdbs)
     print "Revoked/Replaced/Renamed: %s" % len(revokedGbdbs)
     print "New + Persisting: %s" % len(pushGbdbs | ((newGbdbSet & oldGbdbSet) - revokedGbdbs))
     print "Total (New + Persisting + Revoked/Replaced/Renamed): %s" % len(newGbdbSet | oldGbdbSet | revokedGbdbs)
     if args.full:
         print ""
         print "New Gbdb Files (%s):" % len(pushGbdbs)
         printGbdbPath(pushGbdbs, args.database)
         print ""
         print "Persisting Gbdb Files (%s):" % len((newGbdbSet & oldGbdbSet) - revokedGbdbs)
-        printGbdbPath(((newGbdbSet & oldGbdbSet) - revokedGbdbs), args.database)
+        printGbdbPath((newGbdbSet & oldGbdbSet) - revokedGbdbs, args.database)
         print ""
         print "Revoked/Replaced/Renamed Gbdb Files (%s):" % len(revokedGbdbs)
         printGbdbPath(revokedGbdbs, args.database)
     print "\n"
-    print "Supplemental Files:"
+    print "SUPPLEMENTAL FILES:"
     print "New: %s" % len(newSupplementalSet - oldSupplementalSet)
     print "Persisting: %s" % len(oldSupplementalSet & newSupplementalSet)
     print "Removed: %s" % len(oldSupplementalSet - newSupplementalSet)
     print "New + Persisting: %s" % len((newSupplementalSet - oldSupplementalSet) | (oldSupplementalSet & newSupplementalSet))
     print "Total: %s" % len(newSupplementalSet | oldSupplementalSet)
     if args.full:
         print ""
-        print "New Supplemental Files:"
+        print "New Supplemental Files (%s):" % len(newSupplementalSet - oldSupplementalSet)
         printWithPath(newSupplementalSet - oldSupplementalSet, c, args.releaseNew)
         print ""
-        print "Persisting Supplemental Files:"
+        print "Persisting Supplemental Files (%s):" % len(oldSupplementalSet & newSupplementalSet)
         printWithPath(oldSupplementalSet & newSupplementalSet, c, args.releaseNew)
         print ""
-        print "Removed Supplemental Files:"
+        print "Removed Supplemental Files (%s):" % len(oldSupplementalSet - newSupplementalSet)
         printWithPath(oldSupplementalSet - newSupplementalSet, c, args.releaseNew)
     print "\n"
     print "OTHER FILES:"
     print "New: %s" % len(additionalList)
     print "Revoked/Replace: %s" % len(oldAdditionalList)
     print "Total: %s" % len(additionalList | oldAdditionalList)
     if args.full:
         print "" 
         print "New Other Files (%s):" % len(additionalList)
         printWithPath(additionalList, c, args.releaseNew)
         print ""
         print "Revoked Other Files (%s):" % len(oldAdditionalList)
         printWithPath(oldAdditionalList, c, args.releaseNew)
     print "\n"
     print "Files that dropped between releases (%s):" % len(missingFiles)
     printWithPath(missingFiles, c, args.releaseOld)
     print "\n"
     if not args.ignore:
         print "No Errors"
     
-def printReportOne(args, totalFiles, newGbdbSet, newTableSet, additionalList, pushTables, c, pushGbdbs, atticSet, newSupplementalSet, tableSize):
+def printReportOne(args, totalFiles, revokedFiles, newGbdbSet, revokedGbdbs, newTableSet, revokedTables, additionalList, c, atticSet, newSupplementalSet, tableSize):
     print "mkChangeNotes v2"
     print "%s %s Release %s" % (args.database, args.composite, args.releaseNew)
     print ""
     print "QA Count Summaries for Release %s:" % args.releaseNew
-    print "Tables: %d" % int(len(newTableSet))
-    print "Files: %d" % int(len(totalFiles))
-    print "Gbdbs: %d" % int(len(newGbdbSet))
+    print "Tables: %d" % int(len(newTableSet - revokedTables))
+    print "Files: %d" % int(len(totalFiles - revokedFiles))
+    print "Gbdbs: %d" % int(len(newGbdbSet - revokedGbdbs))
     print "Supplemental: %d" % int(len(newSupplementalSet))
     print "Other: %d" % int(len(additionalList))
+    print ""
+    print "REVOKED:"
+    print "Tables: %s" % len(revokedTables)
+    print "Files: %s" % len(revokedFiles)
+    print "Gbdbs: %s" % len(revokedGbdbs)
     print "\n"
     totalsize = 0;
     print "Sizes of New:"
     tableGb = int(tableSize / 1024)
     if tableGb > 1:
         print "Tables: %d MB (%d GB)" % (tableSize, tableGb)
     else:
         print "Tables: %d MB" % tableSize
     totalsize = totalsize + tableSize
     size = int(makeFileSizes(c, args, totalFiles))
     totalsize = totalsize + size
     if int(size/1024) > 1:
         print "Files: %d MB (%d GB)" % (size, int(size/1024))
     else:
         print "Files: %d MB" % size
-    size = int(makeFileSizes(c, args, pushGbdbs))
+    size = int(makeFileSizes(c, args, newGbdbSet))
     totalsize = totalsize + size
     if int(size/1024) > 1:
         print "Gbdbs: %d MB (%d GB)" % (size, int(size/1024))
     else:
         print "Gbdbs: %d MB" % size
     size = int(makeFileSizes(c, args, newSupplementalSet))
     totalsize = totalsize + size
-    if int(size/1-24) > 1:
-        print "Supplemental: %d MB" % (size, int(size/1024))
+    if int(size/1024) > 1:
+        print "Supplemental: %d MB (%d GB)" % (size, int(size/1024))
     else:
         print "Supplemental: %d MB" % size
     size = int(makeFileSizes(c, args, (additionalList)))
     totalsize = totalsize + size
     if int(size/1024) > 1:
         print "Other: %d MB" % size
     else:
         print "Other: %d MB" % size
     if int(totalsize/1024) > 1:
         print "Total: %d MB (%d GB)" % (totalsize, int(totalsize/1024))
     else:
         print "Total: %d MB" % totalsize
     print "\n"
     if args.full:
         print ""
-        print "New Tables (%s):" % len(pushTables)
-        printIter(pushTables)
+        print "New Tables (%s):" % len(newTableSet - revokedTables)
+        printIter(newTableSet - revokedTables)
         print "\n"
-        print "New Download Files (%s):" % len(totalFiles)
-        printWithPath(totalFiles, c, args.releaseNew)
+        print "New Download Files (%s):" % len(totalFiles - revokedFiles)
+        printWithPath(totalFiles - revokedFiles, c, args.releaseNew)
         print "\n"
-        print "New Gbdb Files (%s):" % len(pushGbdbs)
-        printWithPath(pushGbdbs, c, args.releaseNew)
+        print "New Gbdb Files (%s):" % len(newGbdbSet - revokedGbdbs)
+        printGbdbPath(newGbdbSet - revokedGbdbs, args.database)
         print "\n"
-        print "New Supplemental Files:"
+        print "New Supplemental Files (%s):" % len(newSupplementalSet)
         printWithPath(newSupplementalSet, c, args.releaseNew)
         print "\n" 
         print "New Other Files (%s):" % len(additionalList)
         printWithPath(additionalList, c, args.releaseNew)
         print "\n"
+        print "Revoked Tables (%s):" %len(revokedTables)
+        printIter(revokedTables)
+        print "Revoked Files (%s):" % len(revokedFiles)
+        printWithPath(revokedFiles, c, args.releaseNew)
+        print "\n"
+        print "Revoked Gbdbs (%s):" % len(revokedGbdbs)
+        printGbdbPath(revokedGbdbs, args.database)
+        print "\n"
     if not args.ignore:
         print "No Errors"
 
 
 def printErrors(errors):
     errorsDict = {}
     for i in errors:
         line = i.split(":", 1)
         try:
             errorsDict[line[0]].append(line[1])
         except:
             errorsDict[line[0]] = []
             errorsDict[line[0]].append(line[1])
     print "Errors (%s):" % len(errors)
     for i in sorted(errorsDict.keys()):
@@ -485,53 +500,56 @@
     if len(sys.argv) == 1:
         parser.print_help()
         return
     args = parser.parse_args(sys.argv[1:])
     if not args.releaseNew.isdigit():
         parser.print_help()
         return
 
     c = track.CompositeTrack(args.database,args.composite)
 
     loose = args.loose
 
     errors = []
     
     if not args.releaseOld.isdigit():
-        args.releaseOld = 0
-    if int(args.releaseOld) > int(args.releaseNew):
+        args.releaseOld = 'solo'    
+    elif int(args.releaseOld) > int(args.releaseNew):
         errors.append("Old Release is higher than New Release")
         args.releaseOld = args.releaseNew
+        printErrors(errors)
+        return
 
-    if int(args.releaseNew) > 1:
+
+    if int(args.releaseNew) > 1 and str(args.releaseOld) != 'solo':
     
         newReleaseFiles = c.releases[int(args.releaseNew)-1]
         oldReleaseFiles = c.releases[int(args.releaseOld)-1]
 
         newMdb = c.alphaMetaDb
         oldMdb = c.publicMetaDb
 
         #check if all files listed in release directories have associated metaDb entries
         (newMdb, revokedSet, revokedFiles, atticSet, newSupplementalSet, newFileErrors) = checkMetaDbForFiles(newMdb, newReleaseFiles, "alpha metaDb", loose)
         (oldMdb, spam, eggs, ham, oldSupplementalSet, oldFileErrors) = checkMetaDbForFiles(oldMdb, oldReleaseFiles, "public metaDb", loose)
         errors.extend(newFileErrors)
         errors.extend(oldFileErrors)
         
         #checks to see that nothing has disappeared between public and alpha
         errors.extend(checkAlphaForDropped(newMdb, oldMdb, "alpha metaDb", "stanza"))
         missingFiles = checkFilesForDropped(newReleaseFiles, oldReleaseFiles)
-        errors.extend(checkMd5sums(newReleaseFiles, oldReleaseFiles))
+        errors.extend(checkMd5sums(newReleaseFiles, oldReleaseFiles, loose))
 
         #checks and gets tables that are present, also returns a revoked set of tables for new
         (newTableSet, revokedTableSet, newTableError) = checkTableStatus(newMdb, newReleaseFiles, args.database, args.composite, "alpha metaDb", loose, revokedSet)
         (oldTableSet, spam, oldTableError) = checkTableStatus(oldMdb, oldReleaseFiles, args.database, args.composite, "public metaDb", loose, revokedSet)
         errors.extend(newTableError)
         errors.extend(oldTableError)
 
         #same as above except for gbdbs
         (newGbdbSet, revokedGbdbs, newGbdbError) = getGbdbFiles(args.database, newTableSet, revokedTableSet, newMdb)
         (oldGbdbSet, eggs, oldGbdbError) = getGbdbFiles(args.database, oldTableSet, set(), oldMdb)
         errors.extend(newGbdbError)
         errors.extend(oldGbdbError)
     
         #check to see if tables were dropped between releases
         droppedTables = oldTableSet - newTableSet
@@ -550,48 +568,48 @@
         (oldReleaseFiles, additionalList, oldAdditionalList, totalFiles) = separateOutAdditional(oldReleaseFiles, totalFiles, newSupplementalSet, oldSupplementalSet)
 
         #get the stuff you need to push, also table sizes        
         pushTables = set(sorted((newTableSet - oldTableSet)))
         tableSize = getTableSize(pushTables, args.database)
         pushFiles = set(sorted((totalFiles - oldReleaseFiles)))
         pushGbdbs = set(sorted((newGbdbSet - oldGbdbSet)))
         
         #don't print report unless ignore option is on or no errors
         if (not errors) or args.ignore:
             printReport(args, totalFiles, newGbdbSet, newTableSet, additionalList, oldAdditionalList, pushTables, pushFiles, pushGbdbs, c, oldTableSet, oldReleaseFiles, oldGbdbSet, atticSet, revokedFiles, newMdb, revokedTableSet, revokedGbdbs, missingFiles, newSupplementalSet, oldSupplementalSet, tableSize)
         else:
             printErrors(errors)
 
 
-    else:
+    elif args.releaseOld == 'solo':
 
         newReleaseFiles = c.releases[int(args.releaseNew)-1]
         
         newMdb = c.alphaMetaDb
         
         (newMdb, revokedSet, revokedFiles, atticSet, newSupplementalSet, newFileErrors) = checkMetaDbForFiles(newMdb, newReleaseFiles, "alpha metaDb", loose)
         errors.extend(newFileErrors)
         
         (newTableSet, revokedTableSet, newTableError) = checkTableStatus(newMdb, newReleaseFiles, args.database, args.composite, "alpha metaDb", loose, revokedSet)
         errors.extend(newTableError)
         
         tableSize = getTableSize(newTableSet, args.database)
 
         (newGbdbSet, revokedGbdbs, newGbdbError) = getGbdbFiles(args.database, newTableSet, revokedTableSet, newMdb)
         errors.extend(newGbdbError)
         
         #set for easy operations
         totalFiles = set(newReleaseFiles)
         
         #clean out special fiels we don't push i.e. md5sum.history
         totalFiles = cleanSpecialFiles(totalFiles)
         
         #makes list for additional files
         (oldReleaseFiles, additionalList, oldAdditionalList, totalFiles) = separateOutAdditional(set(), totalFiles, newSupplementalSet, set())
         if (not errors) or args.ignore:
-            printReportOne(args, totalFiles, newGbdbSet, newTableSet, additionalList, newTableSet, c, newGbdbSet, atticSet, newSupplementalSet, tableSize) 
+            printReportOne(args, totalFiles, revokedFiles, newGbdbSet, revokedGbdbs, newTableSet, revokedTableSet, additionalList, c, atticSet, newSupplementalSet, tableSize) 
         else:
             printErrors(errors)
 
 if __name__ == '__main__':
     main()
 
