b6c2af4cdead913626b042ed3334f2f81809cfac
wong
  Thu Feb 9 13:53:26 2012 -0800
changed handling of attic objects, and also objects that changed states between old and new for tables and gbdbs
diff --git python/lib/ucscgenomics/mkChangeNotes.py python/lib/ucscgenomics/mkChangeNotes.py
index fcc5a3b..443b1c4 100644
--- python/lib/ucscgenomics/mkChangeNotes.py
+++ python/lib/ucscgenomics/mkChangeNotes.py
@@ -259,56 +259,64 @@
 
         nicesize = self.__determineNiceSize(size)
         if nicesize:
             output.append("%s: %d MB (%s)" % (type, size, nicesize))
         else:
             output.append("%s: %d MB" % (type, size))
         totalsize = totalsize + size
 
         return (output, totalsize, str(size))
 
     def __printSection(self, new, untouched, revoked, all, title, path, summary):
         output = []
         removeline = "Revoked/Replaced/Renamed"
         totaline = "Total (New + Untouched + Revoked/Replaced/Renamed)"
         caps = title.upper()
-        if title == "supplemental":
+        change = set()
+        if title == "tables":
+            change = self.changedTables
+        elif title == "supplemental":
             removeline = "Removed"
             totaline = "Total"
             title = title + " files"
             caps = title.upper()
         elif title == 'gbdbs':
             caps = "GBDBS"
             title = "gbdb files"
+            change = self.changedGbdbs
         elif title == "download":
             title = title + " files"
             caps = title.upper()
         if all:
             output.append("")
             output.append("%s:" % caps)
             output.append("New: %s" % len(new))
             output.append("Untouched: %s" % len(untouched))
             output.append("%s: %s" % (removeline, len(revoked)))
             output.append("New + Untouched: %s" % len(new | untouched))
             output.append("%s: %s" % (totaline, len(all)))
             intersect = new & revoked
             if intersect:
                 output.append("")
                 output.append("These %s objects exist in both new and revoked %s:" % (len(intersect), title))
                 for i in intersect:
                     output.append("%s" % i)
-            
+            if change:
+                output.append("")
+                output.append("These %s objects changed state between public and alpha" % len(change))
+                for i in change:
+                    output.append("%s" % i)
         if all and not summary:
             output.append("")
             output.append("New %s (%s):" % (title.title(), len(new)))
             output.extend(ucscUtils.printIter(new, path))
             output.append("")
             output.append("Untouched %s (%s):" % (title.title(), len(untouched)))
             output.extend(ucscUtils.printIter(untouched, path))
             output.append("")
             output.append("%s %s (%s):" % (removeline, title.title(), len(revoked)))
             output.extend(ucscUtils.printIter(revoked, path))
         if all:
             output.append("")
         return output
 
     def __qaHeader(self, output, newTableSet, filesNoRevoke, newGbdbSet, newSupp, additionalList, revokedTables, revokedFiles, revokedGbdbs, pushFiles, pushGbdbs, args, c):
@@ -416,35 +424,42 @@
             output.append("New: %s" % len(additionalList))
             output.append("Revoked/Replace: %s" % len(removedOther))
             output.append("Total: %s" % len(allOther))
         if otherprint and not args['summary']:
             output.append("")
             output.append("New Other Files (%s):" % len(additionalList))
             output.extend(sorted(list(self.newOthers)))
             output.append("")
             output.append("Revoked Other Files (%s):" % len(removedOther))
             output.extend(ucscUtils.printIter((removedOther), self.releasePath))
         output.append("\n")
 
         output.extend(self.__addMissingToReport(missingFiles, "Files", self.releasePathOld))
         output.append("\n")
         output.extend(self.__addMissingToReport(self.droppedTables, "Tables"))
-        output.extend("\n")
+        output.append("\n")
         if self.atticSet:
-            output.append("Attic Objects")
-            output.extend(ucscUtils.printIter((self.atticSet), self.releasePath))
-
+            if self.newInAttic:
+                output.append("New Attic Objects (%s):" % len(self.newInAttic))
+                output.extend(ucscUtils.printIter((self.newInAttic)))
+            if self.stillInAttic:
+                output.append("Untouched Attic Objects (%s):" % len(self.stillInAttic))
+                output.extend(ucscUtils.printIter((self.stillInAttic)))
+            if self.noMoreAttic:
+                output.append("Untouched Attic Objects (%s):" % len(self.noMoreAttic))
+                output.extend(ucscUtils.printIter((self.noMoreAttic))) 
+            output.append("\n")
         if not args['ignore']:
             output.append("No Errors")
         else:
             output.append("The counts here were generated by ignoring errors, they may not be correct")
 	return output
 
     def __printSectionOne(self, output, set, title):
         output = []
         if set:
             output.append("%s (%s):" % (title, len(set)))
             output.extend(sorted(list(set)))
         else:
             return output
         output.append("\n")
         return output
@@ -553,63 +568,75 @@
             self.oldMdb = c.publicMetaDb
      
            
             if self.verbose >= 2:
                 sys.stderr.write("Checking for missing files\n")
             #make a list of missing files
             self.missingFiles = self.__checkFilesForDropped()
             #filter them out of old release files
 
 
 
             if self.verbose >= 1:
                 sys.stderr.write("Scanning and parsing release directories\n")
             #check if all files listed in release directories have associated metaDb entries
             (self.newMdb, self.revokedSet, self.revokedFiles, self.atticSet, self.newSupplementalSet, newFileErrors) = self.checkMetaDbForFiles("alpha metaDb", "new")
-            (self.oldMdb, spam, eggs, ham, self.oldSupplementalSet, oldFileErrors) = self.checkMetaDbForFiles("public metaDb", "old")
+            (self.oldMdb, spam, eggs, self.oldAtticSet, self.oldSupplementalSet, oldFileErrors) = self.checkMetaDbForFiles("public metaDb", "old")
 
             self.expIds = set(self.newMdb.filter(lambda s: 'expId' in s, lambda s: s['expId']))
 
             if self.verbose >= 2:
                 sys.stderr.write("Checking for attic files\n")
             #check that attic fiels aren't in trackDb
             if self.trackDb:
                 errors.extend(self.__checkAtticNotInTrackDb())
 
 
 
             #checks to see that nothing has disappeared between public and alpha
             if self.verbose >= 1:
                 sys.stderr.write("Checking new metaDb for missing stanzas\n")
             errors.extend(self.__checkAlphaForDropped("alpha metaDb", "stanza"))
             if self.verbose >=1:
                 sys.stderr.write("Checking file md5sums across releases\n")
             errors.extend(self.__checkMd5sums())
 
             #checks and gets tables that are present, also returns a revoked set of tables for new
             if self.verbose >= 1:
                 sys.stderr.write("Checking table status\n")
             (self.newTableSet, self.revokedTableSet, self.newMissingTables, newTableError) = self.checkTableStatus("alpha metaDb", "new")
             (self.oldTableSet, spam, self.droppedTables, oldTableError) = self.checkTableStatus("public metaDb", "old")
 
+            
+
+            self.newInAttic = self.atticSet - self.oldAtticSet
+            self.stillInAttic = self.oldAtticSet & self.atticSet
+            self.oldTableSet = self.oldTableSet - self.atticSet
+            self.noMoreAttic = self.oldAtticSet - self.atticSet
+
+            self.changedTables = self.oldTableSet - self.newTableSet
+
+
             #same as above except for gbdbs
             if self.verbose >= 1:
                 sys.stderr.write("Checking GBDB status\n")
             (self.newGbdbSet, self.revokedGbdbs, newGbdbError) = self.getGbdbFiles("new")
             (self.oldGbdbSet, eggs, oldGbdbError) = self.getGbdbFiles("old")
             #remove missing files from gbdbs
             self.oldGbdbSet = self.oldGbdbSet - self.missingFiles
+            self.oldGbdbSet = self.oldGbdbSet - self.atticSet
+            self.changedGbdbs = self.oldGbdbSet - self.newGbdbSet
             for i in self.missingFiles:
                 if i in self.oldReleaseFiles:
                     del self.oldReleaseFiles[i]
 
             #fill in the errors
             errors.extend(newFileErrors)
             errors.extend(oldFileErrors)
             errors.extend(newTableError)
             errors.extend(oldTableError)
             errors.extend(newGbdbError)
             errors.extend(oldGbdbError)
 
             #for ease of typing
             totalFiles = set(self.newReleaseFiles)
             oldTotalFiles = set(self.oldReleaseFiles)
