b6c2af4cdead913626b042ed3334f2f81809cfac wong Thu Feb 9 13:53:26 2012 -0800 changed handling of attic objects, and also objects that changed states between old and new for tables and gbdbs diff --git python/lib/ucscgenomics/mkChangeNotes.py python/lib/ucscgenomics/mkChangeNotes.py index fcc5a3b..443b1c4 100644 --- python/lib/ucscgenomics/mkChangeNotes.py +++ python/lib/ucscgenomics/mkChangeNotes.py @@ -259,56 +259,64 @@ nicesize = self.__determineNiceSize(size) if nicesize: output.append("%s: %d MB (%s)" % (type, size, nicesize)) else: output.append("%s: %d MB" % (type, size)) totalsize = totalsize + size return (output, totalsize, str(size)) def __printSection(self, new, untouched, revoked, all, title, path, summary): output = [] removeline = "Revoked/Replaced/Renamed" totaline = "Total (New + Untouched + Revoked/Replaced/Renamed)" caps = title.upper() - if title == "supplemental": + change = set() + if title == "tables": + change = self.changedTables + elif title == "supplemental": removeline = "Removed" totaline = "Total" title = title + " files" caps = title.upper() elif title == 'gbdbs': caps = "GBDBS" title = "gbdb files" + change = self.changedGbdbs elif title == "download": title = title + " files" caps = title.upper() if all: output.append("") output.append("%s:" % caps) output.append("New: %s" % len(new)) output.append("Untouched: %s" % len(untouched)) output.append("%s: %s" % (removeline, len(revoked))) output.append("New + Untouched: %s" % len(new | untouched)) output.append("%s: %s" % (totaline, len(all))) intersect = new & revoked if intersect: output.append("") output.append("These %s objects exist in both new and revoked %s:" % (len(intersect), title)) for i in intersect: output.append("%s" % i) - + if change: + output.append("") + output.append("These %s objects changed state between public and alpha" % len(change)) + for i in change: + output.append("%s" % i) if all and not summary: output.append("") output.append("New %s (%s):" % (title.title(), len(new))) output.extend(ucscUtils.printIter(new, path)) output.append("") output.append("Untouched %s (%s):" % (title.title(), len(untouched))) output.extend(ucscUtils.printIter(untouched, path)) output.append("") output.append("%s %s (%s):" % (removeline, title.title(), len(revoked))) output.extend(ucscUtils.printIter(revoked, path)) if all: output.append("") return output def __qaHeader(self, output, newTableSet, filesNoRevoke, newGbdbSet, newSupp, additionalList, revokedTables, revokedFiles, revokedGbdbs, pushFiles, pushGbdbs, args, c): @@ -416,35 +424,42 @@ output.append("New: %s" % len(additionalList)) output.append("Revoked/Replace: %s" % len(removedOther)) output.append("Total: %s" % len(allOther)) if otherprint and not args['summary']: output.append("") output.append("New Other Files (%s):" % len(additionalList)) output.extend(sorted(list(self.newOthers))) output.append("") output.append("Revoked Other Files (%s):" % len(removedOther)) output.extend(ucscUtils.printIter((removedOther), self.releasePath)) output.append("\n") output.extend(self.__addMissingToReport(missingFiles, "Files", self.releasePathOld)) output.append("\n") output.extend(self.__addMissingToReport(self.droppedTables, "Tables")) - output.extend("\n") + output.append("\n") if self.atticSet: - output.append("Attic Objects") - output.extend(ucscUtils.printIter((self.atticSet), self.releasePath)) - + if self.newInAttic: + output.append("New Attic Objects (%s):" % len(self.newInAttic)) + output.extend(ucscUtils.printIter((self.newInAttic))) + if self.stillInAttic: + output.append("Untouched Attic Objects (%s):" % len(self.stillInAttic)) + output.extend(ucscUtils.printIter((self.stillInAttic))) + if self.noMoreAttic: + output.append("Untouched Attic Objects (%s):" % len(self.noMoreAttic)) + output.extend(ucscUtils.printIter((self.noMoreAttic))) + output.append("\n") if not args['ignore']: output.append("No Errors") else: output.append("The counts here were generated by ignoring errors, they may not be correct") return output def __printSectionOne(self, output, set, title): output = [] if set: output.append("%s (%s):" % (title, len(set))) output.extend(sorted(list(set))) else: return output output.append("\n") return output @@ -553,63 +568,75 @@ self.oldMdb = c.publicMetaDb if self.verbose >= 2: sys.stderr.write("Checking for missing files\n") #make a list of missing files self.missingFiles = self.__checkFilesForDropped() #filter them out of old release files if self.verbose >= 1: sys.stderr.write("Scanning and parsing release directories\n") #check if all files listed in release directories have associated metaDb entries (self.newMdb, self.revokedSet, self.revokedFiles, self.atticSet, self.newSupplementalSet, newFileErrors) = self.checkMetaDbForFiles("alpha metaDb", "new") - (self.oldMdb, spam, eggs, ham, self.oldSupplementalSet, oldFileErrors) = self.checkMetaDbForFiles("public metaDb", "old") + (self.oldMdb, spam, eggs, self.oldAtticSet, self.oldSupplementalSet, oldFileErrors) = self.checkMetaDbForFiles("public metaDb", "old") self.expIds = set(self.newMdb.filter(lambda s: 'expId' in s, lambda s: s['expId'])) if self.verbose >= 2: sys.stderr.write("Checking for attic files\n") #check that attic fiels aren't in trackDb if self.trackDb: errors.extend(self.__checkAtticNotInTrackDb()) #checks to see that nothing has disappeared between public and alpha if self.verbose >= 1: sys.stderr.write("Checking new metaDb for missing stanzas\n") errors.extend(self.__checkAlphaForDropped("alpha metaDb", "stanza")) if self.verbose >=1: sys.stderr.write("Checking file md5sums across releases\n") errors.extend(self.__checkMd5sums()) #checks and gets tables that are present, also returns a revoked set of tables for new if self.verbose >= 1: sys.stderr.write("Checking table status\n") (self.newTableSet, self.revokedTableSet, self.newMissingTables, newTableError) = self.checkTableStatus("alpha metaDb", "new") (self.oldTableSet, spam, self.droppedTables, oldTableError) = self.checkTableStatus("public metaDb", "old") + + + self.newInAttic = self.atticSet - self.oldAtticSet + self.stillInAttic = self.oldAtticSet & self.atticSet + self.oldTableSet = self.oldTableSet - self.atticSet + self.noMoreAttic = self.oldAtticSet - self.atticSet + + self.changedTables = self.oldTableSet - self.newTableSet + + #same as above except for gbdbs if self.verbose >= 1: sys.stderr.write("Checking GBDB status\n") (self.newGbdbSet, self.revokedGbdbs, newGbdbError) = self.getGbdbFiles("new") (self.oldGbdbSet, eggs, oldGbdbError) = self.getGbdbFiles("old") #remove missing files from gbdbs self.oldGbdbSet = self.oldGbdbSet - self.missingFiles + self.oldGbdbSet = self.oldGbdbSet - self.atticSet + self.changedGbdbs = self.oldGbdbSet - self.newGbdbSet for i in self.missingFiles: if i in self.oldReleaseFiles: del self.oldReleaseFiles[i] #fill in the errors errors.extend(newFileErrors) errors.extend(oldFileErrors) errors.extend(newTableError) errors.extend(oldTableError) errors.extend(newGbdbError) errors.extend(oldGbdbError) #for ease of typing totalFiles = set(self.newReleaseFiles) oldTotalFiles = set(self.oldReleaseFiles)