e1fad71224c026ac70b99665aa0bf61bacfe2451 wong Fri Dec 9 11:57:14 2011 -0800 added verbose messagings to mkChange Notes diff --git python/lib/ucscgenomics/mkChangeNotes.py python/lib/ucscgenomics/mkChangeNotes.py index 0c0b566..6c4452f 100644 --- python/lib/ucscgenomics/mkChangeNotes.py +++ python/lib/ucscgenomics/mkChangeNotes.py @@ -492,90 +492,108 @@ output.append("\n") if self.droppedTables: output.extend(self.__addMissingToReport(self.droppedTables, "Tables")) return output def __init__(self, args): self.releaseNew = args['releaseNew'] self.releaseOld = args['releaseOld'] self.database = args['database'] self.composite = args['composite'] self.loose = args['loose'] self.ignore = args['ignore'] self.summary = args['summary'] self.specialMdb = args['specialMdb'] self.args = args + if 'verbose' in args: + self.verbose = args['verbose'] + else: + self.verbose = 0 errors = [] c = track.CompositeTrack(self.database, self.composite, None, self.specialMdb) #sanitize arguments if not self.releaseOld.isdigit(): self.releaseOld = 'solo' elif int(self.releaseOld) <= 0: self.releaseOlf = 'solo' elif self.releaseOld > self.releaseNew: self.releaseOld = 'solo' - + if self.verbose >= 1: + sys.stderr.write("Initializing MkChangeNotes\n") self.releasePath = c.httpDownloadsPath + 'release' + args['releaseNew'] self.gbdbPath = "/gbdb/%s/bbi" % args['database'] self.trackDbFile = c.currentTrackDb if not self.trackDbFile: self.trackDb = None errors.append("track: There is no entry in trackDb.wgEncode.ra for %s with the alpha tag" % self.composite) else: self.trackDb = ra.RaFile(self.trackDbFile) if int(self.releaseNew) > 1 and str(self.releaseOld) != 'solo': - + if self.verbose >= 2: + sys.stderr.write("Comparison mode\n") self.newReleaseFiles = c.releases[int(self.releaseNew)-1] self.oldReleaseFiles = c.releases[int(self.releaseOld)-1] self.releasePathOld = c.httpDownloadsPath + 'release' + args['releaseOld'] self.newMdb = c.alphaMetaDb self.oldMdb = c.publicMetaDb - + if self.verbose >= 2: + sys.stderr.write("Checking for missing files\n") #make a list of missing files self.missingFiles = self.__checkFilesForDropped() #filter them out of old release files - + if self.verbose >= 1: + sys.stderr.write("Scanning and parsing release directories\n") #check if all files listed in release directories have associated metaDb entries (self.newMdb, self.revokedSet, self.revokedFiles, self.atticSet, self.newSupplementalSet, newFileErrors) = self.checkMetaDbForFiles("alpha metaDb", "new") (self.oldMdb, spam, eggs, ham, self.oldSupplementalSet, oldFileErrors) = self.checkMetaDbForFiles("public metaDb", "old") self.expIds = set(self.newMdb.filter(lambda s: 'expId' in s, lambda s: s['expId'])) + if self.verbose >= 2: + sys.stderr.write("Checking for attic files\n") #check that attic fiels aren't in trackDb if self.trackDb: errors.extend(self.__checkAtticNotInTrackDb()) #checks to see that nothing has disappeared between public and alpha + if self.verbose >= 1: + sys.stderr.write("Checking new metaDb for missing stanzas\n") errors.extend(self.__checkAlphaForDropped("alpha metaDb", "stanza")) + if self.verbose >=1: + sys.stderr.write("Checking file md5sums across releases\n") errors.extend(self.__checkMd5sums()) #checks and gets tables that are present, also returns a revoked set of tables for new + if self.verbose >= 1: + sys.stderr.write("Checking table status\n") (self.newTableSet, self.revokedTableSet, self.newMissingTables, newTableError) = self.checkTableStatus("alpha metaDb", "new") (self.oldTableSet, spam, self.droppedTables, oldTableError) = self.checkTableStatus("public metaDb", "old") #same as above except for gbdbs + if self.verbose >= 1: + sys.stderr.write("Checking GBDB status\n") (self.newGbdbSet, self.revokedGbdbs, newGbdbError) = self.getGbdbFiles("new") (self.oldGbdbSet, eggs, oldGbdbError) = self.getGbdbFiles("old") #remove missing files from gbdbs self.oldGbdbSet = self.oldGbdbSet - self.missingFiles for i in self.missingFiles: if i in self.oldReleaseFiles: del self.oldReleaseFiles[i] #fill in the errors errors.extend(newFileErrors) errors.extend(oldFileErrors) errors.extend(newTableError) errors.extend(oldTableError) errors.extend(newGbdbError) errors.extend(oldGbdbError) @@ -596,30 +614,32 @@ self.pushFiles = set(sorted((self.totalFiles - self.oldTotalFiles))) self.pushGbdbs = set(sorted((self.newGbdbSet - self.oldGbdbSet))) self.newSupp = self.newSupplementalSet - self.oldSupplementalSet self.newTables = set(self.pushTables) self.newFiles = set(ucscUtils.printIter(self.pushFiles, self.releasePath)) self.newGbdbs = set(ucscUtils.printIter(self.pushGbdbs, self.gbdbPath)) self.newSupplemental = set(ucscUtils.printIter(self.newSupp, self.releasePath)) self.newOthers = set(ucscUtils.printIter(self.additionalList, self.releasePath)) self.fullFiles = sorted(self.totalFiles - self.revokedFiles) self.fullTables = self.oldTableSet & self.newTableSet self.errors = errors #don't output.append(report unless ignore option is on or no errors #module mode doesn't generate output by default + if self.verbose >= 1: + sys.stderr.write("Creating report\n") if (not errors) or self.ignore: self.output = self.printReport(args, c) else: self.output = self.printErrors(errors, self.missingFiles) elif self.releaseOld == 'solo': self.newReleaseFiles = c.releases[int(self.releaseNew)-1] self.oldReleaseFiles = set() self.newMdb = c.alphaMetaDb #check that attic fiels aren't in trackDb if self.trackDb: