b5e17d3a60b11576d512133081db32f598fa8432
wong
  Thu Oct 20 16:03:31 2011 -0700
renamed qaInit.py to qaInit
diff --git python/programs/qaInit/qaInit python/programs/qaInit/qaInit
new file mode 100755
index 0000000..45200c2
--- /dev/null
+++ python/programs/qaInit/qaInit
@@ -0,0 +1,417 @@
+#!/hive/groups/encode/dcc/bin/python
+import sys, os, re, argparse, subprocess, math, datetime
+from ucscgenomics import ra, track
+
+def getMethods(qaDir, args, user, d):
+	cmd = "curl -X GET http://genomewiki.cse.ucsc.edu/genecats/index.php/ENCODE_QA 2>/dev/null"
+	p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+	output = p.stdout.read().split("\n")
+	r = re.compile('<li class="toclevel.*="tocnumber">(\S+)<.*="toctext">(.*)<\/span>')
+	if os.path.exists(qaDir + '/methods'):
+		return
+	
+	f = open(qaDir + '/methods', 'w')
+	f.write(args.composite + "\n")
+	f.write(args.database + "\n")
+	f.write("Release " + args.release + "\n")
+	f.write(str(d) + "\n")
+	f.write(user + "\n")
+	f.write("\n")
+	for line in output:
+		
+		m = r.match(line)
+		if m:
+			tabs = ""
+			for char in m.group(1):
+				if char == ".":
+					tabs = tabs + "    "
+			if tabs == "":
+				f.write("\n")
+			f.write("%s%s %s\n" % (tabs, m.group(1), m.group(2)))
+	return
+
+def writeClaimMail(args, user, qaDir):
+	f = open(qaDir + '/claimMail', 'w')
+	mail = """Hi Kate,
+
+I am claiming the %s (Release %s) %s track.
+Please let me know if there is another track you'd like me to work on instead.
+
+%s
+""" % (args.composite, args.release, args.database, user)
+	
+	f.write(mail)
+	f.close()
+	return
+	
+def makeLinks(c, args, qaDir):
+	notesfile = c._notesDirectory + "%s.release%s.notes" % (args.composite, args.release)
+	downloads = c.downloadsDirectory + "release%s" % args.release
+	
+	if os.path.exists(qaDir + "/notes.file"):
+		os.remove(qaDir + "/notes.file")
+	os.symlink(notesfile, qaDir + "/notes.file")
+	
+	if os.path.exists(qaDir + "/downloads"):
+		os.remove(qaDir + "/downloads")
+	os.symlink(downloads, qaDir + "/downloads")
+	
+	return notesfile
+
+def parseNotes(lines):
+	tables = set()
+	gbdbs = set()
+	files = set()
+	supplemental = set()
+	others = set()
+	size = ""
+	switch = 0
+	p1 = re.compile('New Tables')
+	p2 = re.compile('New Download Files')
+	p3 = re.compile('New Gbdb Files')
+	p4 = re.compile('New Supplemental Files')
+	p5 = re.compile('New Other Files')
+	s = re.compile('Total: (\d+) MB')
+	e = re.compile('^$')
+	for i in lines:
+		if p1.search(i):
+			switch = 1
+			continue
+		if p2.search(i):
+			switch = 2
+			continue
+		if p3.search(i):
+			switch = 3
+			continue
+		if p4.search(i):
+			switch = 4
+			continue
+		if p5.search(i):
+			switch = 5
+			continue
+		
+		sm = s.match(i)
+		if sm:
+			size = sm.group(1)
+			
+		if e.match(i):
+			switch = 0
+			#print "empty line"	
+			
+		if switch == 1:
+			tables.add(i)
+		if switch == 2:
+			files.add(i)
+		if switch == 3:
+			gbdbs.add(i)
+		if switch == 4:
+			supplemental.add(i)
+		if switch == 5:
+			others.add(i)
+			
+	return tables, gbdbs, files, supplemental, others, str(size)
+
+def writeTableMail(tables, args, user, qaDir):
+	
+	if tables:
+		sep = ""
+		tablestr = sep.join(list(sorted(tables)))
+		mail = """Hi Pushers,
+
+For the *%s* database (2 part request):
+
+1) Please push trackDb and friends
+
+2) Please push these %s tables:
+
+%s
+
+from mysqlbeta -> mysqlrr
+
+Reason: releasing ENCODE %s track on %s to the RR http://redmine.soe.ucsc.edu/issues/%s
+
+Thank you!
+
+%s
+""" % (args.database, len(tables), tablestr, args.composite, args.database, args.redmine, user)
+	else:
+		mail = """Hi Pushers,
+
+For the *%s* database:
+
+1) Please push trackDb and friends
+
+from mysqlbeta -> mysqlrr
+
+Reason: releasing _____(fill in metadata or trackDb changes)____ for ENCODE %s track on %s to the RR
+
+Thank you!
+
+%s
+""" % (args.database, args.composite, args.database, user)
+
+	f = open(qaDir + "/pushTableMail", "w")
+	f.write(mail)
+	f.close
+	
+	f = open(qaDir + "/tableList", "w")
+	f.write(tablestr)
+	f.close
+	
+	return tablestr
+	
+def writeGbdbMail(gbdbs, args, user, qaDir):
+	sep = ""
+	gbdbstr = sep.join(list(sorted(gbdbs)))
+	mail = """
+Hi Pushers,
+
+Please push these %s gbdb files:
+
+%s
+
+\tfrom hgwdev --> hgnfs1
+
+Reason: staging %s %s track on beta, http://redmine.soe.ucsc.edu/issues/%s
+
+Thank You!
+
+%s
+""" % (len(gbdbs), gbdbstr, args.composite, args.database, args.redmine, user)
+	f = open(qaDir + "/pushGbdbsMail", "w")
+	f.write(mail)
+	f.close()
+	return gbdbstr
+	
+def writeFileMail(files, others, args, user, qaDir, c):
+	sep = ""
+	filestr = sep.join(list(sorted(files | others)))
+	mail = """Hi Pushers,
+
+Please push these %s files:
+
+%s
+
+from hgwdev to hgdownload:
+%s
+
+Please note the destination on hgdownload is *one directory above the location on dev*
+
+Reason: releasing ENCODE %s on %s to the RR http://redmine.soe.ucsc.edu/issues/%s
+
+Thanks!
+
+%s
+""" % (len(files), filestr, c._rrHttpDir, args.composite, args.database, args.redmine, user)
+
+	f = open(qaDir + "/pushFilesMail", "w")
+	f.write(mail)
+	f.close()
+	subIds = dict()
+	mdb = c.alphaMetaDb
+	p = re.compile('.*(wgE.*)')
+	for i in files:
+		m = p.match(i)
+		file = m.group(1)
+		filestanza = mdb.filter(lambda s: re.match(".*%s.*" % file,s['fileName']), lambda s: s)
+		if filestanza:
+			for j in filestanza:
+				if 'subId' in j:
+					if not j['subId'] in subIds:
+						subIds[j['subId']] = 1
+            
+	return filestr, set(sorted(subIds.keys()))
+
+def writeHtml(args, c, qaDir):
+	f = open(c._trackDbPath, "r")
+	lines = f.readlines()
+	f.close
+	short = ""
+	long = ""
+	for i in lines:
+		m = re.match('((long|short)Label)\s+(.*)', i)
+		
+		if m:
+			if m.group(1) == 'longLabel':
+				long = m.group(3)
+			if m.group(1) == 'shortLabel':
+				short = m.group(3)
+				
+	html = """<TR>
+        <TD><A HREF="http://hgdownload.cse.ucsc.edu/goldenPath/%s/encodeDCC/%s/"
+        TARGET=_BLANK>%s</A></TD>
+        <TD>%s</TD>
+</TR>""" % (args.database, args.composite, short, long)
+
+	f = open(qaDir + "/htmlDownloadSnippet", "w")
+	f.write(html)
+	f.close()
+	return short, long
+	
+def runScript(args, qaDir):
+	cmd = "qaEncodeTracks.csh %s %s/tableList 2>&1" % (args.database, qaDir)
+	p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+	output = p.stdout.read()
+	f = open(qaDir + "/script.output", "w")
+	sep = ""
+	outstr = sep.join(output)
+	f.write(outstr)
+	f.close()
+	return
+    
+def writeSql(tablestr, filestr, gbdbstr, d, short, long, args, notes, size, user, qaDir):
+	newYN = "Y"
+	if int(args.release) > 1:
+		newYN = "N"
+		
+	sql = """tbls={
+%s
+}
+files={
+%s
+}
+sizeMB={
+%s
+}
+currLoc={
+hgwbeta
+}
+reviewer={
+%s
+}
+initdate={
+%s
+}
+lastdate={
+%s
+}
+track={
+%s
+}
+releaseLog={
+%s
+}
+priority={
+L
+}
+dbs={
+%s
+}
+newYN={
+%s
+}
+rank={
+0
+}
+ndxYN={
+Y
+}
+joinerYN={
+X
+}
+makeDocYN={
+X
+}
+openIssues={
+http://redmine.soe.ucsc.edu/issues/%s
+}
+bounces={
+0
+}
+notes={
+%s
+}
+releaseLogUrl={
+../../cgi-bin/hgTrackUi?db=%s&g=%s
+}
+pushState={
+D
+}""" % (tablestr, filestr + "\n" + gbdbstr, size, user, str(d), str(d), short, long, args.database, newYN, args.redmine, notes, args.database, args.composite) 
+	f = open(qaDir + "/release.sql", "w")
+	f.write(sql)
+	f.close()
+	return
+    
+def writePushHtmlMail(args, user, qaDir):
+
+	downloads = 'downloads.html'
+	if re.match('mm\S+', args.database):
+		downloads = downloadsMouse.html
+	
+	mail = """Hi Pushers,
+
+Please push the following file:
+
+/usr/local/apache/htdocs/ENCODE/%s
+
+from hgwbeta --> RR.
+Reason: added the newly released ENCODE %s %s
+
+Thanks!
+
+%s
+""" % (downloads, args.composite, args.database, user)
+
+def changeStatus(subIds):
+	for i in subIds:
+		cmd = "encodeStatus.pl %s reviewing 2>&1" % (i)
+		p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+		output = p.stdout.read()
+	return
+
+def main():
+	parser = argparse.ArgumentParser(
+        prog='qaInit',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='Initializes QA directory for claiming a release',
+        epilog=
+"""Example:
+qaInir hg19 wgEncodeSydhTfbs 1 69
+
+"""
+        )
+	parser.add_argument('database', help='The database, typically hg19 or mm9')
+	parser.add_argument('composite', help='The composite name, wgEncodeCshlLongRnaSeq for instance')
+	parser.add_argument('release', help='The new release to be released')
+	parser.add_argument('redmine', help='The Redmins issue number')
+
+
+	args = parser.parse_args(sys.argv[1:])
+	user = os.environ['USER']
+	c = track.CompositeTrack(args.database, args.composite)
+	qaDir = c.qaInitDir + "release%s" % args.release 
+	d = datetime.date.today()
+	if not os.path.exists(qaDir):
+		os.makedirs(qaDir)
+		
+	getMethods(qaDir, args, user, d)
+	
+	makeLinks(c, args, qaDir)
+	notesFile = makeLinks(c, args, qaDir)
+	writeClaimMail(args, user, qaDir)
+	
+	m = re.match('.*(kent/.*)', notesFile)
+	notes = m.group(1)
+	f = open(notesFile, "r")
+	lines = f.readlines()
+	if not re.match('mkChangeNotes v2', lines[0]):
+		print "notes files is not the correct version"
+		return
+	(tables, gbdbs, files, supplemental, others, size) = parseNotes(lines)
+	
+	tablestr = writeTableMail(tables, args, user, qaDir)
+	gbdbstr = writeGbdbMail(gbdbs, args, user, qaDir)
+	(filestr, subIds) = writeFileMail(files, supplemental | others, args, user, qaDir, c)
+	(short, long) = writeHtml(args, c, qaDir)
+	
+	writePushHtmlMail(args, user, qaDir)
+	writeSql(tablestr, filestr, gbdbstr, d, short, long, args, notes, size, user, qaDir)
+	
+	changeStatus(subIds)
+	
+	runScript(args, qaDir)
+
+
+
+if __name__ == "__main__":
+	main()
\ No newline at end of file
