a945138a550db38e817ea9fe9da16096e0f0ab45 braney Thu Oct 1 08:31:35 2026 -0700 ottoMonitor: remember the ticket it files, and add the ottoWatchers people to every ticket, refs #38101, #38436 fileTicket() now returns the new ticket number, and both callers store it, so the "ticket is already open" check works and a failure that persists is not filed again every night. The number is cleared when the job or track is back to ok. A new "# ottoWatchers:" header line in ottoOwners.tsv names people to add as watchers to every ticket. diff --git src/hg/utils/otto/ottoMonitor/ottoMonitor.py src/hg/utils/otto/ottoMonitor/ottoMonitor.py index fb8687aee0f..7bc99c80e40 100755 --- src/hg/utils/otto/ottoMonitor/ottoMonitor.py +++ src/hg/utils/otto/ottoMonitor/ottoMonitor.py @@ -276,30 +276,39 @@ fields = line.split("\t") if len(fields) < wanted: continue rows[fields[0]] = fields return(rows, comments) def findOnDuty(comments): """The ottoOnDuty header line names whoever is currently running otto.""" for line in comments: if line.lower().startswith("# ottoonduty:"): return(line.split(":", 1)[1].strip()) return(None) +def findWatchers(comments): + """The ottoWatchers header line names people added to every ticket, comma + separated.""" + for line in comments: + if line.lower().startswith("# ottowatchers:"): + return([n.strip() for n in line.split(":", 1)[1].split(",") if n.strip()]) + return([]) + + def loadState(path): if os.path.exists(path): with open(path) as fh: return(json.load(fh)) return({}) def saveState(path, state): os.makedirs(os.path.dirname(path), exist_ok=True) tmp = path + ".tmp" with open(tmp, "w") as fh: json.dump(state, fh, indent=2, sort_keys=True) os.rename(tmp, path) @@ -536,33 +545,34 @@ since = devDate.strftime("%Y-%m-%d %H:%M") entry["behindSince"] = since entry["publicSeen"] = pubText result["behindSince"] = since behind = now - datetime.strptime(since, "%Y-%m-%d %H:%M") if behind < timedelta(days=row["lagDays"]): result["verdict"] = "ok" return(result) result["verdict"] = "notPublic" result["detail"] = ("genome.ucsc.edu has data from %s, hgwdev has %s, " "behind since %s (%d days)" % (pubText, devText, since, behind.days)) return(result) -def fileTicket(result, onDuty, dryRun): +def fileTicket(result, onDuty, watchers, dryRun): """One GB Bug per failing job, to whoever is running otto, with the job's - owner as a watcher and named in the body.""" + owner and the ottoWatchers people as watchers, and the owner named in the + body.""" owner = result["owner"] if result["verdict"] == "notPublic": subject = "otto job %s: %s on genome.ucsc.edu is older than on hgwdev" % ( result["job"], result["name"]) else: subject = "otto job %s has not run since %s" % (result["job"], result["lastRun"]) if result.get("ownerIsOnDuty"): ownerLine = "This job has no individual owner, so it belongs to whoever is running otto." elif owner != "?": ownerLine = "The recorded owner of this job is %s." % owner else: ownerLine = "This job has no recorded owner in ottoOwners.tsv." if result["verdict"] == "notPublic": body = "\n".join([ "The otto failure monitor found that this track has not reached the " @@ -584,144 +594,154 @@ "", "Schedule: %s" % result["cron"], "Last run stamp: %s" % result["lastRun"], "Due: %s" % result.get("due", "-"), "Source check: %s" % result.get("probe", "-"), ]) cmd = [redmineCli, "create", "--project", "genomebrowser", "--tracker", "Bug", "--subject", subject, "--description", body] if onDuty: cmd += ["--assigned-to", onDuty] if dryRun: print(" would file a ticket, run with --file to do it: %s" % subject) return(None) done = subprocess.run(cmd, capture_output=True, text=True) print(done.stdout.strip()) - # dict.fromkeys keeps the order and drops the duplicate when the job's owner - # is the person on duty - for name in dict.fromkeys(n for n in (owner, onDuty) if n and n != "?"): - ticketId = "".join(c for c in done.stdout.split("#")[-1][:6] if c.isdigit()) - if ticketId: + # redmineCli prints "Created #NNNNN: " + match = re.search(r"Created #(\d+)", done.stdout) + if not match: + return(None) + ticketId = match.group(1) + # dict.fromkeys keeps the order and drops a name that is in the list twice, + # such as an owner who is also the person on duty + for name in dict.fromkeys(n for n in [owner, onDuty] + watchers if n and n != "?"): subprocess.run([redmineCli, "watch", ticketId, name], capture_output=True, text=True) - return(done.stdout.strip()) + return(ticketId) def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--owners", default=defaultOwners, help="ottoOwners.tsv") parser.add_argument("--stamps", default=defaultStamps, help="ottoMonitorStamps.tsv") parser.add_argument("--state", default=defaultState, help="state.json") parser.add_argument("--file", action="store_true", help="file a ticket for a real failure. Off by default") parser.add_argument("--job", help="check one job and say everything about it") parser.add_argument("-v", "--verbose", action="store_true", help="report the jobs that are fine and the ones we are blind to") parser.add_argument("--no-state", action="store_true", help="do not read or write state.json") parser.add_argument("--public", default=defaultPublic, help="ottoMonitorPublic.tsv") parser.add_argument("--no-public", action="store_true", help="skip the comparison of hgwdev with genome.ucsc.edu") args = parser.parse_args() owners, comments = readTable(args.owners, 10) stamps, _ = readTable(args.stamps, 3) publicRows = [] onDuty = findOnDuty(comments) + watchers = findWatchers(comments) now = datetime.now() state = {} if args.no_state else loadState(args.state) watched = [j for j, row in owners.items() if row[4] == "yes" and (args.job is None or j == args.job)] if args.job and not watched: sys.exit("no watched job named %s" % args.job) late, blind, other, fine = [], [], [], [] for job in sorted(watched): result = checkJob(job, owners, stamps, now, onDuty) if result["verdict"] == "late": result = classifyLate(result) entry = state.setdefault(job, {}) previous = entry.get("verdict") if result["verdict"] in ("sourceDown", "realFailure"): entry["strikes"] = entry.get("strikes", 0) + 1 if previous == result["verdict"] else 1 late.append(result) else: entry["strikes"] = 0 entry["verdict"] = result["verdict"] entry["lastRun"] = result.get("lastRun", "-") entry["checked"] = now.strftime("%Y-%m-%d %H:%M") result["strikes"] = entry["strikes"] if result["verdict"] == "blind": blind.append(result) elif result["verdict"] in ("unlisted", "unparsed"): other.append(result) elif result["verdict"] == "ok": + entry.pop("ticket", None) fine.append(result) for result in late: print("%s: %s" % (result["job"], result["detail"])) print(" source: %s" % result.get("probe", "-")) print(" owner: %s strike %d" % (result["owner"], result["strikes"])) # a source that is down fixes itself overnight often enough that one bad # night should not become a ticket if result["verdict"] == "sourceDown" and result["strikes"] < 2: print(" source is down, waiting for a second strike before filing") continue entry = state.get(result["job"], {}) if entry.get("ticket"): print(" ticket #%s is already open" % entry["ticket"]) continue - fileTicket(result, onDuty, dryRun=not args.file) + ticketId = fileTicket(result, onDuty, watchers, dryRun=not args.file) + if ticketId: + entry["ticket"] = ticketId for result in other: print("%s: %s" % (result["job"], result["detail"])) # The public-site check. Tracks found in /gbdb, plus the few that # ottoMonitorPublic.tsv lists because /gbdb cannot find them. Its state # lives under one key of its own, keyed by db and track, because one job # can feed several tracks. if not args.no_public: seen = set() for row in readPublicTable(args.public) + discoverPublicTracks(owners, watched): if row["job"] in watched and (row["db"], row["track"]) not in seen: seen.add((row["db"], row["track"])) publicRows.append(row) publicState = state.setdefault("_publicSite", {}) publicFine, publicAbsent = [], [] for row in publicRows: entry = publicState.setdefault("%s.%s" % (row["db"], row["track"]), {}) result = checkPublic(row, entry, now) entry["checked"] = now.strftime("%Y-%m-%d %H:%M") if result["verdict"] == "ok": + entry.pop("ticket", None) publicFine.append(result) continue if result["verdict"] == "notOnPublic": publicAbsent.append(result) continue print("%s: %s: %s" % (row["job"], result["name"], result["detail"])) if result["verdict"] != "notPublic": continue owner = owners[row["job"]][1] result["ownerIsOnDuty"] = owner == onDutyOwner result["owner"] = (onDuty or "?") if result["ownerIsOnDuty"] else owner print(" owner: %s" % result["owner"]) if entry.get("ticket"): print(" ticket #%s is already open" % entry["ticket"]) continue - fileTicket(result, onDuty, dryRun=not args.file) + ticketId = fileTicket(result, onDuty, watchers, dryRun=not args.file) + if ticketId: + entry["ticket"] = ticketId if args.verbose: print("\nblind, cannot tell whether these ran (%d):" % len(blind)) for result in blind: print(" %-20s %s" % (result["job"], result["detail"])) print("\non time (%d):" % len(fine)) for result in fine: print(" %-20s last run %s, due %s" % (result["job"], result.get("lastRun", "-"), result.get("due", "-"))) if publicRows: print("\npublic site up to date (%d):" % len(publicFine)) for result in publicFine: waiting = "" if result.get("behindSince"): waiting = ", behind since %s, within %g days" % (