04d357c39c6ba5ec5899b5341803942a75864c26 jnavarr5 Mon Jun 29 13:18:17 2026 -0700 Fix MLQ automation by updating retired Sonnet model ID to claude-sonnet-4-6 and alerting the QA team on non-retryable Claude API errors instead of silently logging them as overloaded. No RM. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> diff --git src/utils/qa/mlqAutomate.py src/utils/qa/mlqAutomate.py index f5c081908fe..d8cdf471928 100755 --- src/utils/qa/mlqAutomate.py +++ src/utils/qa/mlqAutomate.py @@ -292,31 +292,31 @@ SPAM: [YES or NO] CATEGORY: [Pick one from: {categories_list}] DRAFT_RESPONSE: [If not spam, write a helpful, professional response under 200 words. If spam, write "N/A"] Important: - Mark as SPAM if it is: - Conference/journal solicitations asking for paper submissions - Promotions for workshops, courses, training programs, or webinars - Marketing or promotional emails advertising services or products - Mass-sent announcements unrelated to genome browser support{medical_rule} - Mark as NOT SPAM if it is a genuine question about using the UCSC Genome Browser (general genetics questions without personal identifying info are OK) - For CATEGORY, pick the most specific match. Use "Other" if unsure. - For DRAFT_RESPONSE, be helpful and concise. Ask clarifying questions if needed. Point to relevant documentation when appropriate.""" response = client.messages.create( - model="claude-sonnet-4-20250514", + model="claude-sonnet-4-6", max_tokens=800, messages=[{"role": "user", "content": prompt}] ) result_text = response.content[0].text.strip() # Parse the response is_spam = False category = "Other" draft_response = None for line in result_text.split('\n'): line = line.strip() if line.upper().startswith('SPAM:'): is_spam = 'YES' in line.upper() @@ -382,31 +382,31 @@ - Contains sensitive personal medical information (specific names with genetic test results, medical conditions, family medical history, or personal health details) - these are privacy concerns Mark as NOT SPAM if it is: - A genuine question about the UCSC Genome Browser - A technical support request - A follow-up to an existing conversation - Someone asking how to use browser features for their research - General questions about genetic data without personal identifying information Reply with one line per email in this exact format: EMAIL 1: SPAM or NOT SPAM EMAIL 2: SPAM or NOT SPAM (etc.)""" response = client.messages.create( - model="claude-sonnet-4-20250514", + model="claude-sonnet-4-6", max_tokens=100, messages=[{"role": "user", "content": prompt}] ) result_text = response.content[0].text.strip().upper() results = {} for line in result_text.split('\n'): line = line.strip() match = re.match(r'EMAIL\s*(\d+):\s*(SPAM|NOT SPAM)', line) if match: idx = int(match.group(1)) - 1 # Convert to 0-based index is_spam = match.group(2) == 'SPAM' results[idx] = is_spam @@ -650,30 +650,61 @@ message['Subject'] = f'[MLQ Automation Error] {subject}' encoded = base64.urlsafe_b64encode(message.as_bytes()).decode('utf-8') service.users().messages().send( userId='me', body={'raw': encoded} ).execute() logger.info(f"Sent error notification email: {subject}") return True except Exception as e: logger.error(f"Failed to send error notification email: {e}") return False +# Set once per run when a non-retryable Claude API error is reported, so a broken +# model ID or credential alerts the QA team once per run instead of once per email. +_claude_fatal_alerted = False + + +def handle_claude_api_error(e, context): + """Log a Claude API failure and alert the QA team on non-transient errors. + + Transient errors (429 rate limit, 5xx, connection/timeout) are logged for + retry on the next run. Other 4xx client errors -- an invalid or retired model + ID, bad auth, a malformed request -- won't resolve on their own, so they also + send an email alert to the QA team (at most once per run). The caller still + skips the current item either way. + """ + global _claude_fatal_alerted + + status = getattr(e, 'status_code', None) + is_fatal = status is not None and 400 <= status < 500 and status not in (408, 409, 429) + + if is_fatal: + message = (f"Claude API client error (HTTP {status}) {context}. This will not " + f"resolve on retry and needs investigation. Error: {e}") + logger.error(message) + if not _claude_fatal_alerted: + send_error_notification(f"Claude API client error (HTTP {status})", message) + _claude_fatal_alerted = True + else: + logger.error(f"Claude API unavailable after retries {context}. " + f"Will be retried next run. Error: {e}") + + def delete_moderation_email(gmail_id): """Delete/archive the moderation notification after processing.""" if DRY_RUN: return creds = get_google_credentials() service = build('gmail', 'v1', credentials=creds, cache_discovery=False) try: service.users().messages().trash(userId='me', id=gmail_id).execute() except Exception as e: logger.error(f" Error trashing notification: {e}") def get_emails_from_gmail(group_email, minutes_ago=60): """Get recent emails sent to a mailing list.""" @@ -898,31 +929,31 @@ # Bottom-posted reply: quote header near the start means new content # is below the quoted block. Skip the quoted block, keep what follows. # Allow at most 1 non-blank line above (e.g., a greeting like "Hi,") non_blank_above = sum(1 for l in cleaned if l.strip()) if non_blank_above <= 1: # Skip past the quote header i += quote_header_lines # Skip the quoted lines (starting with ">") while i < len(lines): l = lines[i] l_stripped = unicode_control_pattern.sub('', l).strip() if l_stripped.startswith('>') or l_stripped == '': i += 1 else: break - # Reset cleaned — anything before the quote header was blank/trivial + # Reset cleaned -- anything before the quote header was blank/trivial cleaned = [] continue else: # Top-posted reply: we already have real content above, stop here break if re.match(r'^-{4,}\s*Original Message\s*-{4,}', line_stripped, re.IGNORECASE): break # Gmail forwarded message format: "---------- Forwarded message ---------" if re.match(r'^-{4,}\s*Forwarded message\s*-{4,}', line_stripped, re.IGNORECASE): break # Outlook single-line forward format if re.match(r'^From:.*Sent:.*To:', line_stripped, re.IGNORECASE): break # Outlook/Apple Mail multi-line forward format: @@ -1240,31 +1271,31 @@ 'limit': 100, } headers = {'X-Redmine-API-Key': CONFIG['REDMINE_API_KEY']} resp = requests.get(url, params=params, headers=headers, timeout=30) resp.raise_for_status() data = resp.json() email_list = [e.lower() for e in thread_emails] has_staff_participant = any(e.endswith('@ucsc.edu') for e in email_list) for issue in data.get('issues', []): if normalize_subject(issue['subject']).lower() != normalized.lower(): continue - # Staff replies to mailing list threads don't need email match — + # Staff replies to mailing list threads don't need email match -- # subject match is sufficient since staff wouldn't start a new # unrelated thread with the same subject. The placeholder subject is # shared by all no-subject threads, so it's excluded from this shortcut: # require an email match to avoid merging unrelated conversations. if has_staff_participant and normalized != NO_SUBJECT: return issue['id'] # For external senders, require email match to avoid false positives # on generic subjects email_field = next( (f for f in issue.get('custom_fields', []) if f['id'] == CONFIG['CUSTOM_FIELDS']['Email']), None ) if email_field and email_field.get('value'): @@ -1667,32 +1698,31 @@ continue ticket_status = ticket.get('status', {}).get('name', '').lower() is_closed = 'closed' in ticket_status or 'resolved' in ticket_status comment = f"--- New Email Update ---\n\nFrom: {sender}\n\n{processed_body}" update_ticket(existing_ticket, comment, reopen=is_closed, new_mlm=mlm_name if is_closed else None, attachments=uploaded_attachments) else: # Analyze with Claude for category and draft response try: analysis = analyze_email_with_claude(subject, body, sender, group_email=group_email) except anthropic.APIError as e: - logger.error(f"Anthropic API overloaded after retries, skipping email " - f"'{subject[:50]}'. Will be retried next run. Error: {e}") + handle_claude_api_error(e, f"analyzing email '{subject[:50]}'") continue logger.info(f" Category: {analysis['category']}") # Create new ticket with attachments ticket_id = create_ticket( subject, processed_body, sender_emails, mlm_name, category=analysis['category'], attachments=uploaded_attachments ) if ticket_id and analysis['draft_response']: @@ -1711,32 +1741,31 @@ pending = get_pending_moderation_emails(group_name) logger.info(f" Found {len(pending)} pending message(s)") for msg in pending: msg['group_email'] = group_email all_pending.append(msg) if not all_pending: return # Batch spam check all pending messages in one API call logger.info(f"Batch checking {len(all_pending)} message(s) for spam") try: spam_results = batch_check_spam_with_claude(all_pending) except anthropic.APIError as e: - logger.error(f"Anthropic API overloaded after retries, skipping spam check this run. " - f"Pending messages will be retried in the next run. Error: {e}") + handle_claude_api_error(e, "during the moderation spam check") return # Process results and collect approved messages approved_messages = [] for i, msg in enumerate(all_pending): is_spam = spam_results.get(i, False) if is_spam: logger.info(f" SPAM detected (not approving): {msg['original_subject'][:50]}") delete_moderation_email(msg['gmail_id']) else: logger.info(f" Approving: {msg['original_subject'][:50]}") if moderate_message(msg, approve=True): delete_moderation_email(msg['gmail_id']) approved_messages.append(msg) @@ -1847,32 +1876,31 @@ reopen = is_closed and first_update update_ticket(existing_ticket, comment, reopen=reopen, new_mlm=mlm_name if reopen else None, attachments=uploaded_attachments) first_update = False else: # Analyze email with Claude (single call for spam, category, draft) try: analysis = analyze_email_with_claude( first_email['subject'], first_email['body'], first_email['from'], group_email=thread['group'] ) except anthropic.APIError as e: - logger.error(f"Anthropic API overloaded after retries, skipping email " - f"'{first_email['subject'][:50]}'. Will be retried next run. Error: {e}") + handle_claude_api_error(e, f"analyzing email '{first_email['subject'][:50]}'") continue if analysis['is_spam']: logger.info(f"Skipping spam: {first_email['subject'][:50]}") continue logger.info(f" Category: {analysis['category']}") # Upload attachments from the first email first_attachments = first_email.get('attachments', []) uploaded_attachments = [] if first_attachments: logger.info(f" Uploading {len(first_attachments)} attachment(s)") uploaded_attachments = upload_attachments_to_redmine(first_attachments)