62c58443da3f62b5ac33c1933e6abc3b249b2ba0 lrnassar Tue Jun 3 15:31:31 2025 -0700 Updating hgTracksTiming for two purposes: 1. Making it take command-line arguments so what we can call it separately for the development machines vs. live servers. The reason for this is that we don't need to spam the genome-alert ML for hgwbeta or dev issues. 2. It was previously writing out very low response times that came from failures like the captcha or not finding hgTracks end to the log file. This was throwing off the usage numbers as low values are good, but in this case they are errors. No RM. diff --git src/utils/qa/hgTracksTiming.py src/utils/qa/hgTracksTiming.py index ce2cb43e407..fc2e325b8bf 100644 --- src/utils/qa/hgTracksTiming.py +++ src/utils/qa/hgTracksTiming.py @@ -1,31 +1,55 @@ # Meant as a substitute for hgTracksRandom # Queries a list of GB servers (crontab set to every 15m) and documents their load time # as well as their status code, if they took too long to load, or if hgTracks display did not # fully load. Alerts with a printed error when a negative condition is encountered. # Each run it reads the list of all observed times and regenerates a table and graphs displaying # the change of server load times over time. Once per month it reports as a reminder to check for abnormalities # If running on a new user, you will need to copy the index.html page from qateam and run the function here once: makeSymLinks(user,save_dir) -import requests, subprocess, time, datetime, getpass, os, urllib3, matplotlib +import requests, subprocess, time, datetime, getpass, os, urllib3, matplotlib, argparse, sys import matplotlib.pyplot as plt import matplotlib.ticker as mticker import matplotlib.dates as mdates -import matplotlib from collections import defaultdict from collections import deque +def parseArgs(): + """ + Parse the command line arguments. + """ + parser = argparse.ArgumentParser(description = __doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + required = parser.add_argument_group('required arguments') + + required.add_argument ("servers", + help = "Which servers to query. Options are: 'all', 'RR' for hgw1/2/Euro/Asia, 'dev' for hgwbeta/dev") + if (len(sys.argv) == 1): + parser.print_usage() + print("\nQueries hgTracks on any of the given servers and notes down the response time. Also graphs the response\n" + \ + "over time. Options are: 'all', 'RR' for hgw1/2/Euro/Asia, 'dev' for hgwbeta/dev\n\n" + \ + + "Example runs:\n" + \ + " hgTracksTiming.py all\n" + \ + " hgTracksTiming.py RR\n" + \ + " hgTracksTiming.py dev\n") + + exit(0) + options = parser.parse_args() + return options + def bash(cmd): """Run the cmd in bash subprocess""" try: rawBashOutput = subprocess.run(cmd, check=True, shell=True,\ stdout=subprocess.PIPE, universal_newlines=True, stderr=subprocess.STDOUT) bashStdoutt = rawBashOutput.stdout except subprocess.CalledProcessError as e: raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) return(bashStdoutt) def makeSymLinks(user,save_dir): if user == 'qateam': bash("ln -sf "+save_dir+"*.html /usr/local/apache/htdocs-genecats/qa/test-results/hgTracksTiming/") bash("ln -sf "+save_dir+"*.png /usr/local/apache/htdocs-genecats/qa/test-results/hgTracksTiming/") else: @@ -238,50 +262,74 @@ problem = True status = "FAIL - hgTracks page loaded, but load time over 15s" else: problem = True status = "FAIL - Got status 200 return, but missing the 'END hgTracks' page string of a successful load" if problem == True: print("Potential problem with Genome Browser server.") print(f"URL: {url} | Status: {response.status_code} | Load Time: {load_time:.3f}s | Check: {status}") print("\nSee the latest timing numbers:") if user == 'qateam': print("https://genecats.gi.ucsc.edu/qa/test-results/hgTracksTiming/") else: print("https://hgwdev.gi.ucsc.edu/~"+user+"/cronResults/hgTracksTiming/") + # Add a check here to make sure we are not writing out bad 200 or captcha failures, but still + # writing out problematic > 15s load times + if load_time > .2: with open(filePath, "a") as file: file.write(f"{today}\t{load_time:.3f}s\t{response.status_code}\n") def main(): #Don't try to display the plot, this is for jupyter matplotlib.use('Agg') # Suppress SSL warnings - was due to an asia issue urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) user = getpass.getuser() save_dir = create_save_dir(user) today = datetime.datetime.today().strftime("%Y-%m-%d-%H:%M") + #Parse which servers to query + options = parseArgs() + servers = options.servers + + if servers == 'all': # Dic with all the URLs to test. To temporarily pause testing of URLs for maintenance, expected outage, etc. # Remove it from this dictionary urls = { "hgwdev": "https://hgwdev.gi.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", "hgwbeta": "https://hgwbeta.soe.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", "hgw1": "https://hgw1.soe.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", "hgw2": "https://hgw2.soe.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", "euro": "https://genome-euro.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", "asia": "https://genome-asia.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1" } + elif servers == 'RR': + urls = { + "hgw1": "https://hgw1.soe.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", + "hgw2": "https://hgw2.soe.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", + "euro": "https://genome-euro.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", + "asia": "https://genome-asia.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1" + } + + elif servers == 'dev': + urls = { + "hgwdev": "https://hgwdev.gi.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", + "hgwbeta": "https://hgwbeta.soe.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1" + } + else: + sys.exit("No server specified. Use either 'all', 'RR', or 'dev' to specify which servers to query") + n=0 for server, url in urls.items(): n+=1 filePath = save_dir + server + ".txt" queryServersAndReport(server,url,filePath,today,n,user) createTableOfTimeChanges(filePath,save_dir,server,n,len(urls)) generateGraphs(user,save_dir,filePath,server) checkFileExistsForMonthlyReport(save_dir,user) main()