62c58443da3f62b5ac33c1933e6abc3b249b2ba0 lrnassar Tue Jun 3 15:31:31 2025 -0700 Updating hgTracksTiming for two purposes: 1. Making it take command-line arguments so what we can call it separately for the development machines vs. live servers. The reason for this is that we don't need to spam the genome-alert ML for hgwbeta or dev issues. 2. It was previously writing out very low response times that came from failures like the captcha or not finding hgTracks end to the log file. This was throwing off the usage numbers as low values are good, but in this case they are errors. No RM. diff --git src/utils/qa/hgTracksTiming.py src/utils/qa/hgTracksTiming.py index ce2cb43e407..fc2e325b8bf 100644 --- src/utils/qa/hgTracksTiming.py +++ src/utils/qa/hgTracksTiming.py @@ -1,287 +1,335 @@ # Meant as a substitute for hgTracksRandom # Queries a list of GB servers (crontab set to every 15m) and documents their load time # as well as their status code, if they took too long to load, or if hgTracks display did not # fully load. Alerts with a printed error when a negative condition is encountered. # Each run it reads the list of all observed times and regenerates a table and graphs displaying # the change of server load times over time. Once per month it reports as a reminder to check for abnormalities # If running on a new user, you will need to copy the index.html page from qateam and run the function here once: makeSymLinks(user,save_dir) -import requests, subprocess, time, datetime, getpass, os, urllib3, matplotlib +import requests, subprocess, time, datetime, getpass, os, urllib3, matplotlib, argparse, sys import matplotlib.pyplot as plt import matplotlib.ticker as mticker import matplotlib.dates as mdates -import matplotlib from collections import defaultdict from collections import deque +def parseArgs(): + """ + Parse the command line arguments. + """ + parser = argparse.ArgumentParser(description = __doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + + required = parser.add_argument_group('required arguments') + + required.add_argument ("servers", + help = "Which servers to query. Options are: 'all', 'RR' for hgw1/2/Euro/Asia, 'dev' for hgwbeta/dev") + if (len(sys.argv) == 1): + parser.print_usage() + print("\nQueries hgTracks on any of the given servers and notes down the response time. Also graphs the response\n" + \ + "over time. Options are: 'all', 'RR' for hgw1/2/Euro/Asia, 'dev' for hgwbeta/dev\n\n" + \ + + "Example runs:\n" + \ + " hgTracksTiming.py all\n" + \ + " hgTracksTiming.py RR\n" + \ + " hgTracksTiming.py dev\n") + + exit(0) + options = parser.parse_args() + return options + def bash(cmd): """Run the cmd in bash subprocess""" try: rawBashOutput = subprocess.run(cmd, check=True, shell=True,\ stdout=subprocess.PIPE, universal_newlines=True, stderr=subprocess.STDOUT) bashStdoutt = rawBashOutput.stdout except subprocess.CalledProcessError as e: raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) return(bashStdoutt) def makeSymLinks(user,save_dir): if user == 'qateam': bash("ln -sf "+save_dir+"*.html /usr/local/apache/htdocs-genecats/qa/test-results/hgTracksTiming/") bash("ln -sf "+save_dir+"*.png /usr/local/apache/htdocs-genecats/qa/test-results/hgTracksTiming/") else: bash("ln -sf "+save_dir+"*.html /cluster/home/"+user+"/public_html/cronResults/hgTracksTiming/") bash("ln -sf "+save_dir+"*.png /cluster/home/"+user+"/public_html/cronResults/hgTracksTiming/") def getLastLinesAndMakeList(file_path, num_lines=20): with open(file_path, "r") as file: if num_lines == 'All': all_lines = file.readlines() # Read all lines in the file else: last_lines = deque(file, maxlen=num_lines) # Read only the last 'num_lines' lines dates = [] times = [] if num_lines == 20: for line in last_lines: # Split the line and extract the date and time parts = line.rstrip().split("\t") dates.append(parts[0]) # First part is the date times.append(float(parts[1].split("s")[0])) # Second part is the time, removing the 's' elif num_lines == 80: timeToWrite = [] for i, line in enumerate(last_lines): parts = line.rstrip().split("\t") time = float(parts[1].split("s")[0]) # Apply logic to every 4th line (0, 4, 8, ...) if i % 4 == 0: timeToWrite.append(time) averageTime = round(sum(timeToWrite)/len(timeToWrite),1) dates.append(parts[0]) # First part is the date times.append(averageTime) timeToWrite = [] else: timeToWrite.append(time) elif num_lines == 'All': #Calculate average times of all lines / 20 total_lines = len(all_lines) # Determine 20 evenly spaced line indices indices = [int(i * total_lines / 20) for i in range(20)] timeToWrite = [] for i, line in enumerate(all_lines): parts = line.rstrip().split("\t") time = float(parts[1].split("s")[0]) if i in indices: timeToWrite.append(time) averageTime = round(sum(timeToWrite)/len(timeToWrite),1) dates.append(parts[0]) # First part is the date times.append(averageTime) timeToWrite = [] else: timeToWrite.append(time) return(dates,times) def generateGraphs(user,save_dir,filePath,server): #Create the 3 time scale graphs for each server reportsToGenerate = ['Last 5h','Last 20h','AllTime/20'] n=0 for report in reportsToGenerate: n+=1 # x axis values if report == "Last 5h": dates,times = getLastLinesAndMakeList(filePath, num_lines=20) elif report == "Last 20h": dates,times = getLastLinesAndMakeList(filePath, num_lines=80) elif report == "AllTime/20": dates,times = getLastLinesAndMakeList(filePath, num_lines='All') x_dates = dates y = times # plotting the points plt.plot(x_dates, y, marker='o') # Rotate date labels for better readability plt.gcf().autofmt_xdate() # naming the x axis plt.xlabel('Date/time') # naming the y axis plt.ylabel("Load time in s") plt.xticks(x_dates) plt.title(report + " - " + server) # giving a title to my graph # Ensure the figure is fully rendered before saving plt.gcf().canvas.draw() # Force rendering of the canvas # Save the plot to a file plt.savefig(save_dir + "/" + server + "." + str(n) + ".png", bbox_inches='tight') # Clear the current plot to avoid overlaps with the next plot plt.clf() def create_save_dir(user): save_dir = f"/hive/users/{user}/hgTracksTiming/" os.makedirs(save_dir, exist_ok=True) # Creates the directory if it doesn't exist return save_dir def createTableOfTimeChanges(filePath,save_dir,server,n,totalN): tableFilePath = save_dir + "timeChangesTable.html" monthly_data = defaultdict(list) # Parse the input file with open(filePath, "r") as file: for line in file: parts = line.rstrip().split("\t") date_str = parts[0] # First part is the date time = float(parts[1].split("s")[0]) # Second part is the time, removing the 's' # Extract year and month date_obj = datetime.datetime.strptime(date_str, "%Y-%m-%d-%H:%M") year_month = (date_obj.year, date_obj.month) # Store time value for the year-month combination monthly_data[year_month].append(time) # Calculate averages averages = {ym: sum(times) / len(times) for ym, times in monthly_data.items()} # Generate HTML if server =="hgwdev": writeMode = "w" else: writeMode = "a" with open(tableFilePath, writeMode) as output_file: if server =="hgwdev": output_file.write("
\n\n\n\n\n
\n") elif n%2!=0: output_file.write("
\n\n") else: output_file.write("\n\n
\n") output_file.write("

"+server+"

\n\n") # Create table header (Months) months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] output_file.write("" + "".join(f"" for month in months) + "\n") # Populate table rows with only available months years = sorted(set(year for year, _ in averages.keys())) for year in years: output_file.write(f"") prev_value = None for month in range(1, 13): value = averages.get((year, month), "-") if isinstance(value, float): value = f"{value:.3f}" # Format to 3 decimal places output_file.write(f"") output_file.write("\n") # Calculate and display % change row output_file.write(f"") for month in range(1, 13): current_value = averages.get((year, month)) if prev_value is not None and current_value is not None: percent_change = ((current_value - prev_value) / prev_value) * 100 color = "red" if percent_change > 0 else "green" output_file.write(f"") else: output_file.write("") prev_value = current_value output_file.write("\n") output_file.write("
Year{month}
{year}{value}
Change {percent_change:.2f}% -
\n") if n==totalN: output_file.write("\n") def checkFileExistsForMonthlyReport(save_dir,user): month = datetime.datetime.today().strftime("%m") fileToCheckAndReport = save_dir + "monthFile" + month if not os.path.isfile(fileToCheckAndReport): # The month check file does not exist, report the monthly output # Delete all files matching "monthFile*" in the current directory for filename in os.listdir(save_dir): if filename.startswith("monthFile") and os.path.isfile(filename): os.remove(filename) # Create a new blank file with the specified path with open(fileToCheckAndReport, 'w') as new_file: pass # Creates an empty file print("Monthly reminder to check the hgTracksTiming information for any abnormalities:\n") if user == 'qateam': print("https://genecats.gi.ucsc.edu/qa/test-results/hgTracksTiming/") else: print("https://hgwdev.gi.ucsc.edu/~"+user+"/cronResults/hgTracksTiming/") def queryServersAndReport(server,url,filePath,today,n,user): start_time = time.time() response = requests.get(url, verify=False) # Disable SSL verification end_time = time.time() load_time = end_time - start_time page_content = response.text # Get the page content # Check if the expected string is in the response if "END hgTracks" in page_content: if load_time < 15: problem = False status = "SUCCESS" else: problem = True status = "FAIL - hgTracks page loaded, but load time over 15s" else: problem = True status = "FAIL - Got status 200 return, but missing the 'END hgTracks' page string of a successful load" if problem == True: print("Potential problem with Genome Browser server.") print(f"URL: {url} | Status: {response.status_code} | Load Time: {load_time:.3f}s | Check: {status}") print("\nSee the latest timing numbers:") if user == 'qateam': print("https://genecats.gi.ucsc.edu/qa/test-results/hgTracksTiming/") else: print("https://hgwdev.gi.ucsc.edu/~"+user+"/cronResults/hgTracksTiming/") + # Add a check here to make sure we are not writing out bad 200 or captcha failures, but still + # writing out problematic > 15s load times + if load_time > .2: with open(filePath, "a") as file: file.write(f"{today}\t{load_time:.3f}s\t{response.status_code}\n") def main(): #Don't try to display the plot, this is for jupyter matplotlib.use('Agg') # Suppress SSL warnings - was due to an asia issue urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) user = getpass.getuser() save_dir = create_save_dir(user) today = datetime.datetime.today().strftime("%Y-%m-%d-%H:%M") + #Parse which servers to query + options = parseArgs() + servers = options.servers + + if servers == 'all': # Dic with all the URLs to test. To temporarily pause testing of URLs for maintenance, expected outage, etc. # Remove it from this dictionary urls = { "hgwdev": "https://hgwdev.gi.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", "hgwbeta": "https://hgwbeta.soe.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", "hgw1": "https://hgw1.soe.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", "hgw2": "https://hgw2.soe.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", "euro": "https://genome-euro.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", "asia": "https://genome-asia.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1" } + elif servers == 'RR': + urls = { + "hgw1": "https://hgw1.soe.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", + "hgw2": "https://hgw2.soe.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", + "euro": "https://genome-euro.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", + "asia": "https://genome-asia.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1" + } + + elif servers == 'dev': + urls = { + "hgwdev": "https://hgwdev.gi.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1", + "hgwbeta": "https://hgwbeta.soe.ucsc.edu/cgi-bin/hgTracks?hgt.trackImgOnly=1&hgt.reset=1" + } + else: + sys.exit("No server specified. Use either 'all', 'RR', or 'dev' to specify which servers to query") + n=0 for server, url in urls.items(): n+=1 filePath = save_dir + server + ".txt" queryServersAndReport(server,url,filePath,today,n,user) createTableOfTimeChanges(filePath,save_dir,server,n,len(urls)) generateGraphs(user,save_dir,filePath,server) checkFileExistsForMonthlyReport(save_dir,user) main()