7f9ebf15a57b4c8529177fdb89e4b56f36e57a3b gperez2 Mon Feb 6 16:45:36 2023 -0800 Code review edits: adding variable to get current year for searchedTermsCron.py and updating the relative path for a URL in newsarch.html , refs #30536 diff --git src/utils/qa/searchedTermsCron.py src/utils/qa/searchedTermsCron.py index 460f845..d836ff4 100755 --- src/utils/qa/searchedTermsCron.py +++ src/utils/qa/searchedTermsCron.py @@ -17,64 +17,66 @@ import sys import re import json import io import requests import subprocess from datetime import datetime user = getpass.getuser() def bash(cmd): """Input bash cmd and return stdout""" rawOutput = subprocess.run(cmd,check=True, shell=True, stdout=subprocess.PIPE, universal_newlines=True) return(rawOutput.stdout.split('\n')[0:-1]) +#Variable to get the current year +year=datetime.now().strftime("%Y") #Get the last 5 error logs from the RR/euro -latestLogs=bash("ls /hive/data/inside/wwwstats/RR/"+datetime.now().strftime("%Y")+"/hgw1") -latestLogs_euro=bash("ls /hive/data/inside/wwwstats/euroNode/"+datetime.now().strftime("%Y")+"/") +latestLogs=bash("ls /hive/data/inside/wwwstats/RR/"+year+"/hgw1") +latestLogs_euro=bash("ls /hive/data/inside/wwwstats/euroNode/"+year+"/") latestLogs = latestLogs[len(latestLogs)-5:] latestLogs_euro=latestLogs_euro[len(latestLogs_euro)-5:] #Make a directory for the month (Y-M) try: os.makedirs("/hive/users/qateam/searchedTermsCronArchive/"+datetime.now().strftime("%Y-%m")) except OSError: print("mkdir: cannot create directory /hive/users/qateam/searchedTermsCronArchive/"+datetime.now().strftime("%Y-%m")+": File exists") sys.exit(1) #Nodes to check for error logs nodes = ['RR', 'asiaNode', 'euroNode'] machines = ['hgw1','hgw2'] #Add hgw machines to check #For loop that goes through the RR/euro/asia error logs, trims down the hgSearch line, and writes to a file for node in nodes: with open('/hive/users/qateam/searchedTermsCronArchive/'+datetime.now().strftime("%Y-%m")+'/hgSearchTrimLogs', 'a') as f: if node == 'RR': for machine in machines: for log in latestLogs:#Copy the 5 latest error logs for each of the rr machines - hgSearch=bash("zcat /hive/data/inside/wwwstats/"+node+"/"+datetime.now().strftime("%Y")+"/"+machine+"/"+log+" | grep 'hgSearch' | tr '?' '\t' | cut -f 2 | grep 'search' | uniq") + hgSearch=bash("zcat /hive/data/inside/wwwstats/"+node+"/"+year+"/"+machine+"/"+log+" | grep 'hgSearch' | tr '?' '\t' | cut -f 2 | grep 'search' | uniq") for i in hgSearch: f.write(i+'\n') elif node == 'euroNode': for log in latestLogs_euro: - hgSearch=bash(" zcat /hive/data/inside/wwwstats/"+node+"/"+datetime.now().strftime("%Y")+"/"+log+" | grep 'hgSearch' | tr '?' '\t' | cut -f 2 | grep 'search' | uniq") + hgSearch=bash(" zcat /hive/data/inside/wwwstats/"+node+"/"+year+"/"+log+" | grep 'hgSearch' | tr '?' '\t' | cut -f 2 | grep 'search' | uniq") for i in hgSearch: f.write(i+'\n') else: for log in latestLogs: - hgSearch=bash(" zcat /hive/data/inside/wwwstats/"+node+"/"+datetime.now().strftime("%Y")+"/"+log+" | grep 'hgSearch' | tr '?' '\t' | cut -f 2 | grep 'search' | uniq") + hgSearch=bash(" zcat /hive/data/inside/wwwstats/"+node+"/"+year+"/"+log+" | grep 'hgSearch' | tr '?' '\t' | cut -f 2 | grep 'search' | uniq") for i in hgSearch: f.write(i+'\n') f.close() #Remove duplicates with the same hgsid and save the list to a variable search_lines= bash("cat /hive/users/qateam/searchedTermsCronArchive/"+datetime.now().strftime("%Y-%m")+"/hgSearchTrimLogs | sort | uniq ") #For loop that removes hgsid and counts the search term searches_count = {} for term in search_lines: if len(term.split('&'))>2: term=term.split('&') term.pop(1) term.reverse() term=str(term)[1:-1]