32515a6797b6e68daa3aa94c663760216b24dd67 lrnassar Thu Sep 3 13:30:26 2026 -0700 Usage stats cron fixes from code review. refs #38232 Apply the same db mismatch guard to assemblyStatsCron.py that generateUsageStats.py already had. Both files ask hgTracks for one assembly's default tracks and parse the answer off stderr, but only one of them checked that hgTracks answered about the assembly it was asked about. Without it, a repeat of the db= bug this ticket fixed would quietly fill the default track filter with another assembly's tracks and say nothing. Drop any database name that does not look like one before interpolating it into a shell command. The trimmed logs only ever carry real assembly names, so this is defence in depth rather than a live hole. Fix a comment in resolveHub that still described the lastOkTime fallback, which was replaced by mirror order earlier in this ticket. No change to the report. The row counts for the database usage and non-public hub tables were cut from 15 to 10 in the previous commit, which its message did not mention; that was intentional, to make room for the new GenArk and hubSpace sections. diff --git src/hg/logCrawl/dbTrackAndSearchUsage/generateUsageStats.py src/hg/logCrawl/dbTrackAndSearchUsage/generateUsageStats.py index d6990f214db..67f00547d1e 100755 --- src/hg/logCrawl/dbTrackAndSearchUsage/generateUsageStats.py +++ src/hg/logCrawl/dbTrackAndSearchUsage/generateUsageStats.py @@ -1,697 +1,700 @@ #!/usr/bin/env python3 -import subprocess, os, gzip, argparse, sys, json, operator, datetime +import subprocess, os, gzip, argparse, sys, json, operator, datetime, re from collections import Counter, defaultdict ##### # Define dictionaries/sets/etc. needed to record stats ##### # Making these global so that they can be modified by functions whithout needing # a function argument to specify them # Dictionaries for holding information about db use dbUsers = defaultdict(Counter) # Ex dbUsers struct: {"db":{"hgsid":count}} dbCounts = dict() # Ex dbCounts struct: {"db":count} # Dictionaries for recording information per month dbUsersMonth = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict()))) # Ex dbUsersMonth struct: {"db":{"year":{"month":{"hgsid":count}}}} dbCountsMonth = defaultdict(lambda: defaultdict(lambda: defaultdict())) # Ex dbUsersMonth struct: {"db":{"year":{"month":count}}}} # Dictionaries for holding information about track use per db trackUsers = defaultdict(lambda: defaultdict(lambda: defaultdict())) # Ex trackUsers struct: {"db":{"track":{"hgsid":count}}} trackCounts = defaultdict(dict) # Ex trackCounts struct: {"db":{"track":count}} # Dictionaries for per month information trackUsersMonth = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda:\ defaultdict(lambda: defaultdict())))) # Ex trackUsersMonth struct: {"db":{"year":{"month":{"track":{"hgsid":count}}}}} trackCountsMonth = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda:\ defaultdict()))) # Ex trackCountsMonth struct: {"db":{"year":{"month":{"track":count}}}} # Dictionaries for holding information about track use per hub trackUsersHubs = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict()))) # Ex trackUsersHubs struct: {"hubId":{"db":{"track":{"hgsid":count}}}} trackCountsHubs = defaultdict(lambda: defaultdict(lambda: defaultdict())) # Ex trackCountsHubs struct: {"hubId":{"db":{"track":count}}} # Dictionaries for per month information trackUsersHubsMonth = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda:\ defaultdict(lambda: defaultdict(lambda: defaultdict()))))) # Ex trackUserHubsMonth struct: {"hubId":{"db":{"year":{"month":{"track":{"hgsid":count}}}}}} trackCountsHubsMonth = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict( lambda: defaultdict())))) # Ex trackCountsHubsMonth struct: {"hubId":{"db":{"year":{"month":{"track":count}}}}} monthYearSet = set() # Set containing monthYear strings (e.g. "Aug 2017") # Create a dictionary of hubUrls, shortLabels, and hubStatus ids publicHubs = dict() # Use hgsql to grab hub ID from hubStatus table and shortLabel from hubPublic # for each hub in hubPublic table cmd = ["/cluster/bin/x86_64/hgsql", "hgcentral", "-h", "genome-centdb", "-Ne", "select s.id,p.hubUrl,p.shortLabel\ from hubPublic p join hubStatus s where s.hubUrl=p.hubUrl", ] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) cmdout, cmderr = p.communicate() hubs = cmdout.decode("ASCII") hubs = hubs.split("\n") for hub in hubs: if hub == "": continue else: splitHub = hub.split("\t") publicHubs[splitHub[0]] = [splitHub[1], splitHub[2]] ##### ##### def parseTrackLog(line): """Parse trackLog line and return important fields: db, year, month, hgsid, and a list of tracks""" #### Sample line being processed #### # From Chris Lee's combined/trimmed log format produced by {script name} # [Sun Mar 05 04:11:27 2017] [error] [client ###.###.###.##] trackLog 0 hg38 hgsid_### cytoBandIdeo:1,cloneEndCTD:2 #### splitLine = line.strip().split('\t') date = splitLine[3] month = date.split()[1] year = date.split()[4] ip = splitLine[1] hgsid = splitLine[2] # temporary, really should rewrite script to use a var like ip_hgsid everywhere #hgsid = ip + "_" + hgsid db = splitLine[4] if len(splitLine) > 5: tracks = splitLine[5].split(",") else: tracks = [] return db, year, month, hgsid, tracks def modDicts(db, year, month, hgsid, tracks, toProcess, perMonth=False): """Modify global dictionaries to store information on db, track, and hub track usage""" ##### Set some variables based on input options processDbUsers = toProcess[0] processTrackUsers = toProcess[1] processTrackHubUsers = toProcess[2] ##### Process info about db usage # Count up number of times each hgsid shows up if processDbUsers == True: if hgsid not in dbUsers[db]: # If no entry in dictionary of users, intialize value to 1 dbUsers[db][hgsid] = 1 else: # Otherwise, increment usage by 1 dbUsers[db][hgsid] += 1 # If perMonth is true, then we record information about db usage on a perMonth basis if perMonth == True: if hgsid not in dbUsersMonth[db][year][month]: dbUsersMonth[db][year][month][hgsid] = 1 else: dbUsersMonth[db][year][month][hgsid] += 1 ##### Process per track information if processTrackUsers == True: for track in tracks: # Skip empty entries in trackList if track == "": continue # Remove trailing characters track = track.split(":")[0] # Record track user if hgsid not in trackUsers[db][track]: trackUsers[db][track][hgsid] = 1 else: trackUsers[db][track][hgsid] += 1 # If perMonth is true, then we record information about track usage on a perMonth basis if perMonth == True: # Incremement count for track/hgsid by one if hgsid not in trackUsersMonth[db][year][month][track]: trackUsersMonth[db][year][month][track][hgsid] = 1 else: trackUsersMonth[db][year][month][track][hgsid] += 1 ##### Process public hub tracks if processTrackHubUsers == True: if track.startswith("hub_"): track = track.split(":")[0] splitTrack = track.split("_", 2) if len(splitTrack) > 2: hubId = splitTrack[1] hubTrack = splitTrack[2] # Processed in same way as normal tracks, although now the top level dictionary is # keyed by hubIds, rather than db name if hubId in publicHubs: if hgsid not in trackUsersHubs[hubId][db][hubTrack]: trackUsersHubs[hubId][db][hubTrack][hgsid] = 1 else: trackUsersHubs[hubId][db][hubTrack][hgsid] += 1 if perMonth == True: if hgsid not in trackUsersHubsMonth[hubId][db][year][month][hubTrack]: trackUsersHubsMonth[hubId][db][year][month][hubTrack][hgsid] = 1 else: trackUsersHubsMonth[hubId][db][year][month][hubTrack][hgsid] += 1 def processFile(fileName, toProcess, perMonth=False): """Process a file line by line using the function parseTrackLog and record usage information using the function modDicts""" if fileName.endswith(".gz"): ifh = gzip.open(fileName, "r") else: ifh = open(fileName, "r") for line in ifh: if "str" not in str(type(line)): line = line.decode("ASCII") db, year, month, hgsid, tracks = parseTrackLog(line) modDicts(db, year, month, hgsid, tracks, toProcess, perMonth) # Keep track of month/years covered monthYear = month + " " + year monthYearSet.add(monthYear) def processDir(dirName, toProcess, perMonth=False): """Process files in a directory using processFile function""" fileNames = os.listdir(dirName) for log in fileNames: fileName = os.path.join(dirName, log) processFile(fileName, toProcess, perMonth) def dumpToJson(data, outputFile, outputDir): """output data to named outputFile""" jsonOut = open(os.path.join(outputDir, outputFile), "w") json.dump(data, jsonOut) jsonOut.close() def main(): # Parse command-line arguments parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description="Generates usage statistics for dbs, tracks, and hubs \ tracks using processed Apache error_log files. \nThe processed files can be \ found in the following directory: /hive/users/chmalee/logs/trimmedLogs/result\n\n\ For more information, see RM#26191.") parser.add_argument("-f","--fileName", type=str, help='input file name, \ must be space-separated Apache error_log file') parser.add_argument("-d","--dirName", type=str , help='input directory \ name, files must be space-separated error_log files. No other files should be \ present in this directory.') parser.add_argument("--dbCounts", action='store_true', help='output a \ file containing users per ucsc database') parser.add_argument("--dbUsers", action='store_true', help='output a \ file containing hgsids associated with each ucsc databases and how many uses each one had') parser.add_argument("--dbUsage", action='store_true', help='output a \ file containing hgsids associated with each ucsc databases and how many uses each one had') parser.add_argument("--trackCounts", action='store_true', help='output a \ file containing counts of how many users had a track on for a particular ucsc database') parser.add_argument("--trackUsers", action='store_true', help='output a \ file containing hgsids, the tracks associated with them and how many trackLog lines they showed up in') parser.add_argument("--trackUsage", action='store_true', help='output a \ file containing hgsids associated with each ucsc databases and how many uses each one had') parser.add_argument("--trackHubCounts", action='store_true', help='output a \ file containing counts of how many users had a hub track on for a particular ucsc database or assembly hub') parser.add_argument("--trackHubUsers", action='store_true', help='output a \ file containing hgsids, the hub tracks associated with them and how many trackLog lines they showed up in') parser.add_argument("--trackHubUsage", action='store_true', help='output a \ file containing hgsids associated with each ucsc databases and how many uses each one had') parser.add_argument("--allOutput", action='store_true', help='output a \ file for each option above (dbCounts, dbUsers, dbUsage, trackCounts, trackUsers, trackHubCounts, trackHubUsers)') parser.add_argument("-p","--perMonth", action='store_true', help='output \ file containing info on db/track/hub track use per month') parser.add_argument("-m","--monthYear", action='store_true', help='output \ file containing month/year pairs (e.g. "Mar 2017")') parser.add_argument("-j","--jsonOut", action='store_true', help='output \ json files for summary dictionaries') parser.add_argument("-t","--outputDefaults", action='store_true', help='output a file containing info on default track usage for top 15 most used assemblies') parser.add_argument("-o","--outDir", type=str, help='directory in which to place output files') args = parser.parse_args() # Print help message if no arguments are supplied if len(sys.argv) == 1: parser.print_help(sys.stderr) sys.exit(1) # File and directory options can't be used together. Catch this and exit. if args.fileName != None and args.dirName != None: print("-f/--fileName and -d/--dirName cannot be used together. Choose one and re-run.") sys.exit(1) # Catch it early if input file/directory doesn't exist and exit. if args.fileName: if not os.path.exists(args.fileName): print(args.fileName, "doesn't exist. Please run on a valid file.") exit(1) elif args.dirName: if not os.path.exists(args.dirName): print(args.dirName, "doesn't exist. Please run on a valid directory.") exit(1) if not any([args.dbCounts, args.dbUsers, args.dbUsage, args.trackUsers, args.trackCounts, args.trackUsage, args.trackHubUsers, args.trackHubCounts, args.trackHubUsage, args.allOutput]): print("You must specify at least one of: --allOutput, --dbCounts, --dbUsage, --dbUsers, --trackCounts, --trackUsers, --trackHubCounts, --trackHubUsers.") exit(1) # Setup output directory if args.outDir == None: # If an output directory is unspecified, then a new one with the current date/time is made currDateTime = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") os.makedirs(currDateTime) outDir = currDateTime else: # Otherwise, user supplied a output directory name outDir = args.outDir if not os.path.exists(outDir): # If specied output directory doesn't exist, create it os.makedirs(outDir) # If "all" option is set change everyone to True if args.allOutput == True: args.dbCounts = True args.dbUsers = True args.dbUsage = True args.trackUsers = True args.trackCounts = True args.trackUsage = True args.trackHubUsers = True args.trackHubCounts = True args.trackHubUsage = True # Make list of users to process and build out dictionaries for later usersToProcess = [False, False, False] # Pos 0 - process db users # Pos 1 - process track hub users # Pos 2 - process public track hub users # The reason I've done it this way is to allow people to output just the counts # dbs/tracks/hub tracks without outputting the users files. I could have just set # args.*Users = True if args.*Counts was true, but that would mean that the users file # would also be output, defeating the purpose of the option to only output the *Counts if any([args.dbCounts, args.dbUsers, args.dbUsage]): # If dbCounts or dbUsers speficied, we need to process users usersToProcess[0] = True if any([args.trackCounts, args.trackUsers, args.trackUsage]): usersToProcess[1] = True if any([args.trackHubCounts, args.trackHubUsers, args.trackHubUsage]): usersToProcess[1] = True usersToProcess[2] = True # Process input to create dictionaries containing info about users per db/track/hub track if args.fileName: processFile(args.fileName, usersToProcess, args.perMonth) elif args.dirName: processDir(args.dirName, usersToProcess, args.perMonth) ##### Output files of summaries of db/track/hub track usage information # Output files of db users and db use if args.dbCounts == True: dbCountsFile = open(os.path.join(outDir, "dbCounts.tsv"), "w") if args.dbUsers == True: dbUsersFile = open(os.path.join(outDir, "dbUsers.tsv"), "w") if args.dbUsage == True: dbUsageFile = open(os.path.join(outDir, "dbUsage.tsv"), "w") # Set of nested for loops to go through dictionary level by level and # output and summarize results into appropriate counts dictionary if any([args.dbUsage, args.dbUsers, args.dbCounts]): for db in dbUsers: dbCounts[db] = 0 dbUsage = 0 for hgsid in dbUsers[db]: # Ignore those dbs only used once if dbUsers[db][hgsid] != 1: dbCounts[db] += 1 # Output dbUsers info to a file count = dbUsers[db][hgsid] dbUsage += count if args.dbUsers == True: dbUsersFile.write(db + "\t" + hgsid + "\t" + str(count) + "\n") # Output counts of total users for each db if dbCounts[db] != 0: if args.dbCounts == True: dbCountsFile.write(db + "\t" + str(dbCounts[db]) + "\n") if args.dbUsage == True: dbUsageFile.write(db + "\t" + str(dbUsage) + "\n") # Close our output files if args.dbCounts == True: print("Finished outputting dbCounts.tsv") dbCountsFile.close() if args.dbUsers == True: print("Finished outputting dbUsers.tsv") dbUsersFile.close() if args.dbUsage == True: print("Finished outputting dbUsage.tsv") dbUsageFile.close() # Output files of track users and track use if args.trackCounts == True: trackCountsFile = open(os.path.join(outDir, "trackCounts.tsv"), "w") if args.trackUsers == True: trackUsersFile = open(os.path.join(outDir, "trackUsers.tsv"), "w") if args.trackUsage == True: trackUsageFile = open(os.path.join(outDir, "trackUsage.tsv"), "w") # Set of nested for loops to go through dictionary level by level and # output and summarize results into appropriate counts dictionary if any([args.trackUsage, args.trackUsers, args.trackCounts]): for db in trackUsers: for track in trackUsers[db]: # Initialize count for track to 0 trackCounts[db][track] = 0 trackUsage = 0 for hgsid in trackUsers[db][track]: # Filter out those hgsids who only used track once - likely to be bots if trackUsers[db][track][hgsid] != 1: trackCounts[db][track] += 1 # Output information on how much each user used each track count = trackUsers[db][track][hgsid] trackUsage += count if args.trackUsers == True: trackUsersFile.write(db + "\t" + track + "\t" + hgsid + "\t" + str(count) + "\n") if trackCounts[db][track] != 0: if args.trackCounts == True: # Output counts of total users for each track count = trackCounts[db][track] trackCountsFile.write(db + "\t" + track + "\t" + str(count) + "\n") if args.trackUsage == True: trackUsageFile.write(db + "\t" + track + "\t" + str(trackUsage) + "\n") # Close our output files if args.trackCounts == True: print("Finished outputting trackCounts.tsv") trackCountsFile.close() if args.trackUsers == True: print("Finished outputting trackUsers.tsv") trackUsersFile.close() if args.trackUsage == True: print("Finished outputting trackUsage.tsv") trackUsageFile.close() # Output files of hub track users and hub track use if args.trackHubUsers == True: trackUsersHubsFile = open(os.path.join(outDir, "trackUsersHubs.tsv"), "w") if args.trackHubCounts == True: trackCountsHubsFile = open(os.path.join(outDir, "trackCountsHubs.tsv"), "w") if args.trackHubUsage == True: trackUsageHubsFile = open(os.path.join(outDir, "trackUsageHubs.tsv"), "w") # Set of nested for loops to go through dictionary level by level and # output and summarize results into appropriate counts dictionary if any([args.trackHubUsage, args.trackHubUsers, args.trackHubCounts]): for hubId in trackUsersHubs: hubLabel = publicHubs[hubId][1] for db in trackUsersHubs[hubId]: for track in trackUsersHubs[hubId][db]: # Initialize count for hub track to 0 trackCountsHubs[hubId][db][track] = 0 trackHubUsage = 0 for hgsid in trackUsersHubs[hubId][db][track]: # Filter out those hgsids who only used track once - likely to be bots if trackUsersHubs[hubId][db][track][hgsid] != 1: trackCountsHubs[hubId][db][track] += 1 # Output information on how much each user used each track count = trackUsersHubs[hubId][db][track][hgsid] trackHubUsage += count if args.trackHubUsers == True: trackUsersHubsFile.write(hubLabel + "\t" + db + "\t" + track + "\t" + hgsid +\ "\t" + str(count) + "\n") if trackCountsHubs[hubId][db][track] != 0: if args.trackHubCounts == True: # Output counts of total users for each hub track count = trackCountsHubs[hubId][db][track] trackCountsHubsFile.write(hubLabel + "\t" + db + "\t" + track + "\t" +\ str(count) + "\n") if args.trackHubUsage == True: trackUsageHubsFile.write(hubLabel + "\t" + db + "\t" + track + "\t" +\ str(trackHubUsage) + "\n") # Close our output files if args.trackHubUsers == True: print("Finished outputting trackHubUsers.tsv") trackUsersHubsFile.close() if args.trackHubCounts == True: print("Finished outputting trackHubCounts.tsv") trackCountsHubsFile.close() if args.trackHubUsage == True: print("Finished outputting trackHubUsage.tsv") trackUsageHubsFile.close() # Output file containing info on month/years covered by stats if indicated if args.monthYear == True: monthYearFile = open(os.path.join(outDir, "monthYear.tsv"), "w") for pair in monthYearSet: monthYearFile.write(pair + "\n") print("Finished outputting monthYear.tsv") monthYearFile.close() ##### Output data per month when indicated if args.perMonth == True: if args.dbUsers == True: dbUsersMonthFile = open(os.path.join(outDir, "dbUsers.perMonth.tsv"), "w") if args.dbCounts == True: dbCountsMonthFile = open(os.path.join(outDir, "dbCounts.perMonth.tsv"), "w") if args.dbUsage == True: dbUsageMonthFile = open(os.path.join(outDir, "dbUsage.perMonth.tsv"), "w") # Set of nested for loops to go through dictionary level by level and # output and summarize results into appropriate counts dictionary if any([args.dbUsage, args.dbUsers, args.dbCounts]): for db in dbUsersMonth: for year in dbUsersMonth[db]: for month in dbUsersMonth[db][year]: dbCountsMonth[db][year][month] = 0 dbUsageMonth = 0 for hgsid in dbUsersMonth[db][year][month]: if dbUsersMonth[db][year][month][hgsid] != 1: dbCountsMonth[db][year][month] += 1 # Output dbUsersMonth info to a file count = dbUsersMonth[db][year][month][hgsid] dbUsageMonth += count if args.dbUsers == True: dbUsersMonthFile.write(db + "\t" + year + "\t" + month +\ "\t" + hgsid + "\t" + str(count) + "\n") # Output dbCounts to a file if dbCountsMonth[db][year][month] != 0: if args.dbCounts == True: count = dbCountsMonth[db][year][month] dbCountsMonthFile.write(db + "\t" + year + "\t" + month +\ "\t" + str(count) + "\n") if args.dbUsage == True: dbUsageMonthFile.write(db + "\t" + year + "\t" + month +\ "\t" + str(dbUsageMonth) + "\n") if args.dbUsers == True: print("Finished outputting dbUsers.perMonth.tsv") dbUsersMonthFile.close() if args.dbCounts == True: print("Finished outputting dbCounts.perMonth.tsv") dbCountsMonthFile.close() if args.dbUsage == True: print("Finished outputting dbUsage.perMonth.tsv") dbUsageMonthFile.close() # Summarize user dictionaries to create counts per db/track/hub track if args.trackUsers == True: trackUsersMonthFile = open(os.path.join(outDir, "trackUsers.perMonth.tsv"), "w") if args.trackCounts == True: trackCountsMonthFile = open(os.path.join(outDir, "trackCounts.perMonth.tsv"), "w") if args.trackUsage == True: trackUsageMonthFile = open(os.path.join(outDir, "trackUsage.perMonth.tsv"), "w") # Set of nested for loops to go through dictionary level by level and # output and summarize results into appropriate counts dictionary if any([args.trackUsage, args.trackUsers, args.trackCounts]): for db in trackUsersMonth: for year in trackUsersMonth[db]: for month in trackUsersMonth[db][year]: for track in trackUsersMonth[db][year][month]: trackCountsMonth[db][year][month][track] = 0 trackUsageMonth = 0 for hgsid in trackUsersMonth[db][year][month][track]: if trackUsersMonth[db][year][month][track][hgsid] != 1: trackCountsMonth[db][year][month][track] += 1 count = trackUsersMonth[db][year][month][track][hgsid] trackUsageMonth += count if args.trackUsers == True: trackUsersMonthFile.write(db + "\t" + year + "\t" + month + "\t" +\ hgsid + "\t" + track + "\t" + str(count) + "\n") if trackCountsMonth[db][year][month][track] != 0: if args.trackCounts == True: count = trackCountsMonth[db][year][month][track] trackCountsMonthFile.write(db + "\t" + year + "\t" + month + "\t" +\ track + "\t" + str(count) + "\n") if args.trackUsage == True: trackUsageMonthFile.write(db + "\t" + year + "\t" + month + "\t" +\ track + "\t" + str(trackUsageMonth) + "\n") if args.trackUsers == True: print("Finished outputting trackUsers.perMonth.tsv") trackUsersMonthFile.close() if args.trackCounts == True: print("Finished outputting trackUsers.perMonth.tsv") trackCountsMonthFile.close() if args.trackUsage == True: print("Finished outputting trackUsage.perMonth.tsv") trackUsageMonthFile.close() # Summarize user dictionaries to create counts per db/track/hub track if args.trackHubUsers == True: trackUsersHubsMonthFile = open(os.path.join(outDir, "trackUsersHubs.perMonth.tsv"), "w") if args.trackHubCounts == True: trackCountsHubsMonthFile = open(os.path.join(outDir, "trackCountsHubs.perMonth.tsv"), "w") if args.trackHubUsage == True: trackUsageHubsMonthFile = open(os.path.join(outDir, "trackUsageHubs.perMonth.tsv"), "w") # Set of nested for loops to go through dictionary level by level and # output and summarize results into appropriate counts dictionary if any([args.trackHubUsage, args.trackHubUsers, args.trackHubCounts]): for hubId in trackUsersHubsMonth: hubLabel = publicHubs[hubId][1] for db in trackUsersHubsMonth[hubId]: for year in trackUsersHubsMonth[hubId][db]: for month in trackUsersHubsMonth[hubId][db][year]: for track in trackUsersHubsMonth[hubId][db][year][month]: trackCountsHubsMonth[hubId][db][year][month][track] = 0 trackHubUsageMonth = 0 for hgsid in trackUsersHubsMonth[hubId][db][year][month][track]: if trackUsersHubsMonth[hubId][db][year][month][track][hgsid] != 1: trackCountsHubsMonth[hubId][db][year][month][track] += 1 count = trackUsersHubsMonth[hubId][db][year][month][track][hgsid] trackHubUsageMonth += count if args.trackHubUsers == True: trackUsersHubsMonthFile.write(hubLabel + "\t" + db + "\t" + year +\ "\t" + month + "\t" + track + "\t" + hgsid + "\t" + str(count) + "\n") if trackCountsHubsMonth[hubId][db][year][month][track] != 0: if args.trackHubCounts == True: count = trackCountsHubsMonth[hubId][db][year][month][track] trackCountsHubsMonthFile.write(hubLabel + "\t" + db + "\t" + year +\ "\t" + month + "\t" + track + "\t" +\ str(count) + "\n") if args.trackHubUsage == True: trackUsageHubsMonthFile.write(hubLabel + "\t" + db + "\t" + year +\ "\t" + month + "\t" + track + "\t" +\ str(trackHubUsageMonth) + "\n") if args.trackHubUsers == True: print("Finished outputting trackUsersHubs.perMonth.tsv") trackUsersHubsMonthFile.close() if args.trackHubCounts == True: print("Finished outputting trackCountsHubs.perMonth.tsv") trackCountsHubsMonthFile.close() if args.trackHubUsage == True: print("Finished outputting trackUsageHubs.perMonth.tsv") trackUsageHubsMonthFile.close() ##### ##### Output json files if indicated ##### if args.jsonOut == True: dumpToJson(dbCounts, "dbCounts.json", outDir) dumpToJson(trackCounts, "trackCounts.json", outDir) dumpToJson(trackCountsHubs, "trackCountsHubs.json", outDir) if args.perMonth == True: dumpToJson(dbCountsMonth, "dbCounts.perMonth.json", outDir) dumpToJson(trackCountsMonth, "trackCounts.perMonth.json", outDir) dumpToJson(trackCountsHubsMonth, "trackCountsHubs.perMonth.json", outDir) #if args.monthYear == True: # dumpToJson(monthYearSet, "monthYearSet.json") ##### ##### Output information on default track usage if indicated ##### if args.outputDefaults == True and all([args.dbCounts, args.trackCounts]): # Sort dbs by most popular. Hub-backed dbs (e.g. hub_3671779_hs1) are skipped: hub ids # are handed out per hgcentral, so the ids in the logs never match the ids hgTracks # returns here and every track lookup below would miss. dbCountsSorted = sorted(dbCounts.items(), key=operator.itemgetter(1)) dbCountsSorted.reverse() - dbsToCheck = [db for db, useCount in dbCountsSorted if not db.startswith("hub_")] + #The name is interpolated into a shell command below, so anything that does not look + #like an assembly name is dropped rather than passed to sh + dbsToCheck = [db for db, useCount in dbCountsSorted + if not db.startswith("hub_") and re.match(r'^[A-Za-z0-9_.-]+$', db)] defaultCountsFile = open(os.path.join(outDir, "defaultCounts.tsv"), "w") for db in dbsToCheck[0:15]: # Only the default track stats for the 15 most popular assemblies dbOpt = "db=" + db # HGDB_CONF must be set here so that we use default tracks from beta, not dev # Dev can contain staged tracks that don't exist on RR, leading to errors later in script # The space before dbOpt matters - without it cgiSpoof reads the whole thing as one # variable and db is silently ignored, leaving every assembly with hg38's defaults. cmd = ["cd /usr/local/apache/cgi-bin && HGDB_CONF=$HOME/.hg.conf.beta ./hgTracks hgt.trackImgOnly=1 " + dbOpt] p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) cmdout, cmderr = p.communicate() # Hub shortLabels are user supplied, so an errAbort here can carry non-ASCII errText = cmderr.decode("utf-8", errors="replace") # hgTracks writes the visible track list to stderr as "trackLog N db hgsid t:vis,t:vis", # split into ~800 byte blocks so Apache does not chop the lines. Every numbered block # is part of the list; the trailing "trackLog position" line and the CGI_TIME and # RESOURCE lines are not, and must not be parsed as tracks. tracks = [] mismatch = None for errLine in errText.split("\n"): splitLine = errLine.split(" ") if len(splitLine) > 4 and splitLine[0] == "trackLog" and splitLine[1].isdigit(): if splitLine[2] != db: mismatch = splitLine[2] tracks = [] break tracks.extend(splitLine[4].split(",")) if not tracks: # assemblyStatsCron.py reads these back out and puts them in its report. Writing # to stderr would be pointless: the caller merges and discards it, and the cron # sends its own stderr to /dev/null. if mismatch: warning = "asked hgTracks for " + db + " but it reported " + mismatch else: warning = "no usable trackLog output from hgTracks for " + db defaultCountsFile.write("#WARNING\t" + warning + ", its default tracks are missing from this report\n") print("Warning: " + warning, file=sys.stderr) continue dbUse = dbCounts[db] # output list to file that contains column headings defaultCountsFile.write("#db\ttrackName\ttrackUse\t% using\t% turning off\n#" + db + "\t" + str(dbUse) + "\n") defaultCounts = [] for track in tracks: if track == "": continue # Remove trailing characters track = track.split(":")[0] try: trackUse = trackCounts[db][track] relUse = (trackUse/dbUse)*100 relOff = ((dbUse - trackUse)/dbUse)*100 # Store all this info a in a list so that we can sort my most used tracks later defaultCounts.append([track, trackUse, relUse, relOff]) except KeyError: continue # Sort defaultCounts for current assembly by most used track first defaultCountsSorted = sorted(defaultCounts, key=operator.itemgetter(2)) defaultCountsSorted.reverse() # Output sorted defaultCounts to a file for line in defaultCountsSorted: track = line[0] use = line[1] on = line[2] off = line[3] output = "{}\t{}\t{:d}\t{:3.2f}\t{:3.2f}\n".format(db, track, use, on, off) defaultCountsFile.write(output) print("Finished outputting defaultCounts.tsv") defaultCountsFile.close() elif args.outputDefaults == True: if not any([args.dbCounts, args.trackCounts]): # Need both dbCounts and trackCounts to actually be populated with data to be able to do the default counts print("\nCannot output default tracks if either --trackCounts or --dbCounts is not set. Set both of these options and re-run") #### # Print output directory name, particularly useful if using default output directory names # based on day/time print("\nYour output is in", outDir) if __name__ == "__main__": main()