4395aff08f55db6aea215e9717ceed746054840b lrnassar Wed Sep 2 15:34:32 2026 -0700 Fix and extend the monthly usage stats cron. refs #38232 Adds the GenArk and hubSpace reporting asked for in the ticket, and fixes several things that were quietly producing wrong numbers. New in the report: a summed GenArk row in the database usage table, a section ranking the top GenArk assemblies, and a closing section listing hubSpace usage with one row per user. The non-public hub list now drops GenArk and curated hubs, keeps only each hubSpace user's busiest hub, and reports track collections and ENCODE search hubs as one summed line each - previously a single person uploading a couple of hundred hubs took over most of the list. Fixes in generateUsageStats.py: a missing space meant db= was never passed to hgTracks, so every assembly was reported with hg38's default tracks. Only the first ~800 byte block of hgTracks' track list was read, so hg38 was analysed with 40 of its 91 defaults. Hub backed databases are now skipped, since hub ids differ between hgcentrals and their track names could never match the logs. Fixes in assemblyStatsCron.py: hgTracks was run from the wrong directory and died before logging anything, so the default track list was parsed out of timestamps and CGI_TIME lines - the "non-default track usage" section was really listing default tracks. The startup cleanup was deleting the hubStatus file that genome-asia copies over each month, which dropped asia-only hubs from the counts from February 2025 onward. The report label was computed as today minus 30 days, so the March run labelled itself January and overwrote it, and no February report has ever been published. Hub lookups no longer stop at the first 20 hubs, and are resolved by streaming each mirror's hubStatus once instead of grepping a 341MB file per hub. Counts for hubs are now keyed on the resolved hub URL rather than the track name, so unrelated hubs sharing a common track name are no longer added together. This moves some numbers: CADD reads 280 rather than 293, because three separate CADD download URLs were previously summed into one figure. Table columns, headers included, are now aligned in the emailed output. Warnings about an unreachable mirror or a missing asia file appear in the report body instead of a stderr stream that the cron discards. diff --git src/hg/logCrawl/dbTrackAndSearchUsage/generateUsageStats.py src/hg/logCrawl/dbTrackAndSearchUsage/generateUsageStats.py index 5d39cff6f60..d6990f214db 100755 --- src/hg/logCrawl/dbTrackAndSearchUsage/generateUsageStats.py +++ src/hg/logCrawl/dbTrackAndSearchUsage/generateUsageStats.py @@ -1,674 +1,697 @@ #!/usr/bin/env python3 import subprocess, os, gzip, argparse, sys, json, operator, datetime from collections import Counter, defaultdict ##### # Define dictionaries/sets/etc. needed to record stats ##### # Making these global so that they can be modified by functions whithout needing # a function argument to specify them # Dictionaries for holding information about db use dbUsers = defaultdict(Counter) # Ex dbUsers struct: {"db":{"hgsid":count}} dbCounts = dict() # Ex dbCounts struct: {"db":count} # Dictionaries for recording information per month dbUsersMonth = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict()))) # Ex dbUsersMonth struct: {"db":{"year":{"month":{"hgsid":count}}}} dbCountsMonth = defaultdict(lambda: defaultdict(lambda: defaultdict())) # Ex dbUsersMonth struct: {"db":{"year":{"month":count}}}} # Dictionaries for holding information about track use per db trackUsers = defaultdict(lambda: defaultdict(lambda: defaultdict())) # Ex trackUsers struct: {"db":{"track":{"hgsid":count}}} trackCounts = defaultdict(dict) # Ex trackCounts struct: {"db":{"track":count}} # Dictionaries for per month information trackUsersMonth = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda:\ defaultdict(lambda: defaultdict())))) # Ex trackUsersMonth struct: {"db":{"year":{"month":{"track":{"hgsid":count}}}}} trackCountsMonth = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda:\ defaultdict()))) # Ex trackCountsMonth struct: {"db":{"year":{"month":{"track":count}}}} # Dictionaries for holding information about track use per hub trackUsersHubs = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict()))) # Ex trackUsersHubs struct: {"hubId":{"db":{"track":{"hgsid":count}}}} trackCountsHubs = defaultdict(lambda: defaultdict(lambda: defaultdict())) # Ex trackCountsHubs struct: {"hubId":{"db":{"track":count}}} # Dictionaries for per month information trackUsersHubsMonth = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda:\ defaultdict(lambda: defaultdict(lambda: defaultdict()))))) # Ex trackUserHubsMonth struct: {"hubId":{"db":{"year":{"month":{"track":{"hgsid":count}}}}}} trackCountsHubsMonth = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict( lambda: defaultdict())))) # Ex trackCountsHubsMonth struct: {"hubId":{"db":{"year":{"month":{"track":count}}}}} monthYearSet = set() # Set containing monthYear strings (e.g. "Aug 2017") # Create a dictionary of hubUrls, shortLabels, and hubStatus ids publicHubs = dict() # Use hgsql to grab hub ID from hubStatus table and shortLabel from hubPublic # for each hub in hubPublic table cmd = ["/cluster/bin/x86_64/hgsql", "hgcentral", "-h", "genome-centdb", "-Ne", "select s.id,p.hubUrl,p.shortLabel\ from hubPublic p join hubStatus s where s.hubUrl=p.hubUrl", ] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) cmdout, cmderr = p.communicate() hubs = cmdout.decode("ASCII") hubs = hubs.split("\n") for hub in hubs: if hub == "": continue else: splitHub = hub.split("\t") publicHubs[splitHub[0]] = [splitHub[1], splitHub[2]] ##### ##### def parseTrackLog(line): """Parse trackLog line and return important fields: db, year, month, hgsid, and a list of tracks""" #### Sample line being processed #### # From Chris Lee's combined/trimmed log format produced by {script name} # [Sun Mar 05 04:11:27 2017] [error] [client ###.###.###.##] trackLog 0 hg38 hgsid_### cytoBandIdeo:1,cloneEndCTD:2 #### splitLine = line.strip().split('\t') date = splitLine[3] month = date.split()[1] year = date.split()[4] ip = splitLine[1] hgsid = splitLine[2] # temporary, really should rewrite script to use a var like ip_hgsid everywhere #hgsid = ip + "_" + hgsid db = splitLine[4] if len(splitLine) > 5: tracks = splitLine[5].split(",") else: tracks = [] return db, year, month, hgsid, tracks def modDicts(db, year, month, hgsid, tracks, toProcess, perMonth=False): """Modify global dictionaries to store information on db, track, and hub track usage""" ##### Set some variables based on input options processDbUsers = toProcess[0] processTrackUsers = toProcess[1] processTrackHubUsers = toProcess[2] ##### Process info about db usage # Count up number of times each hgsid shows up if processDbUsers == True: if hgsid not in dbUsers[db]: # If no entry in dictionary of users, intialize value to 1 dbUsers[db][hgsid] = 1 else: # Otherwise, increment usage by 1 dbUsers[db][hgsid] += 1 # If perMonth is true, then we record information about db usage on a perMonth basis if perMonth == True: if hgsid not in dbUsersMonth[db][year][month]: dbUsersMonth[db][year][month][hgsid] = 1 else: dbUsersMonth[db][year][month][hgsid] += 1 ##### Process per track information if processTrackUsers == True: for track in tracks: # Skip empty entries in trackList if track == "": continue # Remove trailing characters track = track.split(":")[0] # Record track user if hgsid not in trackUsers[db][track]: trackUsers[db][track][hgsid] = 1 else: trackUsers[db][track][hgsid] += 1 # If perMonth is true, then we record information about track usage on a perMonth basis if perMonth == True: # Incremement count for track/hgsid by one if hgsid not in trackUsersMonth[db][year][month][track]: trackUsersMonth[db][year][month][track][hgsid] = 1 else: trackUsersMonth[db][year][month][track][hgsid] += 1 ##### Process public hub tracks if processTrackHubUsers == True: if track.startswith("hub_"): track = track.split(":")[0] splitTrack = track.split("_", 2) if len(splitTrack) > 2: hubId = splitTrack[1] hubTrack = splitTrack[2] # Processed in same way as normal tracks, although now the top level dictionary is # keyed by hubIds, rather than db name if hubId in publicHubs: if hgsid not in trackUsersHubs[hubId][db][hubTrack]: trackUsersHubs[hubId][db][hubTrack][hgsid] = 1 else: trackUsersHubs[hubId][db][hubTrack][hgsid] += 1 if perMonth == True: if hgsid not in trackUsersHubsMonth[hubId][db][year][month][hubTrack]: trackUsersHubsMonth[hubId][db][year][month][hubTrack][hgsid] = 1 else: trackUsersHubsMonth[hubId][db][year][month][hubTrack][hgsid] += 1 def processFile(fileName, toProcess, perMonth=False): """Process a file line by line using the function parseTrackLog and record usage information using the function modDicts""" if fileName.endswith(".gz"): ifh = gzip.open(fileName, "r") else: ifh = open(fileName, "r") for line in ifh: if "str" not in str(type(line)): line = line.decode("ASCII") db, year, month, hgsid, tracks = parseTrackLog(line) modDicts(db, year, month, hgsid, tracks, toProcess, perMonth) # Keep track of month/years covered monthYear = month + " " + year monthYearSet.add(monthYear) def processDir(dirName, toProcess, perMonth=False): """Process files in a directory using processFile function""" fileNames = os.listdir(dirName) for log in fileNames: fileName = os.path.join(dirName, log) processFile(fileName, toProcess, perMonth) def dumpToJson(data, outputFile, outputDir): """output data to named outputFile""" jsonOut = open(os.path.join(outputDir, outputFile), "w") json.dump(data, jsonOut) jsonOut.close() def main(): # Parse command-line arguments parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description="Generates usage statistics for dbs, tracks, and hubs \ tracks using processed Apache error_log files. \nThe processed files can be \ found in the following directory: /hive/users/chmalee/logs/trimmedLogs/result\n\n\ For more information, see RM#26191.") parser.add_argument("-f","--fileName", type=str, help='input file name, \ must be space-separated Apache error_log file') parser.add_argument("-d","--dirName", type=str , help='input directory \ name, files must be space-separated error_log files. No other files should be \ present in this directory.') parser.add_argument("--dbCounts", action='store_true', help='output a \ file containing users per ucsc database') parser.add_argument("--dbUsers", action='store_true', help='output a \ file containing hgsids associated with each ucsc databases and how many uses each one had') parser.add_argument("--dbUsage", action='store_true', help='output a \ file containing hgsids associated with each ucsc databases and how many uses each one had') parser.add_argument("--trackCounts", action='store_true', help='output a \ file containing counts of how many users had a track on for a particular ucsc database') parser.add_argument("--trackUsers", action='store_true', help='output a \ file containing hgsids, the tracks associated with them and how many trackLog lines they showed up in') parser.add_argument("--trackUsage", action='store_true', help='output a \ file containing hgsids associated with each ucsc databases and how many uses each one had') parser.add_argument("--trackHubCounts", action='store_true', help='output a \ file containing counts of how many users had a hub track on for a particular ucsc database or assembly hub') parser.add_argument("--trackHubUsers", action='store_true', help='output a \ file containing hgsids, the hub tracks associated with them and how many trackLog lines they showed up in') parser.add_argument("--trackHubUsage", action='store_true', help='output a \ file containing hgsids associated with each ucsc databases and how many uses each one had') parser.add_argument("--allOutput", action='store_true', help='output a \ file for each option above (dbCounts, dbUsers, dbUsage, trackCounts, trackUsers, trackHubCounts, trackHubUsers)') parser.add_argument("-p","--perMonth", action='store_true', help='output \ file containing info on db/track/hub track use per month') parser.add_argument("-m","--monthYear", action='store_true', help='output \ file containing month/year pairs (e.g. "Mar 2017")') parser.add_argument("-j","--jsonOut", action='store_true', help='output \ json files for summary dictionaries') parser.add_argument("-t","--outputDefaults", action='store_true', help='output a file containing info on default track usage for top 15 most used assemblies') parser.add_argument("-o","--outDir", type=str, help='directory in which to place output files') args = parser.parse_args() # Print help message if no arguments are supplied if len(sys.argv) == 1: parser.print_help(sys.stderr) sys.exit(1) # File and directory options can't be used together. Catch this and exit. if args.fileName != None and args.dirName != None: print("-f/--fileName and -d/--dirName cannot be used together. Choose one and re-run.") sys.exit(1) # Catch it early if input file/directory doesn't exist and exit. if args.fileName: if not os.path.exists(args.fileName): print(args.fileName, "doesn't exist. Please run on a valid file.") exit(1) elif args.dirName: if not os.path.exists(args.dirName): print(args.dirName, "doesn't exist. Please run on a valid directory.") exit(1) if not any([args.dbCounts, args.dbUsers, args.dbUsage, args.trackUsers, args.trackCounts, args.trackUsage, args.trackHubUsers, args.trackHubCounts, args.trackHubUsage, args.allOutput]): print("You must specify at least one of: --allOutput, --dbCounts, --dbUsage, --dbUsers, --trackCounts, --trackUsers, --trackHubCounts, --trackHubUsers.") exit(1) # Setup output directory if args.outDir == None: # If an output directory is unspecified, then a new one with the current date/time is made currDateTime = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") os.makedirs(currDateTime) outDir = currDateTime else: # Otherwise, user supplied a output directory name outDir = args.outDir if not os.path.exists(outDir): # If specied output directory doesn't exist, create it os.makedirs(outDir) # If "all" option is set change everyone to True if args.allOutput == True: args.dbCounts = True args.dbUsers = True args.dbUsage = True args.trackUsers = True args.trackCounts = True args.trackUsage = True args.trackHubUsers = True args.trackHubCounts = True args.trackHubUsage = True # Make list of users to process and build out dictionaries for later usersToProcess = [False, False, False] # Pos 0 - process db users # Pos 1 - process track hub users # Pos 2 - process public track hub users # The reason I've done it this way is to allow people to output just the counts # dbs/tracks/hub tracks without outputting the users files. I could have just set # args.*Users = True if args.*Counts was true, but that would mean that the users file # would also be output, defeating the purpose of the option to only output the *Counts if any([args.dbCounts, args.dbUsers, args.dbUsage]): # If dbCounts or dbUsers speficied, we need to process users usersToProcess[0] = True if any([args.trackCounts, args.trackUsers, args.trackUsage]): usersToProcess[1] = True if any([args.trackHubCounts, args.trackHubUsers, args.trackHubUsage]): usersToProcess[1] = True usersToProcess[2] = True # Process input to create dictionaries containing info about users per db/track/hub track if args.fileName: processFile(args.fileName, usersToProcess, args.perMonth) elif args.dirName: processDir(args.dirName, usersToProcess, args.perMonth) ##### Output files of summaries of db/track/hub track usage information # Output files of db users and db use if args.dbCounts == True: dbCountsFile = open(os.path.join(outDir, "dbCounts.tsv"), "w") if args.dbUsers == True: dbUsersFile = open(os.path.join(outDir, "dbUsers.tsv"), "w") if args.dbUsage == True: dbUsageFile = open(os.path.join(outDir, "dbUsage.tsv"), "w") # Set of nested for loops to go through dictionary level by level and # output and summarize results into appropriate counts dictionary if any([args.dbUsage, args.dbUsers, args.dbCounts]): for db in dbUsers: dbCounts[db] = 0 dbUsage = 0 for hgsid in dbUsers[db]: # Ignore those dbs only used once if dbUsers[db][hgsid] != 1: dbCounts[db] += 1 # Output dbUsers info to a file count = dbUsers[db][hgsid] dbUsage += count if args.dbUsers == True: dbUsersFile.write(db + "\t" + hgsid + "\t" + str(count) + "\n") # Output counts of total users for each db if dbCounts[db] != 0: if args.dbCounts == True: dbCountsFile.write(db + "\t" + str(dbCounts[db]) + "\n") if args.dbUsage == True: dbUsageFile.write(db + "\t" + str(dbUsage) + "\n") # Close our output files if args.dbCounts == True: print("Finished outputting dbCounts.tsv") dbCountsFile.close() if args.dbUsers == True: print("Finished outputting dbUsers.tsv") dbUsersFile.close() if args.dbUsage == True: print("Finished outputting dbUsage.tsv") dbUsageFile.close() # Output files of track users and track use if args.trackCounts == True: trackCountsFile = open(os.path.join(outDir, "trackCounts.tsv"), "w") if args.trackUsers == True: trackUsersFile = open(os.path.join(outDir, "trackUsers.tsv"), "w") if args.trackUsage == True: trackUsageFile = open(os.path.join(outDir, "trackUsage.tsv"), "w") # Set of nested for loops to go through dictionary level by level and # output and summarize results into appropriate counts dictionary if any([args.trackUsage, args.trackUsers, args.trackCounts]): for db in trackUsers: for track in trackUsers[db]: # Initialize count for track to 0 trackCounts[db][track] = 0 trackUsage = 0 for hgsid in trackUsers[db][track]: # Filter out those hgsids who only used track once - likely to be bots if trackUsers[db][track][hgsid] != 1: trackCounts[db][track] += 1 # Output information on how much each user used each track count = trackUsers[db][track][hgsid] trackUsage += count if args.trackUsers == True: trackUsersFile.write(db + "\t" + track + "\t" + hgsid + "\t" + str(count) + "\n") if trackCounts[db][track] != 0: if args.trackCounts == True: # Output counts of total users for each track count = trackCounts[db][track] trackCountsFile.write(db + "\t" + track + "\t" + str(count) + "\n") if args.trackUsage == True: trackUsageFile.write(db + "\t" + track + "\t" + str(trackUsage) + "\n") # Close our output files if args.trackCounts == True: print("Finished outputting trackCounts.tsv") trackCountsFile.close() if args.trackUsers == True: print("Finished outputting trackUsers.tsv") trackUsersFile.close() if args.trackUsage == True: print("Finished outputting trackUsage.tsv") trackUsageFile.close() # Output files of hub track users and hub track use if args.trackHubUsers == True: trackUsersHubsFile = open(os.path.join(outDir, "trackUsersHubs.tsv"), "w") if args.trackHubCounts == True: trackCountsHubsFile = open(os.path.join(outDir, "trackCountsHubs.tsv"), "w") if args.trackHubUsage == True: trackUsageHubsFile = open(os.path.join(outDir, "trackUsageHubs.tsv"), "w") # Set of nested for loops to go through dictionary level by level and # output and summarize results into appropriate counts dictionary if any([args.trackHubUsage, args.trackHubUsers, args.trackHubCounts]): for hubId in trackUsersHubs: hubLabel = publicHubs[hubId][1] for db in trackUsersHubs[hubId]: for track in trackUsersHubs[hubId][db]: # Initialize count for hub track to 0 trackCountsHubs[hubId][db][track] = 0 trackHubUsage = 0 for hgsid in trackUsersHubs[hubId][db][track]: # Filter out those hgsids who only used track once - likely to be bots if trackUsersHubs[hubId][db][track][hgsid] != 1: trackCountsHubs[hubId][db][track] += 1 # Output information on how much each user used each track count = trackUsersHubs[hubId][db][track][hgsid] trackHubUsage += count if args.trackHubUsers == True: trackUsersHubsFile.write(hubLabel + "\t" + db + "\t" + track + "\t" + hgsid +\ "\t" + str(count) + "\n") if trackCountsHubs[hubId][db][track] != 0: if args.trackHubCounts == True: # Output counts of total users for each hub track count = trackCountsHubs[hubId][db][track] trackCountsHubsFile.write(hubLabel + "\t" + db + "\t" + track + "\t" +\ str(count) + "\n") if args.trackHubUsage == True: trackUsageHubsFile.write(hubLabel + "\t" + db + "\t" + track + "\t" +\ str(trackHubUsage) + "\n") # Close our output files if args.trackHubUsers == True: print("Finished outputting trackHubUsers.tsv") trackUsersHubsFile.close() if args.trackHubCounts == True: print("Finished outputting trackHubCounts.tsv") trackCountsHubsFile.close() if args.trackHubUsage == True: print("Finished outputting trackHubUsage.tsv") trackUsageHubsFile.close() # Output file containing info on month/years covered by stats if indicated if args.monthYear == True: monthYearFile = open(os.path.join(outDir, "monthYear.tsv"), "w") for pair in monthYearSet: monthYearFile.write(pair + "\n") print("Finished outputting monthYear.tsv") monthYearFile.close() ##### Output data per month when indicated if args.perMonth == True: if args.dbUsers == True: dbUsersMonthFile = open(os.path.join(outDir, "dbUsers.perMonth.tsv"), "w") if args.dbCounts == True: dbCountsMonthFile = open(os.path.join(outDir, "dbCounts.perMonth.tsv"), "w") if args.dbUsage == True: dbUsageMonthFile = open(os.path.join(outDir, "dbUsage.perMonth.tsv"), "w") # Set of nested for loops to go through dictionary level by level and # output and summarize results into appropriate counts dictionary if any([args.dbUsage, args.dbUsers, args.dbCounts]): for db in dbUsersMonth: for year in dbUsersMonth[db]: for month in dbUsersMonth[db][year]: dbCountsMonth[db][year][month] = 0 dbUsageMonth = 0 for hgsid in dbUsersMonth[db][year][month]: if dbUsersMonth[db][year][month][hgsid] != 1: dbCountsMonth[db][year][month] += 1 # Output dbUsersMonth info to a file count = dbUsersMonth[db][year][month][hgsid] dbUsageMonth += count if args.dbUsers == True: dbUsersMonthFile.write(db + "\t" + year + "\t" + month +\ "\t" + hgsid + "\t" + str(count) + "\n") # Output dbCounts to a file if dbCountsMonth[db][year][month] != 0: if args.dbCounts == True: count = dbCountsMonth[db][year][month] dbCountsMonthFile.write(db + "\t" + year + "\t" + month +\ "\t" + str(count) + "\n") if args.dbUsage == True: dbUsageMonthFile.write(db + "\t" + year + "\t" + month +\ "\t" + str(dbUsageMonth) + "\n") if args.dbUsers == True: print("Finished outputting dbUsers.perMonth.tsv") dbUsersMonthFile.close() if args.dbCounts == True: print("Finished outputting dbCounts.perMonth.tsv") dbCountsMonthFile.close() if args.dbUsage == True: print("Finished outputting dbUsage.perMonth.tsv") dbUsageMonthFile.close() # Summarize user dictionaries to create counts per db/track/hub track if args.trackUsers == True: trackUsersMonthFile = open(os.path.join(outDir, "trackUsers.perMonth.tsv"), "w") if args.trackCounts == True: trackCountsMonthFile = open(os.path.join(outDir, "trackCounts.perMonth.tsv"), "w") if args.trackUsage == True: trackUsageMonthFile = open(os.path.join(outDir, "trackUsage.perMonth.tsv"), "w") # Set of nested for loops to go through dictionary level by level and # output and summarize results into appropriate counts dictionary if any([args.trackUsage, args.trackUsers, args.trackCounts]): for db in trackUsersMonth: for year in trackUsersMonth[db]: for month in trackUsersMonth[db][year]: for track in trackUsersMonth[db][year][month]: trackCountsMonth[db][year][month][track] = 0 trackUsageMonth = 0 for hgsid in trackUsersMonth[db][year][month][track]: if trackUsersMonth[db][year][month][track][hgsid] != 1: trackCountsMonth[db][year][month][track] += 1 count = trackUsersMonth[db][year][month][track][hgsid] trackUsageMonth += count if args.trackUsers == True: trackUsersMonthFile.write(db + "\t" + year + "\t" + month + "\t" +\ hgsid + "\t" + track + "\t" + str(count) + "\n") if trackCountsMonth[db][year][month][track] != 0: if args.trackCounts == True: count = trackCountsMonth[db][year][month][track] trackCountsMonthFile.write(db + "\t" + year + "\t" + month + "\t" +\ track + "\t" + str(count) + "\n") if args.trackUsage == True: trackUsageMonthFile.write(db + "\t" + year + "\t" + month + "\t" +\ track + "\t" + str(trackUsageMonth) + "\n") if args.trackUsers == True: print("Finished outputting trackUsers.perMonth.tsv") trackUsersMonthFile.close() if args.trackCounts == True: print("Finished outputting trackUsers.perMonth.tsv") trackCountsMonthFile.close() if args.trackUsage == True: print("Finished outputting trackUsage.perMonth.tsv") trackUsageMonthFile.close() # Summarize user dictionaries to create counts per db/track/hub track if args.trackHubUsers == True: trackUsersHubsMonthFile = open(os.path.join(outDir, "trackUsersHubs.perMonth.tsv"), "w") if args.trackHubCounts == True: trackCountsHubsMonthFile = open(os.path.join(outDir, "trackCountsHubs.perMonth.tsv"), "w") if args.trackHubUsage == True: trackUsageHubsMonthFile = open(os.path.join(outDir, "trackUsageHubs.perMonth.tsv"), "w") # Set of nested for loops to go through dictionary level by level and # output and summarize results into appropriate counts dictionary if any([args.trackHubUsage, args.trackHubUsers, args.trackHubCounts]): for hubId in trackUsersHubsMonth: hubLabel = publicHubs[hubId][1] for db in trackUsersHubsMonth[hubId]: for year in trackUsersHubsMonth[hubId][db]: for month in trackUsersHubsMonth[hubId][db][year]: for track in trackUsersHubsMonth[hubId][db][year][month]: trackCountsHubsMonth[hubId][db][year][month][track] = 0 trackHubUsageMonth = 0 for hgsid in trackUsersHubsMonth[hubId][db][year][month][track]: if trackUsersHubsMonth[hubId][db][year][month][track][hgsid] != 1: trackCountsHubsMonth[hubId][db][year][month][track] += 1 count = trackUsersHubsMonth[hubId][db][year][month][track][hgsid] trackHubUsageMonth += count if args.trackHubUsers == True: trackUsersHubsMonthFile.write(hubLabel + "\t" + db + "\t" + year +\ "\t" + month + "\t" + track + "\t" + hgsid + "\t" + str(count) + "\n") if trackCountsHubsMonth[hubId][db][year][month][track] != 0: if args.trackHubCounts == True: count = trackCountsHubsMonth[hubId][db][year][month][track] trackCountsHubsMonthFile.write(hubLabel + "\t" + db + "\t" + year +\ "\t" + month + "\t" + track + "\t" +\ str(count) + "\n") if args.trackHubUsage == True: trackUsageHubsMonthFile.write(hubLabel + "\t" + db + "\t" + year +\ "\t" + month + "\t" + track + "\t" +\ str(trackHubUsageMonth) + "\n") if args.trackHubUsers == True: print("Finished outputting trackUsersHubs.perMonth.tsv") trackUsersHubsMonthFile.close() if args.trackHubCounts == True: print("Finished outputting trackCountsHubs.perMonth.tsv") trackCountsHubsMonthFile.close() if args.trackHubUsage == True: print("Finished outputting trackUsageHubs.perMonth.tsv") trackUsageHubsMonthFile.close() ##### ##### Output json files if indicated ##### if args.jsonOut == True: dumpToJson(dbCounts, "dbCounts.json", outDir) dumpToJson(trackCounts, "trackCounts.json", outDir) dumpToJson(trackCountsHubs, "trackCountsHubs.json", outDir) if args.perMonth == True: dumpToJson(dbCountsMonth, "dbCounts.perMonth.json", outDir) dumpToJson(trackCountsMonth, "trackCounts.perMonth.json", outDir) dumpToJson(trackCountsHubsMonth, "trackCountsHubs.perMonth.json", outDir) #if args.monthYear == True: # dumpToJson(monthYearSet, "monthYearSet.json") ##### ##### Output information on default track usage if indicated ##### if args.outputDefaults == True and all([args.dbCounts, args.trackCounts]): - # Sort dbs by most popular + # Sort dbs by most popular. Hub-backed dbs (e.g. hub_3671779_hs1) are skipped: hub ids + # are handed out per hgcentral, so the ids in the logs never match the ids hgTracks + # returns here and every track lookup below would miss. dbCountsSorted = sorted(dbCounts.items(), key=operator.itemgetter(1)) dbCountsSorted.reverse() + dbsToCheck = [db for db, useCount in dbCountsSorted if not db.startswith("hub_")] defaultCountsFile = open(os.path.join(outDir, "defaultCounts.tsv"), "w") - for x in range(0, 15): # Will only output the default track stats for the 15 most popular assemblies - db = dbCountsSorted[x][0] + for db in dbsToCheck[0:15]: # Only the default track stats for the 15 most popular assemblies dbOpt = "db=" + db # HGDB_CONF must be set here so that we use default tracks from beta, not dev # Dev can contain staged tracks that don't exist on RR, leading to errors later in script + # The space before dbOpt matters - without it cgiSpoof reads the whole thing as one + # variable and db is silently ignored, leaving every assembly with hg38's defaults. cmd = ["cd /usr/local/apache/cgi-bin && HGDB_CONF=$HOME/.hg.conf.beta ./hgTracks hgt.trackImgOnly=1 " + dbOpt] p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) cmdout, cmderr = p.communicate() - errText = cmderr.decode("ASCII") # Convert binary output into ACSII for processing - # Process stderr output as that's what contains the trackLog lines - splitErrText = errText.split("\n") - trackLog = splitErrText[0] # First element is trackLog line, second is CGI_TIME; only want trackLog - splitLine = trackLog.split(" ") - - # Build list of tracks - tracks = splitLine[4] - tracks = tracks.split(",") + # Hub shortLabels are user supplied, so an errAbort here can carry non-ASCII + errText = cmderr.decode("utf-8", errors="replace") + # hgTracks writes the visible track list to stderr as "trackLog N db hgsid t:vis,t:vis", + # split into ~800 byte blocks so Apache does not chop the lines. Every numbered block + # is part of the list; the trailing "trackLog position" line and the CGI_TIME and + # RESOURCE lines are not, and must not be parsed as tracks. + tracks = [] + mismatch = None + for errLine in errText.split("\n"): + splitLine = errLine.split(" ") + if len(splitLine) > 4 and splitLine[0] == "trackLog" and splitLine[1].isdigit(): + if splitLine[2] != db: + mismatch = splitLine[2] + tracks = [] + break + tracks.extend(splitLine[4].split(",")) + if not tracks: + # assemblyStatsCron.py reads these back out and puts them in its report. Writing + # to stderr would be pointless: the caller merges and discards it, and the cron + # sends its own stderr to /dev/null. + if mismatch: + warning = "asked hgTracks for " + db + " but it reported " + mismatch + else: + warning = "no usable trackLog output from hgTracks for " + db + defaultCountsFile.write("#WARNING\t" + warning + + ", its default tracks are missing from this report\n") + print("Warning: " + warning, file=sys.stderr) + continue dbUse = dbCounts[db] # output list to file that contains column headings defaultCountsFile.write("#db\ttrackName\ttrackUse\t% using\t% turning off\n#" + db + "\t" + str(dbUse) + "\n") defaultCounts = [] for track in tracks: if track == "": continue # Remove trailing characters track = track.split(":")[0] try: trackUse = trackCounts[db][track] relUse = (trackUse/dbUse)*100 relOff = ((dbUse - trackUse)/dbUse)*100 # Store all this info a in a list so that we can sort my most used tracks later defaultCounts.append([track, trackUse, relUse, relOff]) except KeyError: continue # Sort defaultCounts for current assembly by most used track first defaultCountsSorted = sorted(defaultCounts, key=operator.itemgetter(2)) defaultCountsSorted.reverse() # Output sorted defaultCounts to a file for line in defaultCountsSorted: track = line[0] use = line[1] on = line[2] off = line[3] output = "{}\t{}\t{:d}\t{:3.2f}\t{:3.2f}\n".format(db, track, use, on, off) defaultCountsFile.write(output) print("Finished outputting defaultCounts.tsv") defaultCountsFile.close() elif args.outputDefaults == True: if not any([args.dbCounts, args.trackCounts]): # Need both dbCounts and trackCounts to actually be populated with data to be able to do the default counts print("\nCannot output default tracks if either --trackCounts or --dbCounts is not set. Set both of these options and re-run") #### # Print output directory name, particularly useful if using default output directory names # based on day/time print("\nYour output is in", outDir) if __name__ == "__main__": main()