06270582e562da3029320d8be4d527091d012602 lrnassar Fri Jul 31 20:03:55 2026 -0700 Fix and speed up trackCountsParse, the seldom-used track usage report. refs #37975 The script did not run at all, it aborted with a NameError before doing any work, so last year's report was produced by hand-editing a debug line. Also fixes -c and -n, which arrived from argparse as strings and raised TypeError, and -h, which aborted on an unescaped percent sign in a help string. Replaces the per-track tdbQuery calls with a single bulk tdbQuery dump, taking a six month run from hours down to a few minutes. The parent chain walk is now a loop rather than three fixed hops. The hub and custom track filter matches the start of the track name field instead of anywhere on the line, so real tracks such as dbVar_conflict_pathogenic are no longer silently dropped, and it uses awk rather than grep because grep -P failed to drop some rows in a 650k line report. averageTrackCount now averages every month searched rather than only the months in which a track fell under the cutoff. Adds averageSummedCount, monthsBelowCutoff and the per month cutoffs to the output, labels names with no trackDb entry as notInTrackDb, and adds -a/--asOfDate so an earlier window can be reproduced. Cached count files carry a v2 in the name so the previous filtering is not reused silently. diff --git src/utils/qa/trackCountsParse src/utils/qa/trackCountsParse index 07d5b851c45..1e401b2e48d 100755 --- src/utils/qa/trackCountsParse +++ src/utils/qa/trackCountsParse @@ -11,93 +11,195 @@ def parseArgs(): """ Parse the command line arguments. """ parser = argparse.ArgumentParser(description = __doc__, formatter_class=argparse.RawDescriptionHelpFormatter) optional = parser._action_groups.pop() required = parser.add_argument_group('required arguments') required.add_argument ("dbs", help = "Database to query for track counts, e.g. hg19, hg38, mm10.") required.add_argument ("workDir", help = "Work directory to use for processing and final output. Use full path with '/' at the end.") - optional.add_argument ("-c", "--cutOffThreshhold", dest = "cutOffThreshhold", default = .3, - help = "Optional: The % value, as compared to the trackCounts median, to be used " + \ + #The %% is doubled because argparse percent-formats help strings, a bare % aborts -h + optional.add_argument ("-c", "--cutOffThreshhold", dest = "cutOffThreshhold", type = float, default = .3, + help = "Optional: The %% value, as compared to the trackCounts median, to be used " + \ "as a threshhold to choose what tracks should be filtered. Default is .3.") - optional.add_argument ("-n", "--numOfMonthsToCompare", dest = "numOfMonthsToCompare", default = 6, + optional.add_argument ("-n", "--numOfMonthsToCompare", dest = "numOfMonthsToCompare", type = int, default = 6, help = "Optional: The number of months to compare for filtering. " + \ "Default is 6.") optional.add_argument ("-r", "--singleReport", default = False, action = "store_true", help = "Optional: Run as a singleReport. This generates track counts for the specified " + \ "dates. This is useful for seeing track counts over a period of time. Requires the vars below") optional.add_argument ("-s", "--startDate", dest = "startDate", help = "Optional: The start date when running in singleReport mode. " + \ "Date should be formatted as YYYY-MM-DD.") optional.add_argument ("-e", "--endDate", dest = "endDate", help = "Optional: The end date when running in singleReport mode. " + \ "Date should be formatted as YYYY-MM-DD.") + optional.add_argument ("-a", "--asOfDate", dest = "asOfDate", + help = "Optional: Run as if today were this month, formatted as YYYY-MM. Defaults to " + \ + "the current month. This moves the window of months searched, it does not move " + \ + "trackDb, which is always read as it is today. Tracks retired since the window " + \ + "are reported with a group of 'notInTrackDb'.") if (len(sys.argv) == 1): parser.print_usage() print("\nGenerates track counts based on the log parsing paring script '/hive/users/chmalee/logs/byDate/makeUsageReport'.\n" + \ "The default behavior looks at track counts over the last 6 months and generates a list of tracks that\n" + \ "were below 30% of the median track usage every month, allowing for a floor(n/4) exception where n is\n" + \ "the number of months searched. Using the default 6 months, that allows for 1 exemption.\n" + \ "This output can be used in order to identify seldomly used tracks for archiving, retiring or restructuring.\n\n" + \ "Alternatively, the script can be run in 'singleReport(-r)' mode where it will generate a sorted list\n" + \ "of track counts over a period of time. This could be useful for reporting purposes.\n\n" + \ "Example runs:\n" + \ " trackCountsParse hg38 /hive/users/lrnassar/trackCounts/\n" + \ " trackCountsParse hg38 /hive/users/lrnassar/trackCounts/ -c .5 -n 12\n" + \ " trackCountsParse hg38 /hive/users/lrnassar/trackCounts/ -r -s 2023-01-01 -e 2023-12-31\n") exit(0) parser._action_groups.append(optional) options = parser.parse_args() + if options.singleReport: + if options.asOfDate: + parser.error("-a/--asOfDate does not apply in -r/--singleReport mode, use -s and -e") + if not options.startDate or not options.endDate: + parser.error("-r/--singleReport needs both -s/--startDate and -e/--endDate") + if options.asOfDate: + try: + datetime.strptime(options.asOfDate, '%Y-%m') + except ValueError: + parser.error("-a/--asOfDate must be formatted as YYYY-MM, got '"+options.asOfDate+"'") + if options.numOfMonthsToCompare < 1: + parser.error("-n/--numOfMonthsToCompare must be at least 1") return options -def bash(cmd): - """Run the cmd in bash subprocess""" +def bash(cmd,mergeStderr=True): + """Run the cmd in bash subprocess. Callers that parse the output should pass + mergeStderr=False so that a warning cannot splice itself into a data line.""" try: rawBashOutput = subprocess.run(cmd, check=True, shell=True,\ - stdout=subprocess.PIPE, universal_newlines=True, stderr=subprocess.STDOUT) + stdout=subprocess.PIPE, universal_newlines=True,\ + stderr=subprocess.STDOUT if mergeStderr else subprocess.PIPE) bashStdoutt = rawBashOutput.stdout except subprocess.CalledProcessError as e: raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) return(bashStdoutt) def file_exists(filepath): return os.path.isfile(filepath) +#Tracks whose parent chain loops, so the warning is printed once and not once per month per track +loopsAlreadyWarnedAbout = set() + +def buildTrackDbCache(dbs): + """ + Dump the whole trackDb for a db in a single tdbQuery call and return a dictionary of + trackName -> {'shortLabel','group','parentName'}. Querying tdbQuery once per track + takes hours on a large assembly, one dump takes under a second. A track with no + 'parentName' key is top level. + """ + trackDbCache = {} + selectedKeys = ("track","shortLabel","group","parent","superTrack") + query = "tdbQuery -oneLine \"select track,shortLabel,group,parent,superTrack from "+dbs+"\"" + for line in bash(query,mergeStderr=False).split("\n"): + if not line.startswith("track "): + continue + settings = {} + lastKey = None + for field in line.split("|"): + if field == "": + continue + key = field.split(" ")[0] + #tdbQuery does not escape a '|' inside a value, so a shortLabel containing one + #splits into extra fields. Only a field starting with one of the keys we asked + #for begins a new setting, anything else is the rest of the previous value. + if key in selectedKeys and key not in settings: + settings[key] = field[len(key)+1:] + lastKey = key + elif lastKey: + settings[lastKey] += "|" + field + trackName = settings["track"] + trackDbCache[trackName] = {'shortLabel':settings.get("shortLabel",trackName), + 'group':settings.get("group","No group")} + #A subtrack points at its container with 'parent', a member of a superTrack points + #at it with 'superTrack <name>'. A superTrack itself carries 'superTrack on'. + if "parent" in settings: + trackDbCache[trackName]['parentName'] = settings["parent"].split(" ")[0] + elif "superTrack" in settings and settings["superTrack"].split(" ")[0] != "on": + trackDbCache[trackName]['parentName'] = settings["superTrack"].split(" ")[0] + print("trackDb entries cached for "+dbs+": "+str(len(trackDbCache))) + return(trackDbCache) + +def findTopLevelTrack(trackDbCache,trackName): + """ + Walk up the parent chain and return the track at the top. Tracks with no parent, and + tracks that are no longer in trackDb at all, are their own top level. This is a loop + rather than a fixed number of hops so that adding another layer of nesting to trackDb + does not silently strand counts on a middle container. + """ + currentTrack = trackName + visited = set() + while currentTrack in trackDbCache and 'parentName' in trackDbCache[currentTrack]: + if currentTrack in visited: + if currentTrack not in loopsAlreadyWarnedAbout: + loopsAlreadyWarnedAbout.add(currentTrack) + print("WARNING: parent loop in trackDb at track "+currentTrack) + break + visited.add(currentTrack) + currentTrack = trackDbCache[currentTrack]['parentName'] + return(currentTrack) + def generateTrackCounts(dbs,workDir,startDate,endDate): #Generate track usage report binned by month for a specific time frame and dbs #Format date format is XXXX-XX-XX, e.g. 2023-11-01 - outputFileName = workDir+dbs+"."+startDate+"to"+endDate+".trackCounts.txt" - #Run the script and remove the ct line (ct_), hub lines (hub_), dup lines (dup_) - #And header line (^#), as well as remove the first column of repeating database + #The v2 in the name marks counts made with the field anchored awk filter below. Older + #.trackCounts.txt files were filtered differently, and since this function reuses any + #file it finds, sharing the name would silently serve the old filtering forever. + outputFileName = workDir+dbs+"."+startDate+"to"+endDate+".trackCounts.v2.txt" + #Run the script and remove the ct lines (ct_), hub lines (hub_), dup lines (dup_) + #and header line (^#), as well as remove the first column of repeating database. + #The track name is the second column, so the prefixes are matched against the start + #of that field only. Matching them anywhere on the line, as this used to, also drops + #real tracks whose names merely contain the string, such as dbVar_conflict_pathogenic + #containing 'ct_'. This is awk rather than grep because grep -P silently failed to + #drop a few dozen hub_ lines out of the 650k that makeUsageReport emits for a month, + #while awk field matching is exact. The awk also does what the old 'cut -f2-' did. + awkFilter = "awk -F'\\t' 'BEGIN{OFS=\"\\t\"} $1!~/^#/ && $2!~/^(hub_|ct_|dup_)/ " + \ + "{out=$2; for(i=3;i<=NF;i++) out=out OFS $i; print out}'" + reportCmd = "/hive/users/chmalee/logs/byDate/makeUsageReport -t -db " + dbs + \ + " --bin-months -s " + startDate + " -e " + endDate + \ + " | " + awkFilter + " > " + outputFileName if not file_exists(outputFileName): print("Generating new file") - print("/hive/users/chmalee/logs/byDate/makeUsageReport -t -db " + dbs + " --bin-months -s " + startDate + " -e " + endDate + " | grep -v \"hub_\\|ct_\\|dup_\\|^#\" | cut -f2- > " + outputFileName) - cmd = ("/hive/users/chmalee/logs/byDate/makeUsageReport -t -db " + dbs + " --bin-months -s " + startDate + " -e " + endDate + " | grep -v \"hub_\\|ct_\\|dup_\\|^#\" | cut -f2- > " + outputFileName) - bash(cmd) + print(reportCmd) + bash(reportCmd) else: print("File already generated: " + outputFileName) + #An empty report means the log data does not cover these dates. Leaving the file behind + #would cache that emptiness, and an empty month aborts later on an out of range index. + if os.path.getsize(outputFileName) == 0: + os.remove(outputFileName) + sys.exit("Error: no track counts for "+startDate+" to "+endDate+". The usage logs may " + \ + "not reach back that far, or the weekly files for those dates may not be built yet. " + \ + "Check /hive/users/chmalee/logs/byDate/result/*/ for the dates you asked for.") + return(outputFileName) def createDicFromTrackCountsFile(trackCountsFilePath): """ This function reads a file containing track counts and creates a dictionary with track names as keys and their corresponding counts as values. Args: - trackCountsFilePath (str): Path to the file containing track counts. Returns: - dict: A dictionary containing track names as keys and their counts as values. """ trackList = open(trackCountsFilePath, 'r') trackCountsDic = {} @@ -105,192 +207,124 @@ parsedLine = line.rstrip().split("\t") if parsedLine[0].endswith(":"): currentTrackName = parsedLine[0][0:len(parsedLine[0])-1] else: currentTrackName = parsedLine[0] if currentTrackName not in trackCountsDic.keys(): trackCountsDic[currentTrackName] = {} trackCountsDic[currentTrackName]["Count"] = parsedLine[1] else: trackCountsDic[currentTrackName]["Count"] = str(int(trackCountsDic[currentTrackName]["Count"]) + int(parsedLine[1])) trackList.close() totalCount = len(trackCountsDic.keys()) print("Total tracks to parse: "+str(totalCount)) return(trackCountsDic,totalCount) -def checkIfThereIsAHigherLevelParentTrack(parentChildAssociationsDic,trackName,firstTry,dbs): - """ - Recursive function that searches for deeper associations and builds dictionary - """ - if firstTry == False: - currentParentName = parentChildAssociationsDic[trackName]['parentName'] - tdbQuery = bash("tdbQuery \"select * from "+dbs+" where track='"+currentParentName+"'\"").split("\n") - else: - tdbQuery = bash("tdbQuery \"select * from "+dbs+" where track='"+trackName+"'\"").split("\n") - if "compositeTrack" in str(tdbQuery) or "superTrack" in str(tdbQuery) or "parent" in str(tdbQuery): - for entry in tdbQuery: - if entry.startswith("parent"): - parentChildAssociationsDic[trackName]['Container'] = True #I ADDED THIS LINE IF THERE ARE ISSUES - parentName = entry.split(" ")[1] - parentChildAssociationsDic[trackName]['parentName'] = parentName - elif entry.startswith("superTrack"): - entry = entry.split(" ") - if entry[1] != "on": - parentChildAssociationsDic[trackName]['Container'] = True - parentName = entry[1] - parentChildAssociationsDic[trackName]['parentName'] = parentName - else: - if firstTry != False: - parentChildAssociationsDic[trackName]['Container'] = False - return(parentChildAssociationsDic) - -def lookUpTracksToFindParentChildAssociations(trackCountsDic,totalCount,parentChildAssociationsDic,dbs): - """ - This function takes a dictionary of track names and their details, - queries a database to find parent-child relationships for each track, - and updates the dictionary with the associated parent track information. - - Args: - - trackCountsDic (dict): A dictionary containing track names and their details. - - Returns: - - dict: An updated dictionary containing parent-child association information. - """ - n=0 - for trackName in trackCountsDic.keys(): - if trackName not in parentChildAssociationsDic.keys(): - parentChildAssociationsDic[trackName] = {} - n+=1 - if n%2000 == 0: - print(str(n)+" out of "+str(totalCount)) - #Make a first check to see if there are parents - parentChildAssociationsDic = checkIfThereIsAHigherLevelParentTrack(parentChildAssociationsDic,trackName,True,dbs) - #Check to see if there is a higher level parent - if 'parentName' in parentChildAssociationsDic[trackName].keys(): - parentChildAssociationsDic = checkIfThereIsAHigherLevelParentTrack(parentChildAssociationsDic,trackName,False,dbs) - #The top level tracks have container on, but no parent - else: - parentChildAssociationsDic[trackName]['Container'] = False - #Check to see if there is a final higher level parent - if 'parentName' in parentChildAssociationsDic[trackName].keys(): - parentChildAssociationsDic = checkIfThereIsAHigherLevelParentTrack(parentChildAssociationsDic,trackName,False,dbs) - - return(parentChildAssociationsDic) - -def buildFinalDicWithOnlyTopLevelTrackCounts(trackCountsDic,parentChildAssociationsDic): +def buildFinalDicWithOnlyTopLevelTrackCounts(trackCountsDic,trackDbCache): """ - Iterate through the dictionary of all track counts + parental relationships - and create a final dic that only includes all possible top-level tracks - with the highest possible count from any of its children. + Roll every track in the log counts up to its top level container and return a dictionary + of topLevelTrack -> {'max','sum'}. 'max' is the busiest single member and is the metric + the cutoff has always used. 'sum' adds every member together. A wide composite nearly + always has one busy child, so 'max' flatters it relative to a standalone track, while + 'sum' counts a single page view once per displayed subtrack. Neither is right on its + own, so both are carried through to the report. """ finalDicOfTopLevelTracksAndCounts = {} for trackName in trackCountsDic.keys(): - if parentChildAssociationsDic[trackName]['Container'] is False: - if trackName not in finalDicOfTopLevelTracksAndCounts.keys(): - finalDicOfTopLevelTracksAndCounts[trackName] = trackCountsDic[trackName]['Count'] - elif int(trackCountsDic[trackName]['Count']) > int(finalDicOfTopLevelTracksAndCounts[trackName]): - finalDicOfTopLevelTracksAndCounts[trackName] = trackCountsDic[trackName]['Count'] + topLevelTrack = findTopLevelTrack(trackDbCache,trackName) + count = int(trackCountsDic[trackName]['Count']) + if topLevelTrack not in finalDicOfTopLevelTracksAndCounts.keys(): + finalDicOfTopLevelTracksAndCounts[topLevelTrack] = {'max':count,'sum':count} else: - if parentChildAssociationsDic[trackName]['parentName'] not in finalDicOfTopLevelTracksAndCounts.keys(): - finalDicOfTopLevelTracksAndCounts[parentChildAssociationsDic[trackName]['parentName']] = trackCountsDic[trackName]['Count'] - else: - if int(trackCountsDic[trackName]['Count']) > int(finalDicOfTopLevelTracksAndCounts[parentChildAssociationsDic[trackName]['parentName']]): - finalDicOfTopLevelTracksAndCounts[parentChildAssociationsDic[trackName]['parentName']] = trackCountsDic[trackName]['Count'] + if count > finalDicOfTopLevelTracksAndCounts[topLevelTrack]['max']: + finalDicOfTopLevelTracksAndCounts[topLevelTrack]['max'] = count + finalDicOfTopLevelTracksAndCounts[topLevelTrack]['sum'] += count return(finalDicOfTopLevelTracksAndCounts) -def makeFinalFileOnTopLevelTrackCounts(finalDicOfTopLevelTracksAndCounts,pathUrl,dbs): +def makeFinalFileOnTopLevelTrackCounts(finalDicOfTopLevelTracksAndCounts,pathUrl,trackDbCache): """ This function creates a final output file containing details of top-level tracks, including their short labels (if available) and counts. The file is saved at the specified path URL. Args: - finalDicOfTopLevelTracksAndCounts (dict): A dictionary containing top-level track names and their respective counts. - pathUrl (str): The path where the final output file will be saved. """ outputFile = open(pathUrl, 'w') n=0 for key in finalDicOfTopLevelTracksAndCounts.keys(): if key != "": - tdbQuery = bash("tdbQuery \"select * from "+dbs+" where track='"+key+"'\"").split("\n") - if "shortLabel" in str(tdbQuery): - for entry in tdbQuery: - if entry.startswith("shortLabel"): - n+=1 - shortLabel = " ".join(entry.split(" ")[1:]) - outputFile.write(key+"\t"+shortLabel+"\t"+str(finalDicOfTopLevelTracksAndCounts[key])+"\n") - else: n+=1 - outputFile.write(key+"\t"+key+"\t"+str(finalDicOfTopLevelTracksAndCounts[key])+"\n") - print("Final file completed. Tota number of tracks: "+str(n)) + shortLabel = trackDbCache.get(key,{}).get('shortLabel',key) + outputFile.write(key+"\t"+shortLabel+"\t"+str(finalDicOfTopLevelTracksAndCounts[key]['max'])+"\n") + print("Final file completed. Total number of tracks: "+str(n)) outputFile.close() #Order and sort final file bash("sort -t $'\t' -k3 -rn "+pathUrl+" > "+pathUrl+".sorted") print("Final sorted file: "+pathUrl+".sorted") def get_count(dicField): return dicField['trackCounts'] -def makeOrderedDicForSpecificTime(finalDicOfTopLevelTracksAndCounts,dbs): - orderedDic = OrderedDict() +def makeOrderedDicForSpecificTime(finalDicOfTopLevelTracksAndCounts,trackDbCache): listOfCountsToSort = [] n=0 - #Fetch the shortLabels and make a list where each entry is a dic with trackName, shortLabel, and trackCount + #Attach the shortLabels and make a list where each entry is a dic with trackName, shortLabel, and trackCount for key in finalDicOfTopLevelTracksAndCounts.keys(): if key != "": - tdbQuery = bash("tdbQuery \"select * from "+dbs+" where track='"+key+"'\"").split("\n") - if "shortLabel" in str(tdbQuery): - for entry in tdbQuery: - if entry.startswith("shortLabel"): - n+=1 - shortLabel = " ".join(entry.split(" ")[1:]) - listOfCountsToSort.append({'trackName':key,'shortLabel':shortLabel,'trackCounts':int(finalDicOfTopLevelTracksAndCounts[key])}) - else: n+=1 - listOfCountsToSort.append({'trackName':key,'shortLabel':key,'trackCounts':int(finalDicOfTopLevelTracksAndCounts[key])}) + shortLabel = trackDbCache.get(key,{}).get('shortLabel',key) + listOfCountsToSort.append({'trackName':key,'shortLabel':shortLabel, + 'trackCounts':finalDicOfTopLevelTracksAndCounts[key]['max'], + 'summedCounts':finalDicOfTopLevelTracksAndCounts[key]['sum']}) print("Total number of tracks: "+str(n)) #Sort the trackCounts list to return in order to find data that meets cutoff threshhold listOfCountsToSort.sort(key=get_count, reverse=True) return(listOfCountsToSort) def refineTrackCountsBasedOnCutOff(listOfTracks,cutOffThreshhold,period): """ Take an ordered list containing dics of track counts and filter it based on a set threshhold. Then return a new ordered dictionary that contains the tracks below the threshhold with trackNames as keys and shortLabel - and counts as values + and counts as values. The cutoff itself is returned as well so it can be + recorded, the median moves a lot from year to year as the track list grows. """ trackCountCutoff = listOfTracks[int(len(listOfTracks)/2)]['trackCounts']*cutOffThreshhold finalTrackCountsDic = OrderedDict() for track in listOfTracks: if track['trackCounts'] < trackCountCutoff: finalTrackCountsDic[track['trackName']]={'shortLabel':track['shortLabel'],'trackCounts':track['trackCounts'],'countComparedToMaxForPeriod':track['trackCounts']/listOfTracks[0]["trackCounts"],'countComparedToCutoff':track['trackCounts']/trackCountCutoff} print("The trackCount cutoff for "+period+" is: "+str(trackCountCutoff)) - return(finalTrackCountsDic) + return(finalTrackCountsDic,trackCountCutoff) -def getDateRangesForComparison(numOfMonthsToCompare): +def getDateRangesForComparison(numOfMonthsToCompare,asOfDate): """ Based on a number of months to compare given, find the year + month combination followed by the last day of each month to be used in the log query script. Return an ordered dictionary with the date ranges as keys, which will be used as the titles of the respective final ouputs, and the start/end dates as the content. **Note** This subtracts an additional month from the latest month in order to ensure that the logs chosen are complete. """ dateRanges = OrderedDict() + if asOfDate: + date = datetime.strptime(asOfDate, '%Y-%m').strftime('%Y-%m') + else: date = datetime.today().strftime('%Y-%m') for number in range(numOfMonthsToCompare): monthToParse = datetime.strftime(datetime.strptime(date, '%Y-%m') - relativedelta(months=number+1), '%Y-%m') year = monthToParse.split('-')[0] month = monthToParse.split('-')[1] lastDateOfMonth = calendar.monthrange(int(year), int(month))[1] startDate = year+"-"+month+"-01" endDate = year+"-"+month+"-"+str(lastDateOfMonth) dateRanges[startDate+"-"+endDate] = {'startDate':startDate,'endDate':endDate} return(dateRanges) def createFinalListOfTracksThatMeetCutoffEveryMonth(finalDicWithCutOffDics,numOfMonthsToCompare): """ Iterates through all of the monthly dictionaries and creates a final list where only tracks present in every period are present. This is to filter @@ -320,102 +354,115 @@ if initialTrackList[track] <= outlierMonthsExcemption: listOfTracksInEligiblePeriods.append(track) else: listOfTracksFilteredOut.append(track) if listOfTracksFilteredOut != []: print("The following tracks were filtered out because they did not meet") print("the cutoff in all of the months specified:\n") for track in listOfTracksFilteredOut: print(track) return(listOfTracksInEligiblePeriods) def get_avCount(dicField): """Helper function to help sort dict""" return dicField['averageTrackCount'] -def constructSortedFinalTrackDicWithAllData(finalDicWithCutOffDics,listOfTracksInEligiblePeriods,numOfMonthsToCompare,dbs): +def constructSortedFinalTrackDicWithAllData(finalDicWithCutOffDics,allPeriodTopLevelCounts,listOfTracksInEligiblePeriods,numOfMonthsToCompare,trackDbCache): """ Takes in a final list of track names which have met all the conditions for potential archiving and constructs a final dictionary with all of the data sorted. This includes averages over the time period for the track counts as well as the comparison to median/max. The track group - is also queried for use in the ultimate decision. + is also reported for use in the ultimate decision. + + The count averages cover every month searched, using the full monthly counts rather than only + the months in which the track fell below the cutoff. Averaging only the qualifying months hides + the one good month a track was granted an exemption for, which is the month a reader most needs + to see. The two ratio averages still cover only the qualifying months, since a track above the + cutoff has no ratio recorded that month, so monthsBelowCutoff is reported next to them. """ - firstPeriod = next(iter(finalDicWithCutOffDics)) listOfTracksToReport = [] for track in listOfTracksInEligiblePeriods: - group = "" + #A name in the logs with no trackDb entry is either a track retired since, or noise + #from a malformed request. Either way it is not something anyone can act on, so say so. + if track in trackDbCache: + shortLabel = trackDbCache[track]['shortLabel'] + group = trackDbCache[track]['group'] + else: + shortLabel = track + group = "notInTrackDb" addedTrackCounts = 0 + addedSummedCounts = 0 addedCountComparedToMaxForPeriod = 0 addedCountComparedToCutoff = 0 - missingMonths = 0 #This checks for the floor(n/4) tolerance that tracks can be missing + monthsBelowCutoff = 0 for period in finalDicWithCutOffDics.keys(): + #Full counts come from every month, whether or not the track was under the cutoff + if track in allPeriodTopLevelCounts[period].keys(): + addedTrackCounts+=allPeriodTopLevelCounts[period][track]['max'] + addedSummedCounts+=allPeriodTopLevelCounts[period][track]['sum'] if track in finalDicWithCutOffDics[period].keys(): - if group == "": - shortLabel = finalDicWithCutOffDics[period][track]["shortLabel"] - tdbQuery = bash("tdbQuery \"select group from "+dbs+" where track='"+track+"'\"").split("\n") - if 'group' in str(tdbQuery): - group = tdbQuery[0].split(" ")[1] - else: - group = "No group" - addedTrackCounts+=finalDicWithCutOffDics[period][track]['trackCounts'] + monthsBelowCutoff+=1 addedCountComparedToMaxForPeriod+=finalDicWithCutOffDics[period][track]['countComparedToMaxForPeriod'] addedCountComparedToCutoff+=finalDicWithCutOffDics[period][track]['countComparedToCutoff'] - else: - missingMonths+=1 - averageTrackCount = round(addedTrackCounts/(numOfMonthsToCompare-missingMonths),2) - averageCountComparedToMaxForPeriod = round(addedCountComparedToMaxForPeriod/(numOfMonthsToCompare-missingMonths),4) - averageCountComparedToCutoff = round(addedCountComparedToCutoff/(numOfMonthsToCompare-missingMonths),2) - listOfTracksToReport.append({'trackName':track,'shortLabel':shortLabel,'group':group,'averageTrackCount':averageTrackCount,'averageCountComparedToMaxForPeriod':averageCountComparedToMaxForPeriod,'averageCountComparedToCutoff':averageCountComparedToCutoff}) + averageTrackCount = round(addedTrackCounts/numOfMonthsToCompare,2) + averageSummedCount = round(addedSummedCounts/numOfMonthsToCompare,2) + averageCountComparedToMaxForPeriod = round(addedCountComparedToMaxForPeriod/monthsBelowCutoff,4) + averageCountComparedToCutoff = round(addedCountComparedToCutoff/monthsBelowCutoff,2) + listOfTracksToReport.append({'trackName':track,'shortLabel':shortLabel,'group':group,'averageTrackCount':averageTrackCount,'averageSummedCount':averageSummedCount,'averageCountComparedToMaxForPeriod':averageCountComparedToMaxForPeriod,'averageCountComparedToCutoff':averageCountComparedToCutoff,'monthsBelowCutoff':monthsBelowCutoff}) listOfTracksToReport.sort(key=get_avCount, reverse=True) return(listOfTracksToReport) -def writeFinalTrackListToFile(finalOutputTrackDicToReport,workDir,dbs,cutOffThreshhold,numOfMonthsToCompare): +def writeFinalTrackListToFile(finalOutputTrackDicToReport,workDir,dbs,cutOffThreshhold,numOfMonthsToCompare,dateRanges,cutoffsPerPeriod): """ Take the final processed dictionary and write it out to a tsv file including the vars - used in the data generation. + used in the data generation. The file is named for the months actually searched. """ - date = datetime.today().strftime('%Y-%m') - monthToParse = datetime.strftime(datetime.strptime(date, '%Y-%m') - relativedelta(months=numOfMonthsToCompare), '%Y-%m') - fileNamePathStartToEndDate = workDir+monthToParse+"-"+date+"."+dbs+".tracksToArchive.tsv" + #dateRanges runs newest month first, so the oldest month searched is the last entry + newestMonth = dateRanges[next(iter(dateRanges))]['startDate'][0:7] + oldestMonth = dateRanges[next(reversed(dateRanges))]['startDate'][0:7] + fileNamePathStartToEndDate = workDir+oldestMonth+"-"+newestMonth+"."+dbs+".tracksToArchive.tsv" finalOutputFile = open(fileNamePathStartToEndDate,'w') finalOutputFile.write("#Variables used in this file generation: dbs="+dbs+" numOfMonthsToCompare="+str(numOfMonthsToCompare)+" cutOffThreshhold="+str(cutOffThreshhold)+"\n") - finalOutputFile.write("#trackName\tshortLabel\tgroup\taverageTrackCount\taverageCountComparedToMaxForPeriod\taverageCountComparedToCutoff\n") + finalOutputFile.write("#averageTrackCount averages every month searched. Reports made before " + \ + "2026-07 averaged only the months a track was under the cutoff, so the two are not directly comparable.\n") + finalOutputFile.write("#Monthly cutoffs, that month's median top level track count times the threshhold: "+ \ + ", ".join([period+"="+str(round(cutoffsPerPeriod[period],1)) for period in reversed(cutoffsPerPeriod)])+"\n") + finalOutputFile.write("#trackName\tshortLabel\tgroup\taverageTrackCount\taverageSummedCount\taverageCountComparedToMaxForPeriod\taverageCountComparedToCutoff\tmonthsBelowCutoff\n") for track in finalOutputTrackDicToReport: - finalOutputFile.write(track['trackName']+"\t"+track['shortLabel']+"\t"+track['group']+"\t"+str(track['averageTrackCount'])+"\t"+str(track['averageCountComparedToMaxForPeriod'])+"\t"+str(track['averageCountComparedToCutoff'])+"\n") + finalOutputFile.write(track['trackName']+"\t"+track['shortLabel']+"\t"+track['group']+"\t"+str(track['averageTrackCount'])+"\t"+str(track['averageSummedCount'])+"\t"+str(track['averageCountComparedToMaxForPeriod'])+"\t"+str(track['averageCountComparedToCutoff'])+"\t"+str(track['monthsBelowCutoff'])+"\n") finalOutputFile.close() print("\nCutoff tracks file complete: "+fileNamePathStartToEndDate) - print("\nYou nicely format the output as such: tail -n +2 outputFilePath.tsv | tabFmt stdin") + print("\nYou nicely format the output as such: tail -n +4 outputFilePath.tsv | tabFmt stdin") def main(): """Initialize options and call other functions""" options = parseArgs() - dbs,workDir,cutOffThreshhold,numOfMonthsToCompare = options.dbs,options.workDir,options.cutOffThreshhold,options.numOfMonthsToCompare - #Line below exists only for debugging purposes - # dbs,workDir,cutOffThreshhold,numOfMonthsToCompare,singleReport = 'hg38','/hive/users/lrnassar/temp/tmp/',.3,6,False + dbs,workDir,cutOffThreshhold,numOfMonthsToCompare,singleReport = options.dbs,options.workDir,\ + options.cutOffThreshhold,options.numOfMonthsToCompare,options.singleReport + trackDbCache = buildTrackDbCache(dbs) if singleReport == True: - startDate,endDate,parentChildAssociationsDic = options.startDate,options.endDate,{} + startDate,endDate = options.startDate,options.endDate logFile = generateTrackCounts(dbs,workDir,startDate,endDate) trackCountsDic,totalCount = createDicFromTrackCountsFile(logFile) - parentChildAssociationsDic = lookUpTracksToFindParentChildAssociations(trackCountsDic,totalCount,parentChildAssociationsDic,dbs) - finalDicOfTopLevelTracksAndCounts = buildFinalDicWithOnlyTopLevelTrackCounts(trackCountsDic,parentChildAssociationsDic) - makeFinalFileOnTopLevelTrackCounts(finalDicOfTopLevelTracksAndCounts,workDir+"trackCounts.tsv",dbs) + finalDicOfTopLevelTracksAndCounts = buildFinalDicWithOnlyTopLevelTrackCounts(trackCountsDic,trackDbCache) + makeFinalFileOnTopLevelTrackCounts(finalDicOfTopLevelTracksAndCounts,workDir+"trackCounts.tsv",trackDbCache) else: print("Script started: "+strftime("%Y-%m-%d %H:%M:%S", localtime())) - dateRanges = getDateRangesForComparison(numOfMonthsToCompare) - finalDicWithCutOffDics,parentChildAssociationsDic = OrderedDict(),{} + dateRanges = getDateRangesForComparison(numOfMonthsToCompare,options.asOfDate) + finalDicWithCutOffDics,allPeriodTopLevelCounts,cutoffsPerPeriod = OrderedDict(),{},OrderedDict() for period in dateRanges: logFile = generateTrackCounts(dbs,workDir,dateRanges[period]['startDate'],dateRanges[period]['endDate']) trackCountsDic,totalCount = createDicFromTrackCountsFile(logFile) - parentChildAssociationsDic = lookUpTracksToFindParentChildAssociations(trackCountsDic,totalCount,parentChildAssociationsDic,dbs) - finalDicOfTopLevelTracksAndCounts = buildFinalDicWithOnlyTopLevelTrackCounts(trackCountsDic,parentChildAssociationsDic) - listOfTracks = makeOrderedDicForSpecificTime(finalDicOfTopLevelTracksAndCounts,dbs) - finalDicWithCutOffDics[period] = refineTrackCountsBasedOnCutOff(listOfTracks,cutOffThreshhold,period) + finalDicOfTopLevelTracksAndCounts = buildFinalDicWithOnlyTopLevelTrackCounts(trackCountsDic,trackDbCache) + allPeriodTopLevelCounts[period] = finalDicOfTopLevelTracksAndCounts + listOfTracks = makeOrderedDicForSpecificTime(finalDicOfTopLevelTracksAndCounts,trackDbCache) + finalDicWithCutOffDics[period],cutoffsPerPeriod[period] = refineTrackCountsBasedOnCutOff(listOfTracks,cutOffThreshhold,period) print("The number of tracks that met the criteria for "+period+" is: "+str(len(finalDicWithCutOffDics[period].keys()))) listOfTracksInEligiblePeriods = createFinalListOfTracksThatMeetCutoffEveryMonth(finalDicWithCutOffDics,numOfMonthsToCompare) - finalOutputTrackDicToReport = constructSortedFinalTrackDicWithAllData(finalDicWithCutOffDics,listOfTracksInEligiblePeriods,numOfMonthsToCompare,dbs) - writeFinalTrackListToFile(finalOutputTrackDicToReport,workDir,dbs,cutOffThreshhold,numOfMonthsToCompare) + finalOutputTrackDicToReport = constructSortedFinalTrackDicWithAllData(finalDicWithCutOffDics,allPeriodTopLevelCounts,listOfTracksInEligiblePeriods,numOfMonthsToCompare,trackDbCache) + writeFinalTrackListToFile(finalOutputTrackDicToReport,workDir,dbs,cutOffThreshhold,numOfMonthsToCompare,dateRanges,cutoffsPerPeriod) print("Script finished: "+strftime("%Y-%m-%d %H:%M:%S", localtime())) main()