4395aff08f55db6aea215e9717ceed746054840b lrnassar Wed Sep 2 15:34:32 2026 -0700 Fix and extend the monthly usage stats cron. refs #38232 Adds the GenArk and hubSpace reporting asked for in the ticket, and fixes several things that were quietly producing wrong numbers. New in the report: a summed GenArk row in the database usage table, a section ranking the top GenArk assemblies, and a closing section listing hubSpace usage with one row per user. The non-public hub list now drops GenArk and curated hubs, keeps only each hubSpace user's busiest hub, and reports track collections and ENCODE search hubs as one summed line each - previously a single person uploading a couple of hundred hubs took over most of the list. Fixes in generateUsageStats.py: a missing space meant db= was never passed to hgTracks, so every assembly was reported with hg38's default tracks. Only the first ~800 byte block of hgTracks' track list was read, so hg38 was analysed with 40 of its 91 defaults. Hub backed databases are now skipped, since hub ids differ between hgcentrals and their track names could never match the logs. Fixes in assemblyStatsCron.py: hgTracks was run from the wrong directory and died before logging anything, so the default track list was parsed out of timestamps and CGI_TIME lines - the "non-default track usage" section was really listing default tracks. The startup cleanup was deleting the hubStatus file that genome-asia copies over each month, which dropped asia-only hubs from the counts from February 2025 onward. The report label was computed as today minus 30 days, so the March run labelled itself January and overwrote it, and no February report has ever been published. Hub lookups no longer stop at the first 20 hubs, and are resolved by streaming each mirror's hubStatus once instead of grepping a 341MB file per hub. Counts for hubs are now keyed on the resolved hub URL rather than the track name, so unrelated hubs sharing a common track name are no longer added together. This moves some numbers: CADD reads 280 rather than 293, because three separate CADD download URLs were previously summed into one figure. Table columns, headers included, are now aligned in the emailed output. Warnings about an unreachable mirror or a missing asia file appear in the report body instead of a stderr stream that the cron discards. diff --git src/hg/logCrawl/dbTrackAndSearchUsage/generateUsageStats.py src/hg/logCrawl/dbTrackAndSearchUsage/generateUsageStats.py index 5d39cff6f60..d6990f214db 100755 --- src/hg/logCrawl/dbTrackAndSearchUsage/generateUsageStats.py +++ src/hg/logCrawl/dbTrackAndSearchUsage/generateUsageStats.py @@ -589,52 +589,75 @@ dumpToJson(trackCountsHubs, "trackCountsHubs.json", outDir) if args.perMonth == True: dumpToJson(dbCountsMonth, "dbCounts.perMonth.json", outDir) dumpToJson(trackCountsMonth, "trackCounts.perMonth.json", outDir) dumpToJson(trackCountsHubsMonth, "trackCountsHubs.perMonth.json", outDir) #if args.monthYear == True: # dumpToJson(monthYearSet, "monthYearSet.json") ##### ##### Output information on default track usage if indicated ##### if args.outputDefaults == True and all([args.dbCounts, args.trackCounts]): - # Sort dbs by most popular + # Sort dbs by most popular. Hub-backed dbs (e.g. hub_3671779_hs1) are skipped: hub ids + # are handed out per hgcentral, so the ids in the logs never match the ids hgTracks + # returns here and every track lookup below would miss. dbCountsSorted = sorted(dbCounts.items(), key=operator.itemgetter(1)) dbCountsSorted.reverse() + dbsToCheck = [db for db, useCount in dbCountsSorted if not db.startswith("hub_")] defaultCountsFile = open(os.path.join(outDir, "defaultCounts.tsv"), "w") - for x in range(0, 15): # Will only output the default track stats for the 15 most popular assemblies - db = dbCountsSorted[x][0] + for db in dbsToCheck[0:15]: # Only the default track stats for the 15 most popular assemblies dbOpt = "db=" + db # HGDB_CONF must be set here so that we use default tracks from beta, not dev # Dev can contain staged tracks that don't exist on RR, leading to errors later in script + # The space before dbOpt matters - without it cgiSpoof reads the whole thing as one + # variable and db is silently ignored, leaving every assembly with hg38's defaults. cmd = ["cd /usr/local/apache/cgi-bin && HGDB_CONF=$HOME/.hg.conf.beta ./hgTracks hgt.trackImgOnly=1 " + dbOpt] p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) cmdout, cmderr = p.communicate() - errText = cmderr.decode("ASCII") # Convert binary output into ACSII for processing - # Process stderr output as that's what contains the trackLog lines - splitErrText = errText.split("\n") - trackLog = splitErrText[0] # First element is trackLog line, second is CGI_TIME; only want trackLog - splitLine = trackLog.split(" ") - - # Build list of tracks - tracks = splitLine[4] - tracks = tracks.split(",") + # Hub shortLabels are user supplied, so an errAbort here can carry non-ASCII + errText = cmderr.decode("utf-8", errors="replace") + # hgTracks writes the visible track list to stderr as "trackLog N db hgsid t:vis,t:vis", + # split into ~800 byte blocks so Apache does not chop the lines. Every numbered block + # is part of the list; the trailing "trackLog position" line and the CGI_TIME and + # RESOURCE lines are not, and must not be parsed as tracks. + tracks = [] + mismatch = None + for errLine in errText.split("\n"): + splitLine = errLine.split(" ") + if len(splitLine) > 4 and splitLine[0] == "trackLog" and splitLine[1].isdigit(): + if splitLine[2] != db: + mismatch = splitLine[2] + tracks = [] + break + tracks.extend(splitLine[4].split(",")) + if not tracks: + # assemblyStatsCron.py reads these back out and puts them in its report. Writing + # to stderr would be pointless: the caller merges and discards it, and the cron + # sends its own stderr to /dev/null. + if mismatch: + warning = "asked hgTracks for " + db + " but it reported " + mismatch + else: + warning = "no usable trackLog output from hgTracks for " + db + defaultCountsFile.write("#WARNING\t" + warning + + ", its default tracks are missing from this report\n") + print("Warning: " + warning, file=sys.stderr) + continue dbUse = dbCounts[db] # output list to file that contains column headings defaultCountsFile.write("#db\ttrackName\ttrackUse\t% using\t% turning off\n#" + db + "\t" + str(dbUse) + "\n") defaultCounts = [] for track in tracks: if track == "": continue # Remove trailing characters track = track.split(":")[0] try: trackUse = trackCounts[db][track] relUse = (trackUse/dbUse)*100