2aea926850613e4fcef435265d32564573fdb613
lrnassar
  Wed Aug 28 11:23:21 2024 -0700
Adding the runTrimLogs.sh script made by ChrisL to the tree, this has been running for a few years now. It processes the raw error logs into a better format that can be ingested by the various stats scripts such as the monthly usage reports. Primarily it makes sure that any error log lines belonging to a single cart does not span multiple lines. The second script is a small addition to check if the error logs are no longer updating. This has come up various times over the years, most recently botht he asia and euro logs had not updated since May, roughly 4.5 months old. No RM.

diff --git src/utils/qa/makefile src/utils/qa/makefile
index dada9b8..ad550a9 100644
--- src/utils/qa/makefile
+++ src/utils/qa/makefile
@@ -1,167 +1,169 @@
 kentSrc = ../..
 include ../../inc/common.mk
 # ${SCRIPTS} is ${HOME}/bin/scripts (defined in kent/src/inc/common.mk)
 # to install in /cluster/bin/scripts/, run (in this directory):
 # make SCRIPTS=/cluster/bin/scripts
 
 install:
 	@echo Reminder: is your sandbox up to date?
 	cp -p --remove-destination 2bitCompare             ${SCRIPTS}
 	cp -p --remove-destination activeGenbank.csh       ${SCRIPTS}
 	cp -p --remove-destination alertBqueue.csh         ${SCRIPTS}
 	cp -p --remove-destination allowTraffic.csh        ${SCRIPTS}
 	cp -p --remove-destination assemblyStatsCron.py    ${SCRIPTS}
 	cp -p --remove-destination backupCentral.csh       ${SCRIPTS}
 	cp -p --remove-destination bamToBedGraph.csh       ${SCRIPTS}
 	cp -p --remove-destination bamToBigWig.csh         ${SCRIPTS}
 	cp -p --remove-destination bigPush.sh              ${SCRIPTS}
 	cp -p --remove-destination blatPing.csh            ${SCRIPTS}
 	cp -p --remove-destination browserUsage.csh        ${SCRIPTS}
 	cp -p --remove-destination buildPublicSessionThumbnailsIndexPage.py        ${SCRIPTS}
 	cp -p --remove-destination catchBot.csh            ${SCRIPTS}
 	cp -p --remove-destination chain.csh               ${SCRIPTS}
 	cp -p --remove-destination chain2.csh              ${SCRIPTS}
 	cp -p --remove-destination chainNetTrio.csh        ${SCRIPTS}
 	cp -p --remove-destination checkAllStaticLinks.csh ${SCRIPTS}
 	cp -p --remove-destination checkBOT.csh            ${SCRIPTS}
 	cp -p --remove-destination checkCoverage.csh       ${SCRIPTS}
 	cp -p --remove-destination checkEncodeRegions.csh  ${SCRIPTS}
+	cp -p --remove-destination checkErrorLogsAreUpToDate.py ${SCRIPTS}
 	cp -p --remove-destination checkGbdbDiffs.csh      ${SCRIPTS}
 	cp -p --remove-destination checkGenomeMysql        ${SCRIPTS}
 	cp -p --remove-destination checkHitter.csh         ${SCRIPTS}
 	cp -p --remove-destination checkHTTPS              ${SCRIPTS}
 	cp -p --remove-destination checkHubSearchCronWrapper.py ${SCRIPTS}
 	cp -p --remove-destination checkMachineName.csh    ${SCRIPTS}
 	cp -p --remove-destination checkMetaData.csh       ${SCRIPTS}
 	cp -p --remove-destination checkMetaAday.csh       ${SCRIPTS}
 	cp -p --remove-destination checkOffend.csh         ${SCRIPTS}
 	cp -p --remove-destination checkPushedFiles.csh    ${SCRIPTS}
 	cp -p --remove-destination checkScaffoldGaps.csh   ${SCRIPTS}
 	cp -p --remove-destination checkSessionsFromRR.py  ${SCRIPTS}
 	cp -p --remove-destination checkStaticLinks.csh    ${SCRIPTS}
 	cp -p --remove-destination checkStoreUnits.csh     ${SCRIPTS}
 	cp -p --remove-destination checkSync.csh           ${SCRIPTS}
 	cp -p --remove-destination checkSyncAll.csh        ${SCRIPTS}
 	cp -p --remove-destination checkSyncAndMetaDataAll.py ${SCRIPTS}
 	cp -p --remove-destination checkSyncMetaDataWrapper.py ${SCRIPTS}
 	cp -p --remove-destination checkTableStatus.csh    ${SCRIPTS}
 	cp -p --remove-destination checkTableStatusDump.sh ${SCRIPTS}
 	cp -p --remove-destination checkTrackUiLinks.csh   ${SCRIPTS}
 	cp -p --remove-destination checkWigFiles.csh       ${SCRIPTS}
 	cp -p --remove-destination commTrio.csh            ${SCRIPTS}
 	cp -p --remove-destination compareHgFindSpecs.csh  ${SCRIPTS}
 	cp -p --remove-destination comparePublic.csh       ${SCRIPTS}
 	cp -p --remove-destination compareTableLists.csh   ${SCRIPTS}
 	cp -p --remove-destination compareTableToFile.csh  ${SCRIPTS}
 	cp -p --remove-destination compareTrackDbAll.csh   ${SCRIPTS}
 	cp -p --remove-destination compareTrackDbFast.csh  ${SCRIPTS}
 	cp -p --remove-destination compareTrackDbs.csh     ${SCRIPTS}
 	cp -p --remove-destination compareWholeColumn.csh  ${SCRIPTS}
 	cp -p --remove-destination compareWholeTable.csh   ${SCRIPTS}
 	cp -p --remove-destination copyExtSeqRows.csh      ${SCRIPTS}
 	cp -p --remove-destination copyHgcentral           ${SCRIPTS}
 	cp -p --remove-destination countPerChrom.csh       ${SCRIPTS}
 	cp -p --remove-destination countRows.csh           ${SCRIPTS}
 	cp -p --remove-destination cronLinks.csh           ${SCRIPTS}
 	cp -p --remove-destination databaseAday.csh        ${SCRIPTS}
 	cp -p --remove-destination dbUpdateTimes.csh       ${SCRIPTS}
 	cp -p --remove-destination doGenbankTests          ${SCRIPTS}
 	cp -p --remove-destination dropUserTables.csh      ${SCRIPTS}
 	cp -p --remove-destination dumpEmpty.csh           ${SCRIPTS}
 	cp -p --remove-destination filterCartDumps.sh      ${SCRIPTS}
 	cp -p --remove-destination findDbPartners.csh      ${SCRIPTS}
 	cp -p --remove-destination findBigFiles.csh        ${SCRIPTS}
 	cp -p --remove-destination findBlatServer.csh      ${SCRIPTS}
 	cp -p --remove-destination findBrokenLinks.csh     ${SCRIPTS}
 	cp -p --remove-destination findColumn.csh          ${SCRIPTS}
 	cp -p --remove-destination findKgTableSet.csh      ${SCRIPTS}
 	cp -p --remove-destination findLevel               ${SCRIPTS}
 	cp -p --remove-destination findMtNets.csh          ${SCRIPTS}
 	cp -p --remove-destination findNewTables.csh       ${SCRIPTS}
 	cp -p --remove-destination findOrg.csh             ${SCRIPTS}
 	cp -p --remove-destination findPushQLocks.csh      ${SCRIPTS}
 	cp -p --remove-destination gc-stats.pl             ${SCRIPTS}
 	cp -p --remove-destination genbankRun.csh          ${SCRIPTS}
 	cp -p --remove-destination generic_range_check.csh ${SCRIPTS} 
 	cp -p --remove-destination getAssemblies.csh       ${SCRIPTS}
 	cp -p --remove-destination getChainLines.csh       ${SCRIPTS}
 	cp -p --remove-destination getChromFieldName.csh   ${SCRIPTS}
 	cp -p --remove-destination getChromlist.csh        ${SCRIPTS}
 	cp -p --remove-destination getConservation.csh     ${SCRIPTS}
 	cp -p --remove-destination getDbSize.csh           ${SCRIPTS}
 	cp -p --remove-destination getField.csh            ${SCRIPTS}
 	cp -p --remove-destination getJoinerKeyErrors.pl   ${SCRIPTS}
 	cp -p --remove-destination getLastMonth.csh        ${SCRIPTS}
 	cp -p --remove-destination getLiftOver.csh         ${SCRIPTS}
 	cp -p --remove-destination getMatrixLines.csh      ${SCRIPTS}
 	cp -p --remove-destination getMonthLastYear.csh    ${SCRIPTS}
 	cp -p --remove-destination getOrderKey.csh         ${SCRIPTS}
 	cp -p --remove-destination getPubHubContact.sh     ${SCRIPTS}
 	cp -p --remove-destination getRRdatabases.csh      ${SCRIPTS}
 	cp -p --remove-destination getRRdumpfile.csh       ${SCRIPTS}
 	cp -p --remove-destination getRRtableStatus.csh    ${SCRIPTS}
 	cp -p --remove-destination getRRtables.csh         ${SCRIPTS}
 	cp -p --remove-destination getTableSize.csh        ${SCRIPTS}
 	cp -p --remove-destination getTableStatus.csh      ${SCRIPTS}
 	cp -p --remove-destination getTrackName.csh        ${SCRIPTS}
 	cp -p --remove-destination getSplit.csh            ${SCRIPTS}
 	cp -p --remove-destination getVersion.csh          ${SCRIPTS}
 	cp -p --remove-destination getYield.csh            ${SCRIPTS}
 	cp -p --remove-destination graph.csh               ${SCRIPTS}
 	cp -p --remove-destination hubCheckDraftEmails.py  ${SCRIPTS}
 	cp -p --remove-destination hubCheckPublicHubs.sh   ${SCRIPTS}
 	cp -p --remove-destination hubPublicMail           ${SCRIPTS}
 	cp -p --remove-destination hubPublicAutoUpdate     ${SCRIPTS}
 	cp -p --remove-destination hubSearchUpdate         ${SCRIPTS}
 	cp -p --remove-destination igtc.csh                ${SCRIPTS}
 	cp -p --remove-destination idHitter.csh            ${SCRIPTS}
 	cp -p --remove-destination imageComp.py            ${SCRIPTS}
 	cp -p --remove-destination jasparUpdate.py         ${SCRIPTS}
 	cp -p --remove-destination joinerBatchKeys.csh     ${SCRIPTS}
 	cp -p --remove-destination joinerBatchTimes.csh    ${SCRIPTS}
 	cp -p --remove-destination kgTables                ${SCRIPTS}
 	cp -p --remove-destination gsTables                ${SCRIPTS}
 	cp -p --remove-destination pbTables                ${SCRIPTS}
 	cp -p --remove-destination knownGene.csh           ${SCRIPTS}
 	cp -p --remove-destination linkCheckExclude        ${SCRIPTS}
 	cp -p --remove-destination longSessionCartDumps.sh ${SCRIPTS}
 	cp -p --remove-destination makeCytoBandIdeo.csh    ${SCRIPTS}
 	cp -p --remove-destination makeFilledBlockBed.csh  ${SCRIPTS}
 	cp -p --remove-destination makeHub.sh              ${SCRIPTS}
 	cp -p --remove-destination makeTouchCommands.csh   ${SCRIPTS}
 	cp -p --remove-destination makeUniProtFile.csh     ${SCRIPTS}
 	cp -p --remove-destination net.csh                 ${SCRIPTS}
 	cp -p --remove-destination pairLastzWrapper.py     ${SCRIPTS}
 	cp -p --remove-destination proteins.csh            ${SCRIPTS}
 	cp -p --remove-destination printMan.csh            ${SCRIPTS}
 	cp -p --remove-destination qaAutoTrack.sh          ${SCRIPTS}
 	cp -p --remove-destination qaConfig.csh            ${SCRIPTS}
 	cp -p --remove-destination qaConfig.bash           ${SCRIPTS}
 	cp -p --remove-destination qaGbTracks              ${SCRIPTS}
 	cp -p --remove-destination qaEncodeTracks.csh      ${SCRIPTS}
 	cp -p --remove-destination qaEnsGenes.csh          ${SCRIPTS}
 	cp -p --remove-destination qaTestScript.py         ${SCRIPTS}
 	cp -p --remove-destination qaThis.csh              ${SCRIPTS}
 	cp -p --remove-destination realTime.csh            ${SCRIPTS}
 	cp -p --remove-destination removeAlphas            ${SCRIPTS}
 	cp -p --remove-destination retirePushQ.csh         ${SCRIPTS}
 	cp -p --remove-destination runBits                 ${SCRIPTS}
 	cp -p --remove-destination runCheckHgFindSpec      ${SCRIPTS}
 	cp -p --remove-destination runJoiner.csh           ${SCRIPTS}
 	cp -p --remove-destination runQaAutoTrack.sh       ${SCRIPTS}
+	cp -p --remove-destination runTrimLogs.sh          ${SCRIPTS}
 	cp -p --remove-destination searchedTermsCron.py    ${SCRIPTS}
 	cp -p --remove-destination sessionStats.csh        ${SCRIPTS}
 	cp -p --remove-destination showLastGit.csh         ${SCRIPTS}
 	cp -p --remove-destination showProcesslist.csh     ${SCRIPTS}
 	cp -p --remove-destination staticpaths             ${SCRIPTS}
 	cp -p --remove-destination syncAday.csh            ${SCRIPTS}
 	cp -p --remove-destination trackCountsParse        ${SCRIPTS}
 	cp -p --remove-destination trackDbGlobal.csh       ${SCRIPTS}
 	cp -p --remove-destination updateTimes.csh         ${SCRIPTS}
 	cp -p --remove-destination updateTimesDb.sh        ${SCRIPTS}
 	cp -p --remove-destination userLookup              ${SCRIPTS}
 	cp -p --remove-destination whoMadeIt.csh           ${SCRIPTS}
 	@echo Reminder: was your sandbox up to date?
 
 clean::
 	@echo nothing to clean in src/utils/qa/