0bcea84e83ceef4cccba34202e1792e8ad05d980 lrnassar Fri Jul 31 18:09:53 2026 -0700 Adding monthly otto notifier that compares our VCEP hub spec versions against the ClinGen CSpec registry. refs #37795 New script src/hg/utils/otto/vcepVersions/checkVcepVersions.py scrapes the published version from each VCEP hub description page on hgdownload (ENIGMA BRCA1/BRCA2 and InSiGHT Lynch syndrome) and compares it to the current released version in the ClinGen CSpec registry, which embeds its specification list as inline JSON on the affiliation page. Silent when the versions agree, prints a report when a hub is behind or when the check itself fails. Only released specs for the genes a hub actually displays are compared, since an affiliation can carry specs for more genes than we show, on their own schedule. InSiGHT 50099 for instance also holds APC and MUTYH. Also adds the monthly crontab entry, mailing otto-group. diff --git src/hg/utils/otto/otto.crontab src/hg/utils/otto/otto.crontab index 2fe655dc489..9c32cadf840 100644 --- src/hg/utils/otto/otto.crontab +++ src/hg/utils/otto/otto.crontab @@ -1,184 +1,189 @@ # DO NOT EDIT THE OTTO CRONTAB FILE DIRECTLY! # This file is in the kent git repository: ~/kent/src/hg/utils/otto/otto.crontab # Please edit and commit changes to the git repository, # and then ssh otto@hgwdev and run crontab on the edited and checked in file, # for example crontab ~lrnassar/kent/src/hg/utils/otto/otto.crontab # To save the current crontab back into git: # crontab -l > ~max/kent/src/hg/utils/otto/otto.crontab # logout # cd ~max/kent/src/hg/utils/otto/ # git commit otto.crontab -m 'updating crontab' # Note that you can see the list of auto-pushes of files from hgwdev to the RR by running: # cat /etc/crontab # on Hgwdev. TMPDIR=/data/tmp ottomeister=otto-group@ucsc.edu #Format: Minute - Hour - DayofMonth month Day-of-week MAILTO=otto-group@ucsc.edu # Daily sandbox update 41 00 * * 1-7 /cluster/home/otto/bin/readOnlyKentMirror # save shell history and last log into log files in ~/lastLog/ # last day of the month to catch them just before log rotation 38 23 30 4,6,9,11 * ~/lastLog/lastLog.sh 38 23 31 1,3,5,7,8,10,12 * ~/lastLog/lastLog.sh 38 23 28 2 * ~/lastLog/lastLog.sh 38 23 29 2 * ~/lastLog/lastLog.sh ####################################################### ###################### OTTO ########################### ####################################################### SHELL=/bin/sh PATH=/cluster/bin/scripts:/cluster/bin/x86_64:/usr/local/bin:/bin:/usr/bin MAILTO=otto-group@ucsc.edu #md5sum checker script - makes sure otto scripts in hive match git tree 0 1 * * 1-5 ~/kent/src/hg/utils/otto/ottoCompareGitVsHiveFiles.py # PanelApp MAILTO=max@soe.ucsc.edu,otto-group@ucsc.edu 10 10 * * 2 /hive/data/outside/otto/panelApp/doPanelApp.py #DECIPHER - Reactivated 5/13/2022 by Lou 11 4 * * * /hive/data/outside/otto/decipher/decipherWrapper.sh #GWAS Catalog 41 4 * * 3 /hive/data/outside/otto/gwas/gwasWrapper.sh #GeneReviews 00 8 * * 2 /hive/data/outside/otto/geneReviews/geneReviewsWrapper.sh #dbVar 13 8 2 * * /hive/data/outside/otto/dbVar/dbVarWrapper.sh #Orphanet 10 7 10 * * /hive/data/outside/otto/orphanet/orphanetWrapper.sh #ClinVar MAILTO=otto-group@ucsc.edu 08 00 8 * * umask 002; /hive/data/outside/otto/clinvar/doUpdate.sh #MANE 11 5 * * 1 python3 /hive/data/outside/otto/mane/doMane.py #GenCC 8 16 * * 2 python3 /hive/data/outside/otto/genCC/doGenCC.py #G2P (Gene2Phenotype) - monthly on the 5th; G2P publishes near month-end, so an #early-month run catches the previous month's update (no-ops if unchanged) 22 6 5 * * /hive/data/outside/otto/g2p/g2pWrapper.sh #OMIM 17 4 * * * /hive/data/outside/otto/omim/omimWrapper.sh 2>&1 #LOVD 14 13 * * mon /hive/data/outside/otto/lovd/doUpdate.sh #MitoMap 55 8 * * 3 /hive/data/outside/otto/mitoMap/checkMitoMapUpdate.sh #RefSeq Historical update notifier (RM #35766) - silent unless NCBI has a new release 40 8 * * 3 /hive/data/outside/otto/refSeqHistorical/checkRefSeqHistoricalUpdate.sh # InSiGHT VCEP ClinVar curated variants weekly update 08 03 * * 2 umask 002; /hive/data/outside/otto/insight/doUpdate.sh +# VCEP hub spec version notifier (RM #37795) - monthly, silent unless one of our +# VCEP hub description pages is behind the ClinGen CSpec registry +MAILTO=otto-group@ucsc.edu +40 06 15 * * /hive/data/outside/otto/vcepVersions/checkVcepVersions.py + # UniProt massive 110+ database update, takes 3-4 days. #Format: Minute - Hour - DayofMonth month Day-of-week MAILTO=max@soe.ucsc.edu,otto-group@ucsc.edu 00 07 26 * * /hive/data/outside/otto/uniprot/doUpdate.sh run #wuhCor1 uniprot otto cron by Max, May 2021 00 4 * * * /hive/data/outside/otto/uniprot/covidCheck.sh #ClinGen 00 9 * * * /hive/data/outside/otto/clinGen/clinGenWrapper.sh #ClinGen CSpec 11 11 * * 3 /hive/data/outside/otto/clinGen/clinGenCspec/makeClinGenCspec.sh #VarChat 22 22 * * 4 /hive/data/outside/otto/varChat/varChatOtto.sh #vista 0 9 1 * * /hive/data/outside/otto/vista/vistaOtto.sh # CIViC update. Releases are typically on the 1st of the month 12 12 2 * * /hive/data/outside/otto/civic/doCivicUpdate.sh #GRC Incident MAILTO=hiram@soe.ucsc.edu,otto-group@ucsc.edu 33 06 * * * /hive/data/outside/otto/grcIncidentDb/runUpdate.sh makeItSo # liftOver request watch - sends email to configured address in hg.conf # e.g.: chainFileRequestEmail=... - running every 7 minutes 1,8,15,22,29,36,43,50,57 * * * * /hive/data/outside/otto/liftRequest/devAndRR.sh # GenArk update hgcentral.genark table, once a week on Tuesday # the script itself sends email for notices about activity 37 10 * * 2 /hive/data/inside/GenArk/updateHgcentral.sh makeItSo ##### this script is making a listing that is used by the pushRR.sh script. ##### this takes a significant amount of time to run, 2 or 3 hours. ##### Needs to be done before the pushRR.sh script below is running at 01:03 58 18 * * * /hive/data/inside/GenArk/pushRR/devList.sh ##### see also, in the source tree: src/hg/utils/otto/genArk/pushRR/README.txt # push out the /gbdb/hubs/GC[AF]/ hierarchy to: # hgwbeta hgw0 hgw1 hgw2 euroNode # There is an equivalent 'pull' script running on the asiaNode # as of 2025-07-24 this script now protects itself from overrunning itself # with a lock file. It will email when it is doing that. 03 01 * * * /hive/data/inside/GenArk/pushRR/pushRR.sh ### verify liftOverChain and quickLiftChain tables have ### identical fromDb,toDb pairs: 11 3 * * * /hive/data/inside/GenArk/checkChainTables/toFromCounts.sh #NCBI RefSeq 23 8 * * 3 /hive/data/outside/otto/ncbiRefSeq/ottoNcbiRefSeq.sh #Index page public session thumbnails update MAILTO=otto-group@ucsc.edu 11 1 * * 4 /cluster/software/bin/python /cluster/bin/scripts/buildPublicSessionThumbnailsIndexPage.py /usr/local/apache/htdocs/ #Add a new daily tip to the home page 5 5 * * 1-5 /cluster/software/bin/python /cluster/bin/scripts/generateTipOfDay.py -s ~/kent/src/hg/htdocs/allTipsRaw.html #OMIM Upload HGDB_CONF=/cluster/home/otto/.hg.conf.beta 15 05 * * * /hive/data/outside/otto/omim/omimUploadWrapper.sh MAILTO=cells@ucsc.edu # Cell Browser news sec update by Matt/Brittney Sept 2022 0 0 * * 1-5 umask 002; ~bwick/miniconda3/bin/python3.8 ~bwick/cellBrowser/ucsc/updateNewsSec -r # Update Cell Browser .json facets files using tabulate_facets.pl 0 0 * * 1-5 cd /usr/local/apache/htdocs-cells-submit/facets && ~bwick/cellBrowser/ucsc/tabulate_facets.pl /usr/local/apache/htdocs-cells-beta/dataset.json 0 0 * * 1-5 cd /usr/local/apache/htdocs-cells-submit/facets && /usr/local/apache/htdocs-cells-submit/facets/tsv_to_json.py # Malacards update. Not done monthly, but running monthly anyways, as they don't have a fixed schedule #Format: Minute - Hour - DayofMonth month Day-of-week MAILTO=mhaeussl@ucsc.edu 04 04 20 * * /hive/data/outside/otto/malacards/update.sh MAILTO=jbirgmei@ucsc.edu # PubTator3 dbSNP track update 28 08 * * * /bin/zsh -c 'cd /hive/data/outside/otto/pubtatorDbSnp && ./doUpdate.sh' MAILTO=cells@ucsc.edu @reboot umask 0000; /cluster/home/max/htdocs-cells-submit/tusd/tusd_linux_amd64/tusd -upload-dir /hive/data/inside/cells/upload/ -port 8443 -max-size 100000000000 # cbAnnotServer: Cell Browser login/annotation API (RM #37492). Keeps gunicorn # (127.0.0.1:5051, Apache proxies /api to it) alive; restarts if down / on reboot. * * * * * umask 002; CBANNOT_ENV_FILE=/hive/data/inside/cells/cbAnnotServer/cbAnnot.env ~mspeir/cellBrowser/src/cbAnnotServer/deploy/watchdog.sh >> /hive/data/inside/cells/cbAnnotServer/logs/watchdog.log 2>&1 @reboot umask 002; CBANNOT_ENV_FILE=/hive/data/inside/cells/cbAnnotServer/cbAnnot.env ~mspeir/cellBrowser/src/cbAnnotServer/deploy/watchdog.sh >> /hive/data/inside/cells/cbAnnotServer/logs/watchdog.log 2>&1