6d3ac828268e706ba9dbc4baa01d444839c9aec2 braney Wed Sep 2 13:05:49 2026 -0700 trackDb: write the gbdb file list to $TMPDIR, not the scanned directory The parallel beta make (0b36c1276f7) aborted the v503 final build and both attempts at the #38231 build patch: /cluster/bin/x86_64/tdbQuery -check -release=beta -strict \ 'select count(*) from mm10' -root=.../trackDb No such file or directory stat failed in listDirX: .../trackDb/musFur1.gbdbList.txt make[1]: *** [makefile:281: mm10_beta] Error 255 Note the two different databases -- that is the signature. The %_beta recipe writes $*.gbdbList.txt into the trackDb directory, uses it, then removes it. loadTracks runs tdbQuery -check with -root set to that same directory, and listDirX stats every entry it finds there, so at -j 8 one database unlinks its list file between another database's readdir and stat. Naming each file after its database, which is what the original commit relied on, prevents two recipes from writing the same name but not one recipe from seeing another's file. Put the list in $TMPDIR instead, where nothing scans it, in all three recipes that build one (beta, publicTest, public). Fixing listDirX to tolerate ENOENT would not have helped: the recipe runs the installed /cluster/bin/x86_64/tdbQuery, not the binary the build compiles. This also keeps stray list files out of the working directory, where they have occasionally been picked up by a commit. The header comment claimed per-database naming was sufficient for -j; correct it and say where scratch files belong. Verified by expansion with make -n; a real parallel run has to wait for the next beta make. refs #35489 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> diff --git src/hg/makeDb/trackDb/makefile src/hg/makeDb/trackDb/makefile index 04c2c66a64c..65d71203ad5 100644 --- src/hg/makeDb/trackDb/makefile +++ src/hg/makeDb/trackDb/makefile @@ -1,34 +1,42 @@ kentSrc = ../../.. # Make your private trackDb with # make update # Your private trackDb with additional options, for example: # make EXTRA="-strict -settings" update # Make it for genome-test with # make alpha # Make for ENCODE reporting (includes release alpha and release beta) # make encodeReport # DO NOT RUN TWO OF THESE MAKES AT THE SAME TIME # Two makes using the same table names will fight over the same tables. # # Running ONE make with -j is fine, and is much faster, since the databases are -# independent of each other. Every per-database recipe writes only to files and -# tables named after that database, and hgTrackDb and hgFindSpec put their -# temporary files in $$TMPDIR under process-unique names. Pass -O as well so -# that the output of each database stays together in the log: +# independent of each other. Every per-database recipe writes only to tables +# named after that database, and hgTrackDb and hgFindSpec put their temporary +# files in $$TMPDIR under process-unique names. Pass -O as well so that the +# output of each database stays together in the log: # make -O -j 8 beta +# +# Naming a scratch file after its database is not by itself enough to make a +# recipe parallel-safe: loadTracks runs tdbQuery -check with -root set to this +# directory, and tdbQuery stats every entry it finds here. A file that one +# recipe creates and deletes is therefore visible to every other recipe, and +# unlinking it between another job's readdir and stat aborts that job with +# "stat failed in listDirX". Write per-database scratch files to $$TMPDIR (see +# GBDB_LIST below), not to this directory. # Browser supports multiple trackDb's so that individual developers # can change things rapidly without stepping on other people's toes. # Usually when updating it is best to update your own trackDb and # test it to make sure it works and that you have git updated all # of trackDb/ before doing a make alpha. Note that you # must specify which trackDb you are using in your .hg.conf file # or in the cgi-bin-$(USER)/hg.conf file. Something like: # db.trackDb=trackDb_YourUserName # note: new group ARCHIVED_DBS created for assemblies still needed # to support Conservation tracks in other assemblies. These should # have only chromInfo table on hgwbeta and RR. make will not rebuild # trackDb on hgwdev unless the db is specified explicitly on the # command line. @@ -206,30 +214,37 @@ rheMac1 \ danRer1 \ mm5 \ mm6 \ danRer2 # if trix build breaks, disable by setting to /bin/true instead of ./buildTrix # BUILD_TRIX = /bin/true BUILD_TRIX = ./buildTrix HIVE_TRIX = /hive/data/inside/trix DATA_TRIX = /data/trix ALPHA_MACHINE = hgwdev.gi.ucsc.edu BETA_MACHINE = qateam@hgwbeta.soe.ucsc.edu PUBLIC_MACHINE = qateam@hgw0.soe.ucsc.edu +# Where the per-database /gbdb file list goes. This must live outside the +# directory tdbQuery -check scans; see the note at the top of this file. It is +# named for the database and the user so that concurrent makes by two users do +# not share it. Recursively expanded on purpose: $* is only set inside the +# pattern recipes that use it. ${TMPDIR:-/tmp} is left for the shell. +GBDB_LIST = $${TMPDIR:-/tmp}/${USER}.$*.gbdbList.txt + # For the new demo-$USER machines, the "USER" passed in to makefile contains a hyphen which # needs to be replaced with an underscore for mysql table names. USERTABLE = $(shell echo ${USER} | sed -e 's/-/_/g') update: ${DBS:%=%_update} ${MKDIR} ${CGI_BIN}-${USER}/encode %_update: ./loadTracks -noHtmlCheck -strict -addVersion ${EXTRA} trackDb_${USERTABLE} hgFindSpec_${USERTABLE} $* ./checkMetaDb alpha metaDb_${USERTABLE} $* # now build the relatedTracks table for each db ./buildRelatedTracks relatedTrack_${USERTABLE} $* rm -rf /dev/shm/trackDbCache/$*.trackDb_${USERTABLE} ./maybeBuildHub ${USER} trackDb_${USERTABLE} $* # if you want to test track search tool with your own trix file @@ -263,62 +278,62 @@ betaTest_all: ${DBS:%=%_betaTest} beta: onbeta clean ${GIT} pull ${MAKE} beta_all beta_all: ${DBS:%=%_beta} %_betaTest: # now do loads on hgwbeta HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta ./loadTracks -addVersion ${EXTRA} -strict -remoteLogin=${BETA_MACHINE} -release=beta trackDb_betaTest hgFindSpec_betaTest $* %_beta: # now do loads on hgwbeta - -ssh ${BETA_MACHINE} find /gbdb/$* -type f -size +1c > $*.gbdbList.txt - HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta ./loadTracks -addVersion ${EXTRA} -strict -gbdbList=$*.gbdbList.txt -release=beta trackDb hgFindSpec $* - rm -f $*.gbdbList.txt + -ssh ${BETA_MACHINE} find /gbdb/$* -type f -size +1c > ${GBDB_LIST} + HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta ./loadTracks -addVersion ${EXTRA} -strict -gbdbList=${GBDB_LIST} -release=beta trackDb hgFindSpec $* + rm -f ${GBDB_LIST} HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta ./checkMetaDb beta metaDb $* # the -alsoTo also makes the files for hgwdev-beta (we share the tables with # hgwbeta), without building the same index a second time HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta HGDB_TRACKDB=trackDb ${BUILD_TRIX} -alsoTo=${ALPHA_MACHINE}:${HIVE_TRIX} trackDb metaDb cv/beta/cv.ra ${BETA_MACHINE} ${DATA_TRIX} $* # now build the relatedTracks table for each db HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta ./buildRelatedTracks relatedTrack $* -ssh ${BETA_MACHINE} rm -rf /dev/shm/trackDbCache/$*.trackDb HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta HGDB_TRACKDB=trackDb ./maybeBuildHub beta trackDb $* publicTest: onbeta clean ${MAKE} publicTest_all publicTest_all: ${DBS:%=%_publicTest} public: onbeta clean ${GIT} pull ${MAKE} public_all public_all: ${DBS:%=%_public} %_publicTest: - -ssh ${PUBLIC_MACHINE} find /gbdb/$* -type f -size +1c > $*.gbdbList.txt - HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta ./loadTracks -addVersion ${EXTRA} -strict -gbdbList=$*.gbdbList.txt -release=public trackDb_publicTest hgFindSpecTest_publicTest $* - rm -f $*.gbdbList.txt + -ssh ${PUBLIC_MACHINE} find /gbdb/$* -type f -size +1c > ${GBDB_LIST} + HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta ./loadTracks -addVersion ${EXTRA} -strict -gbdbList=${GBDB_LIST} -release=public trackDb_publicTest hgFindSpecTest_publicTest $* + rm -f ${GBDB_LIST} %_public: - -ssh ${PUBLIC_MACHINE} find /gbdb/$* -type f -size +1c > $*.gbdbList.txt - HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta ./loadTracks -addVersion ${EXTRA} -strict -gbdbList=$*.gbdbList.txt -release=public trackDb_public hgFindSpec_public $* - rm -f $*.gbdbList.txt + -ssh ${PUBLIC_MACHINE} find /gbdb/$* -type f -size +1c > ${GBDB_LIST} + HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta ./loadTracks -addVersion ${EXTRA} -strict -gbdbList=${GBDB_LIST} -release=public trackDb_public hgFindSpec_public $* + rm -f ${GBDB_LIST} HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta ./checkMetaDb public metaDb_public $* HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta HGDB_TRACKDB=trackDb_public ${BUILD_TRIX} trackDb_public metaDb_public cv/public/cv.ra ${BETA_MACHINE} ${DATA_TRIX} $* # now build the relatedTracks table for each db HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta ./buildRelatedTracks relatedTrack_public $* -ssh ${BETA_MACHINE} rm -rf /dev/shm/trackDbCache/$*.trackDb_public HGDB_CONF=/cluster/home/${USER}/.hg.conf.beta HGDB_TRACKDB=trackDb_public ./maybeBuildHub public trackDb_public $* # this will fail if we are not in a beta checkout: checkbeta: ${GIT} branch | egrep '^[*] v[0-9]+_branch' > /dev/null # not sure if anyone actually uses the target below. It used to be "beta". buildBeta: checkbeta clean strict encodeReport: