2332b3a609d7fc057db833551347b1be5f2555b4 hiram Fri Apr 7 10:19:16 2023 -0700 show total run time and construct trackDb when possible in the GenArk hub no redmine diff --git src/hg/utils/automation/pairLastz.sh src/hg/utils/automation/pairLastz.sh index 6408d77..0c05ca4 100755 --- src/hg/utils/automation/pairLastz.sh +++ src/hg/utils/automation/pairLastz.sh @@ -126,30 +126,35 @@ oDate=`egrep -m 1 -i "^#[[:space:]]*Date:" ${asmRpt} | tr -d '\r' | sed -e 's/.*ate: \+//;'` ;; *) oDate="" ;; esac printf "%s" "${oDate}" } # check if this database is actually a database browser or a promoted # hub. It could have a MySQL database, but it won't have a chromInfo function promotedHub() { export db=$1 } +############################################################################## +############################################################################## +### start seconds +export startT=`date "+%s"` + export target="$1" export query="$2" export tClade="$3" export qClade="$4" export tGcPath=$(gcPath $target) export qGcPath=$(gcPath $query) export tAsmId=$(asmId $target) export qAsmId=$(asmId $query) printf "# tq: '${target}' '${query}' '${tClade}' '${qClade}'\n" 1>&2 printf "# tq gcPath: '${tGcPath}' '${qGcPath}'\n" 1>&2 printf "# tq asmId: '${tAsmId}' '${qAsmId}'\n" 1>&2 # upper case first character export Target="${tAsmId^}" @@ -164,57 +169,61 @@ export swapDir="/hive/data/genomes/${query}/bed/blastz.${tAsmId}.swap" export queryExists="/hive/data/genomes/${query}/bed" export swapLink="/hive/data/genomes/${query}/bed/lastz.${tAsmId}" export targetSizes="/hive/data/genomes/${target}/chrom.sizes" export querySizes="/hive/data/genomes/${query}/chrom.sizes" export target2bit="/hive/data/genomes/${target}/${target}.2bit" export query2bit="/hive/data/genomes/${query}/${query}.2bit" export trackHub="" export rBestTrackHub="" export tRbestArgs="" export qRbestArgs="" export tSwapRbestArgs="" export qSwapRbestArgs="" export tFullName="" export qFullName="" +export tTdb="xxx" +export qTdb="xxx" # override those specifications if assembly hub case $target in GC[AF]_*) trackHub="-trackHub -noDbNameCheck" tFullName="-tAsmId $target" rBestTrackHub="-trackHub" buildDir="/hive/data/genomes/asmHubs/allBuild/${tGcPath}/${target}/trackData/lastz${Query}.${DS}" symLink="/hive/data/genomes/asmHubs/allBuild/${tGcPath}/${target}/trackData/lastz.${qAsmId}" targetExists="/hive/data/genomes/asmHubs/allBuild/${tGcPath}/${target}/trackData" targetSizes="/hive/data/genomes/asmHubs/${tGcPath}/${tAsmId}/${tAsmId}.chrom.sizes.txt" target2bit="/hive/data/genomes/asmHubs/${tGcPath}/${tAsmId}/${tAsmId}.2bit" + tTdb="/hive/data/genomes/asmHubs/allBuild/${tGcPath}/${target}/doTrackDb.bash" tRbestArgs="-target2Bit=\"${target2bit}\" \\ -targetSizes=\"${targetSizes}\"" tSwapRbestArgs="-query2bit=\"${target2bit}\" \\ -querySizes=\"${targetSizes}\"" ;; esac case $query in GC[AF]_*) trackHub="-trackHub -noDbNameCheck" qFullName="-qAsmId $query" rBestTrackHub="-trackHub" swapDir="/hive/data/genomes/asmHubs/allBuild/${qGcPath}/${query}/trackData/blastz.${tAsmId}.swap" swapLink="/hive/data/genomes/asmHubs/allBuild/${qGcPath}/${query}/trackData/lastz.${tAsmId}" queryExists="/hive/data/genomes/asmHubs/allBuild/${qGcPath}/${query}/trackData" querySizes="/hive/data/genomes/asmHubs/${qGcPath}/${qAsmId}/${qAsmId}.chrom.sizes.txt" query2bit="/hive/data/genomes/asmHubs/${qGcPath}/${qAsmId}/${qAsmId}.2bit" + qTdb="/hive/data/genomes/asmHubs/allBuild/${qGcPath}/${query}/doTrackDb.bash" qRbestArgs="-query2Bit=\"${query2bit}\" \\ -querySizes=\"${querySizes}\"" qSwapRbestArgs="-target2bit=\"${query2bit}\" \\ -targetSizes=\"${querySizes}\"" ;; esac if [ ! -d "${targetExists}" ]; then printf "ERROR: can not find ${targetExists}\n" 1>&2 exit 255 fi if [ ! -d "${queryExists}" ]; then printf "ERROR: can not find ${queryExists}\n" 1>&2 @@ -239,68 +248,69 @@ if [ ! -s "${querySizes}" ]; then printf "ERROR: can not find ${querySizes}\n" 1>&2 exit 255 fi export doneCount=0 export primaryDone=0 export swapDone=0 if [ -L "${symLink}" ]; then printf "# ${query} -> ${target} already done\n" 1>&2 doneCount=`echo $doneCount | awk '{printf "%d", $1+1}'` primaryDone=1 else - printf "# no symLink: $symLink\n" 1>&2 + printf "# primaryDone $primaryDone no symLink: $symLink\n" 1>&2 fi if [ -L "${swapLink}" ]; then printf "# swap ${query} -> ${target} already done\n" 1>&2 doneCount=`echo $doneCount | awk '{printf "%d", $1+1}'` swapDone=1 fi export working=`ls -d ${targetExists}/lastz${Query}.* 2> /dev/null | wc -l` if [ "${working}" -gt 0 ]; then if [ "${primaryDone}" -eq 0 ]; then printf "# in progress, ${query} -> ${target}:\n" 1>&2 printf "# " 1>&2 ls -ogd ${targetExists}/lastz${Query}.* 1>&2 fi buildDir=`ls -d ${targetExists}/lastz${Query}.*` printf "# continuing: %s\n" "${buildDir}" 1>&2 fi export primaryPartsDone=`ls $buildDir/fb.* 2> /dev/null | wc -l` if [ "$primaryPartsDone" -gt 0 ]; then primaryDone="$primaryPartsDone" + printf "# primaryPartsDone $primaryPartsDone primaryDone $primaryDone\n" 1>&2 fi if [ -d "${swapDir}" ]; then printf "# swap in progress ${query} -> ${target}\n" 1>&2 printf "# " 1>&2 ls -ogd "${swapDir}" 1>&2 if [ "${doneCount}" -ne 2 ]; then + printf "# doneCount $doneCount -ne 2 exit 0 since swap in progress\n" 1>&2 exit 0 fi fi if [ "${doneCount}" -eq 2 ]; then printf "# all done\n" 1>&2 fi - export tOrgName="$(orgName $target)" export qOrgName="$(orgName $query)" export tOrgDate="$(orgDate $target)" export qOrgDate="$(orgDate $query)" export tAsmSize="$(asmSize $target)" export qAsmSize="$(asmSize $query)" export tSequenceCount="$(seqCount $target)" export qSequenceCount="$(seqCount $query)" printf "# working: %s\n" "${buildDir}" 1>&2 printf "# target: $target - $tOrgName - $tOrgDate - $tClade - $tSequenceCount sequences\n" 1>&2 printf "# query: $query - $qOrgName - $qOrgDate - $qClade - $qSequenceCount sequences\n" 1>&2 LC_NUMERIC=en_US printf "# sizes: target: %'d - query: %'d\n" "${tAsmSize}" "${qAsmSize}" 1>&2 export seq1Limit="40" if [ "${tSequenceCount}" -gt 50000 ]; then @@ -437,30 +447,37 @@ ${tRbestArgs} \\ ${qRbestArgs} \\ \${targetDb} \${queryDb}) > rbest.log 2>&1 grep -w real rbest.log | sed -e 's/^/ # /;' sed -e 's/^/ #/;' fb.\${targetDb}.chainRBest.\${QueryDb}.txt " > ${buildDir}/run.sh chmod +x ${buildDir}/run.sh ### run the primary alignment printf "running: time (${buildDir}/run.sh) >> ${buildDir}/do.log 2>&1\n" 1>&2 time (${buildDir}/run.sh) >> ${buildDir}/do.log 2>&1 +# rebuild trackDb if possible here +if [ -x "${tTdb}" ]; then + ${tTdb} +else + printf "# do not find tTdb '%s'\n" "${tTdb}" 1>&2 +fi + fi ### if [ $primaryDone -eq 0 ]; then #### print out the makeDoc.txt to this point into buildDir/makeDoc.txt printf "############################################################################## # LASTZ ${tOrgName} ${Target} vs. $qOrgName ${Query} # (DONE - $DS - $userName) mkdir $buildDir cd $buildDir printf '${defString} ' > DEF time (~/kent/src/hg/utils/automation/doBlastzChainNet.pl ${trackHub} -verbose=2 \`pwd\`/DEF -syntenicNet \\ @@ -523,30 +540,38 @@ time (~/kent/src/hg/utils/automation/doRecipBest.pl ${rBestTrackHub} -load -workhorse=hgwdev -buildDir=\`pwd\` \\ ${tSwapRbestArgs} \\ ${qSwapRbestArgs} \\ \${queryDb} \${targetDb}) > rbest.log 2>&1 grep -w real rbest.log | sed -e 's/^/ # /;' sed -e 's/^/ # /;' fb.\${queryDb}.chainRBest.\${Target}.txt " > ${swapDir}/runSwap.sh chmod +x ${swapDir}/runSwap.sh printf "# running ${swapDir}/runSwap.sh\n" 1>&2 time (${swapDir}/runSwap.sh) >> ${swapDir}/doSwap.log 2>&1 + +# rebuild trackDb if possible here +if [ -x "${qTdb}" ]; then + ${qTdb} +else + printf "# do not find qTdb '%s'\n" "${qTdb}" 1>&2 +fi + fi ### if [ "$swapDone" -eq 0 ]; then ### continue the make doc printf "\n cd ${swapDir}\n" >> ${buildDir}/makeDoc.txt printf "\n time (~/kent/src/hg/utils/automation/doBlastzChainNet.pl ${trackHub} -swap -verbose=2 \\ ${tFullName} ${qFullName} ${buildDir}/DEF -swapDir=\`pwd\` \\ -syntenicNet -workhorse=hgwdev -smallClusterHub=hgwdev -bigClusterHub=ku \\ -chainMinScore=${minScore} -chainLinearGap=${linearGap}) > swap.log 2>&1 grep -w real swap.log | sed -e 's/^/ # /;' " >> ${buildDir}/makeDoc.txt (grep -w real ${swapDir}/swap.log || true) | sed -e 's/^/ # /;' >> ${buildDir}/makeDoc.txt @@ -560,31 +585,46 @@ printf "\ time (~/kent/src/hg/utils/automation/doRecipBest.pl ${rBestTrackHub} -load -workhorse=hgwdev -buildDir=\`pwd\` \\ ${tSwapRbestArgs} \\ ${qSwapRbestArgs} \\ ${qAsmId} ${tAsmId}) > rbest.log 2>&1 grep -w real rbest.log | sed -e 's/^/ # /;'\n" >> ${buildDir}/makeDoc.txt (grep -w real ${swapDir}/rbest.log || true) | sed -e 's/^/ # /;' >> ${buildDir}/makeDoc.txt printf "\n sed -e 's/^/ # /;' fb.${qAsmId}.chainRBest.${Target}.txt\n" >> ${buildDir}/makeDoc.txt (sed -e 's/^/ # /;' ${swapDir}/fb.${qAsmId}.chainRBest.${Target}.txt || true) >> ${buildDir}/makeDoc.txt printf "\n##############################################################################\n" >> ${buildDir}/makeDoc.txt ### show completed makeDoc.txt #### cat ${buildDir}/makeDoc.txt +### end seconds +export endT=`date "+%s"` + export toAddress="$userName" export fromAddress="$userName" export subject="pair lastz DONE $target $query" printf "To: $toAddress From: $fromAddress Subject: $subject ################################################################## " > /tmp/send.txt.$$ date >> /tmp/send.txt.$$ printf "##################################################################\n" >> /tmp/send.txt.$$ cat ${buildDir}/makeDoc.txt >> /tmp/send.txt.$$ +### show elapsed time +printf "%s\t%s\n" "${endT}" "${startT}" | awk -F$'\t' '{ +seconds=$1-$2 +hours=int(seconds/3600) +minutes=int((seconds-(hours*3600))/60) +s=seconds % 60 +printf "### elapsed time: %02dh %02dm %02ds\n\n", hours, minutes, s +}' >> /tmp/send.txt.$$ + +date >> /tmp/send.txt.$$ +printf "##################################################################\n" >> /tmp/send.txt.$$ + cat /tmp/send.txt.$$ | /usr/sbin/sendmail -t -oi rm -f /tmp/send.txt.$$