881817e2d9a77d5df168ae466beb23c8e6415c78
hiram
  Mon Aug 17 10:19:49 2026 -0700
updated script to eliminate the obsolete rsync commands refs #32596

diff --git src/hg/utils/otto/genArk/ncbiMirror/ncbiRsync.sh src/hg/utils/otto/genArk/ncbiMirror/ncbiRsync.sh
index e2e4df6fbd9..652f0836194 100755
--- src/hg/utils/otto/genArk/ncbiMirror/ncbiRsync.sh
+++ src/hg/utils/otto/genArk/ncbiMirror/ncbiRsync.sh
@@ -1,94 +1,97 @@
 #!/bin/bash
 
 usage() {
   printf "usage: ncbiRsync.sh <type>\n" 1>&2
   printf "where <type> is either GCF (==refseq) or GCA (==genbank)\n" 1>&2
   exit 255
 }
 
 if [ $# -ne 1 ]; then
   usage
 fi
 
 export type=$1
 
 if [ "$type" != "GCF" ]; then
   if [ "$type" != "GCA" ]; then
     printf "specified <type> must be either GCF or GCA\n" 1>&2
     usage
   fi
 fi 
 
 export HGDB_CONF="/cluster/home/hiram/.hg.conf"
 
 TOP="/hive/data/outside/ncbi/genomes/cronUpdates"
 
 cd $TOP
 
 YYYY=`date "+%Y"`
 MM=`date "+%m"`
 DD=`date "+%d"`
 DS=`date "+%F"`
 T=`date "+%T"`
 epoch=`date "+%s"`
 
 mkdir -p "$TOP/log/$YYYY/$MM"
 export logFile="$TOP/log/$YYYY/$MM/$DS.$epoch.$type.log"
 
 printf "### %s %s %s starting initial count scan\n" "${epoch}" "${DS}" "${T}" >> "${logFile}"
 
 cd /hive/data/outside/ncbi/genomes
 export countBefore=`find ./$type -mindepth 4 -maxdepth 4 -type d | awk -F'/' '{print $6}' | sort | wc -l`
 
 epoch=`date "+%s"`
 DS=`date "+%F"`
 T=`date "+%T"`
 LC_NUMERIC=en_US printf "### %s %s %s assemblies before rsync: %'d\n" "${epoch}" "${DS}" "${T}" "${countBefore}" >> "${logFile}"
 
+# rsync is obsolete at NCBI in 2025 - turned off due to robot abuse
+if [ 1 -eq 0 ]; then
 time ( rsync -avPL --stats --prune-empty-dirs --exclude "suppressed/" \
      --timeout=1200 \
      --include "*/" --include "chr2acc" --include "*_rm.run" \
      --include "README_patch_release.txt" --include "*_feature_count.txt.gz" \
      --include "*_feature_table.txt.gz" --include "*.bed" \
      --include "*_pseudo_without_product.fna.gz" \
      --include "chr*.fna.gz" --include "*_localID2acc" \
      --include "README_*annotation_release_*" --include "README.txt" \
      --include "*chr2scaf" --include "assembly_status*" \
      --include "*_assembly_stats.txt" --include "*_genomic.gbff.gz" \
      --include "*placement.txt" --include "*definitions.txt" \
      --include "*_protein.faa.gz" --include "*_rna.gbff.gz" \
      --include "*_rna.fna.gz" --include "*_genomic_gaps.txt.gz" \
      --include "*_rm.out.gz" --include "*.agp.gz" --include "*regions.txt" \
      --include "*_genomic.fna.gz" --include "*.gff.gz" --include "*.gff" \
      --include "*_assembly_report.txt" --include "alt.scaf.fna.gz" \
      --include "unplaced.scaf.fna.gz" --exclude "*_cds_from_genomic_fna.gz" \
      --exclude "*_rna_from_genomic_fna.gz" \
      --exclude "*" rsync://ftp.ncbi.nlm.nih.gov/genomes/all/$type/ ./$type/ ) \
            >> ${logFile} 2>&1
+fi
 
 epoch=`date "+%s"`
 DS=`date "+%F"`
 T=`date "+%T"`
 printf "### %s %s %s completed rsync\n" "${epoch}" "${DS}" "${T}" >> "${logFile}"
 
 export countAfter=`find ./$type -mindepth 4 -maxdepth 4 -type d | awk -F'/' '{print $6}' | sort | wc -l`
 export newCount=`echo $countAfter $countBefore | awk '{printf "%d", $1 - $2}'`
 epoch=`date "+%s"`
 DS=`date "+%F"`
 T=`date "+%T"`
 LC_NUMERIC=en_US printf "### %s %s %s assemblies after rsync: %'d - %'d = %'d new assemblies\n" "${epoch}" "${DS}" "${T}" "${countAfter}" "${countBefore}" "${newCount}" >> "${logFile}"
 
 # the extra tr and tail prevent gigantic outputs since a lot of the
 # log is the progress counter from rsync which is one gigantic line
 # separated by ctrl-M
 
 printf "#### head -3 logFile ####\n" 1>&2
 head -3 "${logFile}" | tr '
' '\n' | tail -25 1>&2
 printf "#### tail -25 logFile ####\n" 1>&2
 tail -25 "${logFile}" | tr '
' '\n' | tail -25 1>&2
 
 cd /hive/data/outside/ncbi/genomes/reports
 ./fetch.sh $type >> "${logFile}" 2>&1
 
 printf "#### tail -45 logFile ####\n" 1>&2
 tail -45 "${logFile}" | tr '
' '\n' | tail -45 1>&2