8ecaa12a2e01c4438e7e9290e6cfb657c5da9795
max
  Thu Aug 5 09:36:49 2021 -0700
changing robots.txt, refs  #27953

diff --git src/hg/htdocs/robots.rr.txt src/hg/htdocs/robots.rr.txt
index d6cda17..7309454 100644
--- src/hg/htdocs/robots.rr.txt
+++ src/hg/htdocs/robots.rr.txt
@@ -1,20 +1,7 @@
 User-agent: *
-Crawl-Delay: 1 # Slow'em down a little
-Disallow: /cgi-bin/
-Disallow: /admin/
-Disallow: /goldenPath/stats.html # Disallow for some gp files/dirs
-Disallow: /goldenPath/statistics/
+Disallow: /admin/stats/
 Disallow: /goldenPath/certificate.html
 Disallow: /goldenPath/certificates/
-Disallow: /goldenPath/gbdDescriptions.html
-Disallow: /goldenPath/gbdDescriptionsOld.html
-Disallow: /goldenPath/mapPlots/
-Disallow: /*.js$   # Disallow all files ending with these extensions 
-Disallow: /*.inc$
-Disallow: /*.css$
-Disallow: /*.gif$
-Disallow: /*.jpg$
-Disallow: /*.png$
 
 User-agent: Riddler
 Disallow: /