9ebab8b8598edcc9ef8cf70fc69ff0b419b7323f
greg
  Mon Jul 30 16:43:32 2012 -0700
Cleaned up file.  Removed redundant stuff. Removed complete block from goldenPath which made bots not index our help pages. Told bots not to index certain file types. And added slight crawl delay so they won't overwhelm servers.
diff --git src/hg/htdocs/robots.rr.txt src/hg/htdocs/robots.rr.txt
index d186621..e4cbfc9 100644
--- src/hg/htdocs/robots.rr.txt
+++ src/hg/htdocs/robots.rr.txt
@@ -1,20 +1,17 @@
 User-agent: *
-Disallow: /goldenPath/
-Disallow: /cgi-bin
-Disallow: //cgi-bin
-Disallow: /admin/
-Disallow: /qa/
-
-User-agent: msnbot Crawl-delay: 1200 Disallow: /
-
-User-agent: Googlebot
-Disallow: /admin/
-Disallow: /goldenPath/
-Disallow: /qa/
+Crawl-Delay: 1 # Slow'em down a little
 Disallow: /cgi-bin/
-
-User-agent: htdig
 Disallow: /admin/
-Disallow: /goldenPath/
-Disallow: /qa/
-Disallow: /cgi-bin/
+Disallow: /goldenPath/stats.html # Disallow for some gp files/dirs
+Disallow: /goldenPath/statistics/
+Disallow: /goldenPath/certificate.html
+Disallow: /goldenPath/certificates/
+Disallow: /goldenPath/gbdDescriptions.html
+Disallow: /goldenPath/gbdDescriptionsOld.html
+Disallow: /goldenPath/mapPlots/
+Disallow: /*.js$   # Disallow all files ending with these extensions 
+Disallow: /*.inc$
+Disallow: /*.css$
+Disallow: /*.gif$
+Disallow: /*.jpg$
+Disallow: /*.png$
\ No newline at end of file