8c38547aec886c96cc0317da86140f7c12ef6b9f
ann
  Thu Feb 16 17:06:23 2012 -0800
Jorge edited the RR version to disallow msnbot, and to clean up the cgi-bin entries (collapse down into one).
diff --git src/hg/htdocs/robots.rr.txt src/hg/htdocs/robots.rr.txt
index e19dfea..d186621 100644
--- src/hg/htdocs/robots.rr.txt
+++ src/hg/htdocs/robots.rr.txt
@@ -1,30 +1,20 @@
 User-agent: *
 Disallow: /goldenPath/
 Disallow: /cgi-bin
 Disallow: //cgi-bin
 Disallow: /admin/
 Disallow: /qa/
 
+User-agent: msnbot Crawl-delay: 1200 Disallow: /
+
 User-agent: Googlebot
 Disallow: /admin/
 Disallow: /goldenPath/
 Disallow: /qa/
-Disallow: /cgi-bin/hgTracks
-Disallow: /cgi-bin/hgc
-Disallow: /cgi-bin/hgTrackUi
-Disallow: /cgi-bin/hgNear
-Disallow: /cgi-bin/hgBlat
-Disallow: /cgi-bin/hgLiftOver
-Disallow: /cgi-bin/hgPcr
-Disallow: /cgi-bin/hgTables
-Disallow: /cgi-bin/hgText
-Disallow: /cgi-bin/pbGateway
-Disallow: /cgi-bin/pbGlobal
+Disallow: /cgi-bin/
 
 User-agent: htdig
 Disallow: /admin/
 Disallow: /goldenPath/
 Disallow: /qa/
-Disallow: /cgi-bin/hgTracks 
-Disallow: /cgi-bin/hgTables
-Disallow: /cgi-bin/hgLiftOver
+Disallow: /cgi-bin/