8c38547aec886c96cc0317da86140f7c12ef6b9f ann Thu Feb 16 17:06:23 2012 -0800 Jorge edited the RR version to disallow msnbot, and to clean up the cgi-bin entries (collapse down into one). diff --git src/hg/htdocs/robots.rr.txt src/hg/htdocs/robots.rr.txt index e19dfea..d186621 100644 --- src/hg/htdocs/robots.rr.txt +++ src/hg/htdocs/robots.rr.txt @@ -1,30 +1,20 @@ User-agent: * Disallow: /goldenPath/ Disallow: /cgi-bin Disallow: //cgi-bin Disallow: /admin/ Disallow: /qa/ +User-agent: msnbot Crawl-delay: 1200 Disallow: / + User-agent: Googlebot Disallow: /admin/ Disallow: /goldenPath/ Disallow: /qa/ -Disallow: /cgi-bin/hgTracks -Disallow: /cgi-bin/hgc -Disallow: /cgi-bin/hgTrackUi -Disallow: /cgi-bin/hgNear -Disallow: /cgi-bin/hgBlat -Disallow: /cgi-bin/hgLiftOver -Disallow: /cgi-bin/hgPcr -Disallow: /cgi-bin/hgTables -Disallow: /cgi-bin/hgText -Disallow: /cgi-bin/pbGateway -Disallow: /cgi-bin/pbGlobal +Disallow: /cgi-bin/ User-agent: htdig Disallow: /admin/ Disallow: /goldenPath/ Disallow: /qa/ -Disallow: /cgi-bin/hgTracks -Disallow: /cgi-bin/hgTables -Disallow: /cgi-bin/hgLiftOver +Disallow: /cgi-bin/