#
# Stop all other robots indexing "resource" directories
#
User-agent: *
Disallow: /cgi-bin
Disallow: /downloads
Disallow: /graphics
Disallow: /photos
Disallow: /resources
Disallow: /special
Disallow: /stats
#
# Ban Wayback Machine (to stop people nicking old resources)
#
User-agent: ia_archiver
Disallow: /
#
# Stop MSNbot and Inktomi Slurp from visiting too often
#
User-agent: msnbot
Crawl-delay: 120
#
User-agent: slurp
Crawl-delay: 120
#