#http://www.httpblog.com/index.php?s=robots.txt
User-Agent: msnbot
Crawl-Delay: 10
Disallow: /*rev=*
Disallow: /*rev=*

# Yahoo
User-agent: Slurp
Crawl-delay: 30

# http://www.cuil.com/twiceler/robot.html
User-agent: Twiceler
Crawl-delay: 30

# Apparently Googlebot ignores Crawl-delay for now
# Can use Google's webmaster tools - I've created an account for frostie -
# and reduced crawl speed - Piers 8jan09
User-agent: Googlebot
Crawl-delay: 30
Disallow: /nets/*/browser/*rev=*
Disallow: /nets/*/log/*rev=*