# robots.txt for http://www.example.com/

User-agent: *
Crawl-delay: 5
Request-rate: 1/5
Disallow: /cgi-bin/