# allow these robots
User-agent: Googlebot
Disallow:
Allow: /
User-agent: Yahoo! Slurp
Disallow:
Allow: /
Crawl-delay: 5
User-agent: Slurp
Disallow:
Allow: /
Crawl-delay: 5
User-agent: bingbot
Disallow:
Allow: /
Crawl-delay: 5
User-agent: msnbot
Disallow:
Allow: /
Crawl-delay: 5
User-agent: msnbot-media
Disallow:
Allow: /
Crawl-delay: 5
User-agent: ia_archiver
Disallow:
Allow: /
Crawl-delay: 5
User-agent: Teoma
Disallow:
Allow: /
Crawl-delay: 5
User-agent: *
Disallow: /