# www.robotstxt.org/

# Block Apple's bot completely
User-agent: Applebot
Disallow: /

# Rules for all other bots
User-agent: *
Disallow: /admin/
Disallow: /accounts/
Disallow: /news/

# Allow all crawlers to access static files
Allow: /static/

# Set a reasonable crawl rate
Crawl-delay: 30