# Allowing all sorts of Google crawlers. The crawl delay is set by Google's
# webmaster tools.
# Webmaster tools: http://www.google.com/webmasters/tools
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: Adsbot-Google
Allow: /

# Multiple crawlers can come from msnbot/1.1 AND msnbot/2.0b
# MS = Motherfucking tampon-Suckers.
# Webmaster tools: http://www.bing.com/webmaster
User-agent: msnbot/1.1
Crawl-delay: 300
Allow: /

User-agent: msnbot/2.0b
Crawl-delay: 300
Allow: /

# The Ask.com crawler will download only one page at a time from our site.
User-agent: Teoma
Crawl-delay: 5
Allow: /

# Multiple crawlers can come from Yahoo according to their documentation:
# http://help.yahoo.com/l/us/yahoo/search/indexing/slurp-03.html
# Webmaster tools: https://siteexplorer.search.yahoo.com/mysites
User-agent: Slurp
Crawl-delay: 300
Allow: /

# This is for the blog.limewire.com album image scraper. We may try and block
# all other crawlers
# with this user agent from other IP addresses other than our own at the load
# balancer level.
# See http://developer.yahoo.com/yql/provider/
User-agent: Yahoo Pipes 2.0
Allow: /

# Disallow all other assholes.
User-agent: *
Disallow: /