User-agent: Slurp Crawl-delay: 300 # stop broken crawler from hammering us Disallow: /download/ # disallow indexing of humongous binary files Disallow: /updates/ User-agent: twiceler # asshats Disallow: / User-agent: IRLbot # more asshats Disallow: / User-agent: * # applies to all robots Disallow: /download/ # disallow indexing of humongous binary files Disallow: /updates/