robots.txt:
#www.robotstxt.org
#www.user-agents.org

User-agent: *
Disallow: /logs/


User-agent: ia_archiver
Disallow: /
User-agent: Arquivo-web-crawler (compatible; heritrix/1.12.1 +http://arquivo-web.fccn.pt)
Disallow: /
User-agent: Bitacle bot/1.1
Disallow: /
User-agent: Bitacle Robot (V:1.0;) (http://www.bitacle.com)
Disallow: /
User-agent: IAArchiver-1.0
Disallow: /
User-agent: ia_archiver-web.archive.org
Disallow: /
User-agent: ia_archiver/1.6
Disallow: /
User-agent: Mozilla/5.0 (compatible; heritrix/1.5.0-200506231921 http://pandora.nla.gov.au/crawl.html)
Disallow: /
User-agent: Mozilla/5.0 (compatible;archive.org_bot/heritrix-1.9.0-200608171144 +http://pandora.nla.gov.au/crawl.html)
Disallow: /
User-agent: webster-internet.de pad browser
Disallow: /
User-agent: archive.org_bot
Disallow: /