# deny these

User-agent: Heritrix
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: larbin
Disallow: /

User-agent: seekbot
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: YahooSeeker
Disallow: /

# allow others

User-agent: *
Disallow: /cgi-data/
Disallow: /counter/
Disallow: /ALT/
Disallow: /test/