# Disallow supporting directories
User-agent: *
Crawl-delay: 20
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /F1-bug/
Disallow: /Klara-Thomas/
Disallow: /AustraliaDay/
Disallow: /Germany-200705/
Disallow: /Mick70/
Disallow: /Mex02/
Disallow: /Oz03/
Disallow: /gs/
Disallow: /Oz06/
Disallow: /scripts/
Disallow: /themes/
Disallow: /plugins/
Disallow: /languages/
Disallow: /interface/
Disallow: /flash/
Disallow: /config/
Disallow: /content/
Disallow: /docs/
Disallow: /calendar/
Disallow: /calendar/year.php
Disallow: /calendar/day.php
Disallow: /calendar/week.php
Disallow: /calendar/month.php
Disallow: /calendar/print.php
# allow major search engines
User-agent: Googlebot
Disallow: /calendar/
User-agent: Twiceler
Disallow: /calendar/
User-agent: MSNBot
Disallow: /calendar/
User-agent: Slurp
Disallow: /calendar/
User-agent: Teoma
Disallow: /calendar/
User-agent: Gigabot
Disallow: /calendar/
User-agent: Scrubby
Disallow: /calendar/
User-agent: Robozilla
Disallow: /calendar/
# Disallow stuff I don't like
User-agent: Googlebot-Image
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: Scrubby
Disallow: /
User-agent: Robozilla
Disallow: /
User-agent: Nutch
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: baiduspider
Disallow: /
User-agent: yahoo-mmcrawler
Disallow: /
User-agent: psbot
Disallow: /
User-agent: asterias
Disallow: /
User-agent: yahoo-blogs/v3.9
Disallow: /
# Disallow stuff I don't know
User-agent: *
Disallow: /
# slow down - crawlers shouldn't be in a hurry
Crawl-delay: 20