# robots.txt file
# Last updated: 29.9.2008

User-agent: Xenu
Disallow:
User-agent: Googlebot
Disallow:
User-agent: Googlebot-Image
Disallow:
User-agent: Mediapartners-Google
Disallow:
User-agent: Adsbot-Google
Disallow:
User-Agent: W3C-checklink
Disallow:

User-agent: *
Crawl-delay: 10
Request-rate: 1/10
Visit-time: 0600-0800
# PmWiki files and directories
Disallow: /
Disallow: /pmwiki.php/Main/AllRecentChanges
Disallow: /pmwiki.php/
Disallow: /index.php/
Disallow: /work.d/
Disallow: */search
Disallow: SearchWiki
Disallow: *RecentChanges
Disallow: RecentChanges
Disallow: *action=
Disallow: action=

User-agent: BecomeBot
Disallow: /
User-agent: digout4u
Disallow: /
User-agent: extractorpro
Disallow: /
User-agent: fast
Disallow: /
User-agent: GetRight
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: go-ahead-got-it
Disallow: /
User-agent: HTTPClient
Disallow: /
User-agent: JetBot
Disallow: /
User-agent: teomaagent1
Disallow: /
User-agent: directhit
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: LinkWalker
Disallow: /
User-agent: lycos
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: msnbot
Disallow: /
User-agent: nearsite
Disallow: /
User-agent: netattache
Disallow: /
User-agent: NEWT ActiveX
Disallow: /
User-agent: Scooter
Disallow: /
User-agent: sitesnagger
Disallow: /
User-agent: Slurp
Disallow: /
User-agent: T-Rex
Disallow: /
User-agent: teleport
Disallow: /
User-agent: Teoma
Disallow: /
User-agent: TovekTools Web Indexer
Disallow: /
User-agent: tutorgig
Disallow: /
User-agent: Web Downloader
Disallow: /
User-agent: webwhacker
Disallow: /
User-agent: webzip
Disallow: /
User-agent: wget
Disallow: /

# Easily.co.uk folder structures
User-agent: *
Crawl-delay: 10
Request-rate: 1/10
Visit-time: 0600-0800
Disallow: /cgi-bin/
Disallow: /img/
Disallow: /images/
Disallow: /_baks/
Disallow: /_mm/
Disallow: /_notes/
Disallow: /_private/
Disallow: /error/
Disallow: /_borders/
Disallow: /_derived/
Disallow: /_fpclass/
Disallow: /_overlay/
Disallow: /_private/
Disallow: /_themes/
Disallow: /_vti_bin/
Disallow: /_vti_cnf/
Disallow: /_vti_log/
Disallow: /_vti_map/
Disallow: /_vti_pvt/
Disallow: /_vti_txt/
Disallow: /_vti_inf.html
Disallow: /postinfo.html

# Joomla folder structures
User-agent: *
Crawl-delay: 10
Request-rate: 1/10
Visit-time: 0600-0800
Disallow: /administrator/
Disallow: /cache/
Disallow: /components/
Disallow: /editor/
Disallow: /images/
Disallow: /includes/
Disallow: /installation/
Disallow: /language/
Disallow: /libraries/
Disallow: /media/
Disallow: /modules/
Disallow: /plugins/
Disallow: /templates/
Disallow: /installation/
Disallow: /tmp/
Disallow: /xmlrpc/