# http://www.robotstxt.org/wc/robots.html

# A list of know bots can be found at
# http://www.robotstxt.org/wc/active/html/index.html

# See http://www.sxw.org.uk/computing/robots/check.html
# for syntax checking

User-agent: *
Disallow: ?q=admin
Disallow: /admin/
Disallow: /comment/reply/
Disallow: ?q=comment/reply
Disallow: /error.php
Disallow: /cron.php
Disallow: /xmlrpc.php
Disallow: /database/
Disallow: /includes/
Disallow: /modules/
Disallow: /scripts/
Disallow: /themes/
Disallow: /.htaccess
Disallow: /update.php

# Add other dirs or files here
# For example /webreports/
Disallow: /files
Disallow: /misc
Disallow: /plesk-stat
Disallow: /private

# Disallow roverbot.com 
User-agent: Roverbot
Disallow: /

# Disallow Internet Archive Wayback Machine
User-agent: ia_archiver
Disallow: /

# Disallow Psycheclone
User-agent: psycheclone
Disallow: /

# Disallow LinkWalker
User-agent: LinkWalker
Disallow: /

# Disallow UbiCrawler
User-agent: UbiCrawler
Disallow: /

# Disallow Wget
User-agent: Wget
Disallow: /

# Disallow SiteSucker
User-agent: SiteSucker
Disallow: /

# Disallow Shim-Crawler
User-agent: Shim-Crawler
Disallow: /