diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 55a689b6f760d9927213eeae6c47d5e80b5ecf04..ee539a5fb70a746fa2e6d6188d9ec7ceecb90e3c 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -24,7 +24,7 @@ Drupal x.x.x, xxxx-xx-xx (development version) * optionally allow people to cancel their vote. - distributed authentication: * added default server option. -- fixed critical SQL issue, see SA-2006-005 +- added default robots.txt to control crawlers. - database API: * added db_table_exists(). - blogapi module: diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000000000000000000000000000000000000..c41c1a4243e18c303ff8844a48e209ef0d60a975 --- /dev/null +++ b/robots.txt @@ -0,0 +1,57 @@ +# robots.txt +# +# This file aims to prevent the crawling and idexing of certain parts +# of your site by web crawlers and spiders run by sites like Yahoo! +# and Google. By telling these "robots" where not to go on your site, +# you save bandwidth and server resources. +# +# For more information about the robots.txt standard, see: +# http://www.robotstxt.org/wc/robots.html +# +# See this site for syntax checking: +# http://www.sxw.org.uk/computing/robots/check.html + +User-agent: * +Crawl-delay: 10 +# Directories +Disallow: /files/ +Disallow: /database/ +Disallow: /includes/ +Disallow: /misc/ +Disallow: /modules/ +Disallow: /sites/ +Disallow: /themes/ +Disallow: /scripts/ +Disallow: /updates/ +Disallow: /profiles/ +# Files +Disallow: /xmlrpc.php +Disallow: /cron.php +Disallow: /update.php +Disallow: /install.php +Disallow: /INSTALL.mysql.txt +Disallow: /INSTALL.pgsql.txt +Disallow: /CHANGELOG.txt +Disallow: /MAINTAINERS.txt +Disallow: /LICENSE.txt +Disallow: /UPGRADE.txt +# Paths (clean URLs) +Disallow: /admin/ +Disallow: /aggregator/ +Disallow: /comment/reply/ +Disallow: /contact/ +Disallow: /logout/ +Disallow: /node/add/ +Disallow: /search/ +Disallow: /user/register/ +Disallow: /user/password/ +# Paths (no clean URLs) +Disallow: /?q=admin/ +Disallow: /?q=aggregator/ +Disallow: /?q=comment/reply/ +Disallow: /?q=contact/ +Disallow: /?q=logout/ +Disallow: /?q=node/add/ +Disallow: /?q=search/ +Disallow: /?q=user/password/ +Disallow: /?q=user/register/