Index: robots.txt
===================================================================
RCS file: /cvs/drupal/drupal/robots.txt,v
retrieving revision 1.9
diff -u -p -r1.9 robots.txt
--- robots.txt	27 Jun 2007 22:37:44 -0000	1.9
+++ robots.txt	4 Oct 2007 00:32:24 -0000
@@ -18,7 +18,9 @@
 # http://www.sxw.org.uk/computing/robots/check.html
 
 User-agent: *
-Crawl-delay: 10
+# Uncomment the following line if your server is getting hit too hard by robots
+# Crawl-delay: 10
+
 # Directories
 Disallow: /database/
 Disallow: /includes/
@@ -43,21 +45,37 @@ Disallow: /LICENSE.txt
 Disallow: /UPGRADE.txt
 # Paths (clean URLs)
 Disallow: /admin/
+# If you want your aggregator pages to be indexed, comment out the following line
+Disallow: /aggregator
 Disallow: /comment/reply/
-Disallow: /contact/
-Disallow: /logout/
+Disallow: /contact$
+Disallow: /logout$
+Disallow: /node$
 Disallow: /node/add/
+Disallow: /search$
 Disallow: /search/
-Disallow: /user/register/
-Disallow: /user/password/
-Disallow: /user/login/
+Disallow: /user$
+Disallow: /user/register
+Disallow: /user/password
+Disallow: /user/login
 # Paths (no clean URLs)
 Disallow: /?q=admin/
+# If you want your aggregator pages to be indexed, comment out the following line
+Disallow: /?q=aggregator
 Disallow: /?q=comment/reply/
-Disallow: /?q=contact/
-Disallow: /?q=logout/
+Disallow: /?q=contact
+Disallow: /?q=logout
+Disallow: /?q=node$
 Disallow: /?q=node/add/
+Disallow: /?q=search$
 Disallow: /?q=search/
-Disallow: /?q=user/password/
-Disallow: /?q=user/register/
-Disallow: /?q=user/login/
+Disallow: /?q=user$
+Disallow: /?q=user/password
+Disallow: /?q=user/register
+Disallow: /?q=user/login
+# Miscellaneous
+Disallow: /*sort=
+Disallow: /*/feed$
+Disallow: /*/track$
+Disallow: /tracker?
+Disallow: /tracker/