|
|
1
2
|
|
Editor: page
Time: 2009/10/14 21:53:51 GMT-7
|
|
Note: new version
|
changed:
-# By default we allow robots to access all areas of our site
-# already accessible to anonymous users
User-Agent: googlebot
Disallow: /*subscribeform
Disallow: /*diff$
Disallow: /*clearCache$
User-Agent: slurp
Disallow: /*subscribeform
Disallow: /*diff$
Disallow: /*clearCache$
changed:
-Disallow:
-
-# Add Googlebot-specific syntax extension to exclude forms
-# that are repeated for each piece of content in the site
-# the wildcard is only supported by Googlebot
-# http://www.google.com/support/webmasters/bin/answer.py?answer=40367&ctx=sibling
-
-User-Agent: Googlebot
-Disallow: /*subscribeform$
Sitemap: http://axiom-wiki.newsynthesis.org/SiteMap?bare=1
Disallow: /*subscribeform
added:
Disallow: /*clearCache$
# Define access-restrictions for robots/spiders
# http://www.robotstxt.org/wc/norobots.html
User-Agent: googlebot
Disallow: /*subscribeform
Disallow: /*diff$
Disallow: /*clearCache$
User-Agent: slurp
Disallow: /*subscribeform
Disallow: /*diff$
Disallow: /*clearCache$
User-agent: *
Sitemap: http://axiom-wiki.newsynthesis.org/SiteMap?bare=1
Disallow: /*subscribeform
Disallow: /*diff$
Disallow: /*clearCache$