# Updated 2009.04.20 11:13 AM
#
# Reference  http://en.wikipedia.org/wiki/Robots.txt
#
User-agent: *
Crawl-delay: 5 # Sets number of seconds to wait between requests to 5 seconds. may not work
Request-rate: 1/5 # Maximum rate is one page every 5 seconds. may not work
Disallow: /*?decorator=printable # remove printable version links, non-display URLs
Disallow: /*javascript* # remove any javascript links, per log analysis
Disallow: /admin/* # administrator links
Disallow: /label/* # remove all label links
Disallow: /labels/* # remove all label links
Disallow: /labels-javascript # remove label javascript