# exclude help system from robots
USER-AGENT: *
DISALLOW: /downloads/ 
DISALLOW: /media/ 
DISALLOW: /images/
DISALLOW: /javascript/
DISALLOW: /stats/
DISALLOW: /morning/
DISALLOW: /evening/
# but allow htdig to index our doc-tree
uSER-AGENT: susedig
DISALLOW:
# disallow stress test
USER-AGENT: stress-agent
DISALLOW: /