#
# http://www.robotstxt.org/wc/exclusion.html#robotstxt
#
# Note that you need a separate "Disallow" line for every URL prefix to exclude
# -- you cannot say "Disallow: /cgi-bin/ /tmp/".
# Also, you may not have blank lines in a record,
#  as they are used to delimit multiple records.
# Lines containing only a comment are discarded completely,
#  and therefore do not indicate a record boundary.

## http://www.singingfish.com/spider.html indexes media files:
# User-agent: asterias

## http://www.alexa.com/support/for_webmasters.html
# User-agent: ia_archiver


Sitemap: http://herbgoddard.com/sitemap.xml

User-Agent: NPBot
Disallow: /

User-Agent: sitecheck.internetseer.com
Disallow: /file-to-disallow

# 20041030:
# http://www.whois.sc/" "SurveyBot/2.3 (Whois Source)"
User-Agent: SurveyBot
Disallow: /

# 20040404:
User-agent: Uptimebot
Disallow: /


User-agent: *
Disallow: /_vti_bin/
Disallow: /_vti_cnf/
Disallow: /_vti_log/
Disallow: /_vti_pvt/
Disallow: /_vti_txt/
Disallow: /cgi/
Disallow: /cgi-bin/
Disallow: /conf/
Disallow: /img/
Disallow: /logs/
Disallow: /ns-icons/
#