#================================================================
# Disallow: /help  > disallows /help.html & /help/index.html
# Disallow: /help/ > disallow /help/index.html > allow /help.html
#================================================================
# Allow
User-Agent: W3C-checklink
Disallow:
User-Agent: W3C_Validator/1.432.2.10
Disallow:
User-Agent: Googlebot/2.1 (+http://www.google.com/bot.html)
Disallow:
User-Agent: Mozilla/5.0 (compatible; googlebot/2.1; +http://www.google.com/bot.html)
Disallow:
User-Agent: YahooFeedSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://my.yahoo.com/s/publishers.html)
Disallow:
User-Agent: Yahoo-MMCrawler/3.x (mms dash mmcrawler dash support at yahoo dash inc dot com)
Disallow:
User-Agent: Mozilla/5.0 (compatible; Yahoo! Slurp;http://help.yahoo.com/help/us/ysearch/slurp)
Disallow:
User-Agent: Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)
Disallow:
User-Agent: Mozilla/3.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)
Disallow:
User-Agent: Scooter-3.2.EX
Disallow:
User-Agent: Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
Disallow:
User-Agent: msnbot/1.0 (+http://search.msn.com/msnbot.htm)
Disallow:
User-Agent: Baiduspider ( http://www.baidu.com/search/spider.htm)
Disallow:
#================================================================
User-agent: *
#================================================================
# always excluded
#----------------
Disallow: /cgi-bin
Disallow: /stats
Disallow: /logs
Disallow: /awstats.data
#================================================================
# directories
#------------
Disallow: /badbot
Disallow: /email
Disallow: /images
Disallow: /includes
Disallow: /mailing
Disallow: /proposal
Disallow: /meta
# Disallow: /pdf
Disallow: /search
Disallow: /unsubscribe
#================================================================
# files
#------
# Disallow: /navigation.swf
# Disallow: /_navigation.swf
Disallow: /sitemap.xsd
Disallow: /index_sitemap.xsd
#================================================================
#================================================================
# excluded robots list
#---------------------
#
#================================================================
#================================================================
# google wildcards
#-----------------
# User-agent: googlebot
Disallow: *.cgi
Disallow: *.gz
Disallow: *.pl
Disallow: *.log
# Disallow: *.swf
Disallow: *.gif
Disallow: *.jpg
# Disallow: *.pdf
# Disallow: *.txt
# Disallow: *.swf
Disallow: *.csv
#----------------------------------------------------------------
# User-agent: googlebot-image
Disallow: *.cgi
Disallow: *.gz
Disallow: *.pl
Disallow: *.log
Disallow: *.swf
Disallow: *.gif
Disallow: *.jpg
# Disallow: *.pdf
# Disallow: *.txt
# Disallow: *.swf
Disallow: *.csv
#================================================================
#================================================================