# robots.txt for gardenmedicinals.com # Any empty [Disallow] value, indicates that all URLs can be retrieved. # At least one Disallow field needs to be present in a record. User-Agent:Baiduspider+(+http://www.baidu.com/search/spider.htm) Disallow:/ User-Agent:Zao/0.2 (http://www.kototoi.org/zao/) Disallow:/ Crawl-delay: 5 # maximum rate is one page every 5 seconds # Request-rate: 1/5 # Folowing are additional options: # User-Agent:MSIECrawler # Disallow:/