# ROBOTS.TXT - V1.1.12 User-agent: MS Search 4.0 Robot Disallow: / # Firefox prefetch User-agent: Fasterfox Disallow: / User-agent: * # Never set "Crawl-d/elay" to less than 30 seconds (120 is recommended as a lowest number) # Since Google is not using it at all, you can set it higher to protect against "wild" robots Crawl-delay: 16384 # SITEMAPCARD Disallow: /zstop/ Disallow: /media/video/ Disallow: /imgint/ Disallow: /scripts/ Disallow: /zscripts/ # special robots User-agent: Slurp User-agent: Yahoo-MMCrawler User-agent: msnbot User-agent: msnbot-media User-agent: ia_archiver User-agent: twiceler Crawl-delay: 172800 Disallow: /zstop/ Disallow: /media/video/ Disallow: /imgint/ Disallow: /scripts/ Disallow: /zscripts/ # --------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------