robots.txt details

2606b2eb9104e9438f6a5b12345fecf8

saved: | size: 469kb | checksum: 2606b2eb9104e9438f6a5b12345fecf8

websites using this robots.txt

content



User-agent: Slurp
Crawl-delay: 100
Disallow:

User-agent: gsa-crawler-www
Crawl-delay: 100

User-agent: Googlebot
Crawl-delay: 100

User-agent: Mediapartners-Google
Disallow:

User-agent: Yahoo-NewsCrawler
Disallow:

User-Agent: msnbot
Crawl-delay: 100
Disallow:

User-Agent: *
Disallow: /config/
Disallow: /handlers/
Disallow: /includes/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /models/
Disallow: /modules/
Disallow: /views/
Allow: /