robots.txt details

saved: size: 664 kb md5 checksum: d28df5dfdb0654efc78a08bbad3f5757

websites using this robots.txt

content


#
# robots.txt for epages
#
# applies to all robots
# disallow TBO, PBO
# prevent mass indexing of search pages
User-agent: *
Disallow: /epages/Site.admin/
Disallow: /epages/*ViewAction=FacetedSearchProducts
Disallow: /epages/*ViewAction=RemoteSearchProducts
Disallow: /epages/*ViewAction=ViewRemoteSearchProducts
Disallow: /epages/*ViewAction=MobileSF-ViewDetailRemoteSearchProducts
Disallow: /epages/*ViewAction=MobileSF-DetailRemoteSearchProducts
Disallow: /epages/*ViewAction=MobileSF-ViewDetailFacetedSearchProducts
Disallow: /epages/*ViewAction=MobileSF-DetailFacetedSearchProducts

User-agent: BLEXBot
Crawl-delay: 60 

User-agent: pricebot
Crawl-delay: 10