robots.txt details

7d457db4c6a5b7b4cebcba948bc472c6

saved: | size: 662kb | checksum: 7d457db4c6a5b7b4cebcba948bc472c6

content


#
# robots.txt for epages
#
# applies to all robots
# disallow TBO, PBO
# prevent mass indexing of search pages
User-agent: *
Disallow: /epages/Site.admin/
Disallow: /epages/*ViewAction=FacetedSearchProducts
Disallow: /epages/*ViewAction=RemoteSearchProducts
Disallow: /epages/*ViewAction=ViewRemoteSearchProducts
Disallow: /epages/*ViewAction=MobileSF-ViewDetailRemoteSearchProducts
Disallow: /epages/*ViewAction=MobileSF-DetailRemoteSearchProducts
Disallow: /epages/*ViewAction=MobileSF-ViewDetailFacetedSearchProducts
Disallow: /epages/*ViewAction=MobileSF-DetailFacetedSearchProducts

User-agent: pricebot
Crawl-delay: 10

User-agent: BLEXBot
Crawl-delay: 60