robots.txt details

aee5aa02fc566074535aedfa35925c5d

saved: | size: 1700kb | checksum: aee5aa02fc566074535aedfa35925c5d

content


# Specific allow rules to override the disallow rules below
User-agent: *
Allow: /api2/ProfileApi/ProfilePhotos

# Tell search engines not to index these folders
User-agent: *
Disallow: /Stats
Disallow: /stats
Disallow: /Signin
Disallow: /signin
Disallow: /Secure
Disallow: /secure
Disallow: /Payment
Disallow: /payment
Disallow: /Mobile/Payment
Disallow: /mobile/payment
Disallow: /Search/Listing
Disallow: /search/listing
Disallow: /Search/Talent
Disallow: /search/talent
Disallow: /Error.aspx
Disallow: /error.aspx
Disallow: /Error404.aspx
Disallow: /error404.aspx
Disallow: /DisplayError.aspx
Disallow: /displayerror.aspx
Disallow: /listing/Apply
Disallow: /listing/apply
Disallow: /webapi
Disallow: /api2
Disallow: /Services
Disallow: /report
Disallow: *SaveSearch.aspx
Disallow: *savesearch.aspx
Disallow: *EmailToFriend.aspx
Disallow: *emailtofriend.aspx
Disallow: *EmailPhotoToFriend.aspx
Disallow: *emailphototofriend.aspx
Disallow: /*?*return_url=*


User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-Agent: XSpider
Disallow: /

User-Agent: Twitterbot
Disallow: /*thepussypalace*

# Set bot limits
User-agent: bingbot
Crawl-delay: 1

User-agent: Yandex
Crawl-delay: 2

User-agent: DotBot
Crawl-delay: 2

User-agent: SemrushBot
Crawl-delay: 2

User-agent: AhrefsBot
Crawl-Delay: 2

User-agent: Sogou
Crawl-delay: 2

User-agent: Exabot
Crawl-delay: 2

User-agent: MJ12bot
Crawl-Delay: 2

User-agent: Amazonbot
Crawl-Delay: 2

User-agent: PimEyesBot
Disallow: /

User-agent: pimeyes
Disallow: /

User-agent: MauiBot
Crawl-Delay: 2

User-agent: BLEXBot
Crawl-Delay: 2