robots.txt details

saved: size: 8737 kb md5 checksum: c2cca90282c6e934561c1abb0d1bbe28

websites using this robots.txt

content


User-agent: Googlebot
#allow: /*?q=*
#allow: /watermark.php*
#Allow: /*.js*
#Allow: /*.css*
#Allow: /*.png*
#Allow: /*.jpg*
#Allow: /*.gif*
Disallow: /*?filter=*
Disallow: /*?sticker=*
Disallow: /*?brands=*
Disallow: /*?country=*
Disallow: /*?min=*
Disallow: /*?max=*
Disallow: /*?sale=*
Disallow: /partners.php
Disallow: /ajax/*
Disallow: /iframe/
Disallow: /cron/
Disallow: /loader/
Disallow: /order/
Disallow: /search/
Disallow: /user/
Disallow: /news/
#Disallow: /push-it/
#Disallow: /ua/push-it/
Disallow: /_fotos_*
Disallow: /statik/price/
#Disallow: /articles/*?*
Disallow: /vk/
Disallow: /youtube-player/
Disallow: /*?qr=*
Disallow: /*.php*$
Disallow: /*?noredirect*
Disallow: /*?gomobile*
Disallow: /*?cat=*
Disallow: /*?comment=*
Disallow: /*?fb_comment_id=*
Disallow: /*?show=*
Disallow: /*?fbclid=*
Disallow: /*?Fsid=*
Disallow: /*?rand=*
Disallow: /*?brand_index=*
Disallow: /*?feature=*
Disallow: /*?nbsp*
Disallow: /*?sort=*
Disallow: /*?sa=*
Disallow: /*?notesale=*
Disallow: /*?basket=*
Disallow: /*?source=post_page*
Disallow: /*/reviews/
Disallow: /*?pg=*
Disallow: /*?show*
Disallow: /*?utm_source*
Disallow: */compare/*
Disallow: /*?gclid*
Disallow: /*?type*
Disallow: */embed/*/
Disallow: */.well-known/*

User-agent: Yandex
#allow: /*?q=*
#allow: /watermark.php*
#Allow: /*.js*
#Allow: /*.css*
#Allow: /*.png*
#Allow: /*.jpg*
#Allow: /*.gif*
Disallow: /*?filter=*
Disallow: /*?sticker=*
Disallow: /*?brands=*
Disallow: /*?country=*
Disallow: /*?min=*
Disallow: /*?max=*
Disallow: /*?sale=*
Disallow: /partners.php
Disallow: /ajax/*
Disallow: /iframe/
Disallow: /cron/
Disallow: /loader/
Disallow: /order/
Disallow: /search/
Disallow: /user/
Disallow: /news/
Disallow: /_fotos_*
Disallow: /statik/price/
#Disallow: /articles/*?*
Disallow: /vk/
Disallow: /youtube-player/
Disallow: /*?qr=*
Disallow: /*noredirect
Disallow: /*.php*$
Disallow: /*?noredirect*
Disallow: /*?gomobile*
Disallow: /*?cat=*
Disallow: /*?comment=*
Disallow: /*?fb_comment_id=*
Disallow: /*?show=*
Disallow: /*?fbclid=*
Disallow: /*?Fsid=*
Disallow: /*?rand=*
Disallow: /*?brand_index=*
Disallow: /*?feature=*
Disallow: /*?nbsp*
Disallow: /*?sort=*
Disallow: /*?sa=*
Disallow: /*?notesale=*
Disallow: /*?basket=*
Disallow: /*?source=post_page*
Disallow: /*/reviews/
Disallow: /*?pg=*
Disallow: /*?show*
Disallow: /*?utm_source*
Disallow: */compare/*
Disallow: /*?gclid*
Disallow: /*?type*
Disallow: */embed/*/
Disallow: */.well-known/*
Clean-param: filter&all_brands&cat_id&ref&show&yclid&max&min&sort&noseoredirect&clear_cache&price_up&price_down&pg&cpamit_uid&brand_index&order_by&utm_source&utm_medium&utm_content&utm_campaign /*

User-agent: *
#allow: /*?q=*
#allow: /watermark.php*
#Allow: /*.js*
#Allow: /*.css*
#Allow: /*.png*
#Allow: /*.jpg*
#Allow: /*.gif*
Disallow: /*?filter=*
Disallow: /*?sticker=*
Disallow: /*?brands=*
Disallow: /*?country=*
Disallow: /*?min=*
Disallow: /*?max=*
Disallow: /*?sale=*
Disallow: /partners.php
Disallow: /ajax/*
Disallow: /iframe/
Disallow: /cron/
Disallow: /loader/
Disallow: /noredirect/
Disallow: /order/
Disallow: /search/
Disallow: /user/
Disallow: /news/
Disallow: /_fotos_*
Disallow: /statik/price/
#Disallow: /articles/*?*
Disallow: /vk/
Disallow: /youtube-player/
Disallow: /*?qr=*
Disallow: /*noredirect
Disallow: /*.php*$
Disallow: /*?noredirect*
Disallow: /*?gomobile*
Disallow: /*?cat=*
Disallow: /*?comment=*
Disallow: /*?fb_comment_id=*
Disallow: /*?show=*
Disallow: /*?fbclid=*
Disallow: /*?Fsid=*
Disallow: /*?rand=*
Disallow: /*?brand_index=*
Disallow: /*?feature=*
Disallow: /*?nbsp*
Disallow: /*?sort=*
Disallow: /*?sa=*
Disallow: /*?notesale=*
Disallow: /*?basket=*
Disallow: /*?source=post_page*
Disallow: /*/reviews/
Disallow: /*?pg=*
Disallow: /*?show*
Disallow: /*?utm_source*
Disallow: */compare/*
Disallow: /*?gclid*
Disallow: /*?type*
Disallow: */embed/*/
Disallow: */.well-known/*

User-agent: Baiduspider
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: YodaoBot
Disallow: /

User-Agent: SputnikBot
Disallow: /

User-agent: moget
Disallow: /

User-agent: ichiro
Disallow: /

User-agent: NaverBot
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: oBot
Disallow: /

User-agent: AddThis
Disallow: /

User-agent: wotbox
Disallow: /

User-agent: Embedly
Disallow: /

User-agent: 200PleaseBot
Disallow: /

User-agent: PaperLiBot
Disallow: /

User-agent: Genieo
Disallow: /

User-agent: ShowyouBot
Disallow: /

User-agent: TweetmemeBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: SurveyBot
Disallow: /

User-agent: sistrix
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BDCbot
Disallow: /

User-agent: BDCbot/1.0
Disallow: / 

User-agent: rogerbot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: linkdexbot
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: 360Spider-Image
Disallow: /

User-agent: 360Spider-Video
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: HaoSouSpider
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: grapeshot
Disallow: /

User-agent: proximic
Disallow: /

User-agent: LinkpadBot
Disallow: /

User-agent: Sunrise
Disallow: /

User-agent: Butterfly
Disallow: /

User-agent: Linguee Bot
Disallow: /

User-agent: TwengaBot-2.0
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: 008
Disallow: /

User-agent: AMZNKAssocBot/4.0
Disallow: /

User-agent: NerdyBot
Disallow: /

User-agent: trovitBot
Disallow: /

User-agent: ShopWiki
Disallow: /

User-agent: 007ac9
Disallow: /

User-agent: SiteExplorer
Disallow: /

User-agent: XoviBot
Disallow: /

User-agent: SMTBot
Disallow: /

User-agent: meanpathbot
Disallow: /

User-agent: MixrankBot
Disallow: /

User-agent: EasouSpider
Disallow: /

User-agent: Riddler
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: Swiftbot
Disallow: /

User-agent: Qwantify
Disallow: /

User-agent: psbot
Disallow: /

User-agent: CrazyWebCrawler-Spider
Disallow: /

User-agent: HyperCrawl
Disallow: /

User-agent: daumoa
Disallow: /

User-agent: coccoc
Disallow: /

User-agent: netseer
Disallow: /

User-agent: BacklinkCrawler
Disallow: /

User-agent: Alexabot
Disallow: /

User-agent: SurdotlyBot
Disallow: /

User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: yacybot
Disallow: /

User-agent: QuerySeekerSpider
Disallow: /

User-agent: WBSearchBot
Disallow: /

User-agent: DeuSu
Disallow: /

User-agent: TDJBOT
Disallow: /

User-agent: Findxbot/1.0
Disallow: /

User-agent: Findxbot
Disallow: /

User-agent: Y! J-ASR
Disallow: /

User-agent: InfoWeb
Disallow: /

User-agent: Nutch Bots
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: Ezooms
Disallow: /

User-agent: UnwindFetchor
Disallow: /

User-agent: Flipboard
Disallow: /

User-agent: voltron
Disallow: /

User-agent: IstellaBot
Disallow: /

User-agent: silk
Disallow: /

User-agent: Wget
Disallow: /

User-agent: UptimeRobot
Disallow: /

User-agent: UptimeRobot/2.0
Disallow: /

User-agent: ScreenerBot
Disallow: /

User-agent: SearchmetricsBot
Disallow: /

User-agent: WeSEE
Disallow: /

User-agent: linkdexbot/2.0
Disallow: /

User-agent: linkdexbot/2.1
Disallow: /

User-agent: linkdexbot/2.2
Disallow: /

User-agent: Uptimebot
Disallow: /

User-agent: Uptimebot/1.0
Disallow: /

User-agent: Lipperhey
Disallow: /

User-agent: ltx71
Disallow: /

User-agent: Python-urllib/2.7
Disallow: /

User-agent: Sogou web spider/4.0
Disallow: /

User-agent: Baiduspider/2.0
Disallow: /

User-agent: PrivacyAwareBot/1.1
Disallow: /

User-agent: yoozBot-2.2
Disallow: /

User-agent: wget
Disallow: /

User-agent: tweetmeme
Disallow: /

User-agent: DoCoMo/2.0
Disallow: /

User-agent: ichiro/4.0
Disallow: /

User-agent: ichiro/3.0
Disallow: /

User-agent: TwengaBot-Discover
Disallow: /

User-agent: TwengaBot/2.0
Disallow: /

User-agent: TwengaBot
Disallow: /

User-agent: bendercrawler
Disallow: /

User-agent: baidu
Disallow: /

User-agent: bsalsa
Disallow: /

User-agent: phpservermon/3.1.1
Disallow: /

User-agent: Gluten Free Crawler/1.0
Disallow: /

User-agent: Gluten Free Crawler
Disallow: /

User-agent: Sogou Pic Spider/3.0
Disallow: /

User-agent: Sogou head spider/3.0
Disallow: /

User-agent: Sogou Orion spider/3.0
Disallow: /

User-agent: Sogou-Test-Spider/4.0
Disallow: /

User-agent: Sogou Pic Agent
Disallow: /

User-agent: Baiduspider+
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-agent: Baiduspider-image
Disallow: /

User-agent: YoudaoBot/1.0
Disallow: /

User-agent: ChangeDetection
Disallow: /

User-agent: SurdotlyBot
Disallow: /

User-agent: libwww-perl
Disallow: /

User-agent: MojeekBot
Disallow: /

User-agent: Sogou web spider
Disallow: /