robots.txt details

saved: size: 1856 kb md5 checksum: 28a2a24e7b26122b6cc3de58000f0f38

websites using this robots.txt

content


Sitemap: https://sneakerpeeker.cz/sitemap.xml.gz

### Automatically banned scanners and crawlers section ###
User-agent: 008
Disallow: /
user-agent: AhrefsBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: metajobbot
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: Ezooms
Disallow: /
User-agent: fyberspider
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: MojeekBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: wotbox
Disallow: /
### Section end ###

User-agent: TwengaBot-Discover
Disallow: /
User-agent: TwengaBot/2.0
Disallow: /
User-agent: TwengaBot
Disallow: /
User-agent: TwengaBot-2.0
Disallow: /

User-agent: msnbot
Disallow: /
User-agent: bingbot
Disallow: /

User-agent: Slurp
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: Yandex
Disallow: /
User-agent: *
Crawl-delay: 1
Disallow: /basketchange.php
Disallow: /basketedit.php
Disallow: /basket-onchange.php
Disallow: /client-address.php
Disallow: /client-addresses.php
Disallow: /client-new.php
Disallow: /client-orders.php
Disallow: /client-save.php
Disallow: /client-files.php
Disallow: /download-attachment.php
Disallow: /login.php
Disallow: /loginedit.php
Disallow: /loginonce.php
Disallow: /membership-card.php
Disallow: /order1.php
Disallow: /order2.php
Disallow: /order3.php
Disallow: /ordercancel.php
Disallow: /orderconfirm.php
Disallow: /orderdetails.php
Disallow: /order-document.php
Disallow: /order-newpayment.php
Disallow: /order-payment.php
Disallow: /order-postauction.php
Disallow: /order-wrappers.php
Disallow: /payments-confirm.php
Disallow: /products-bought.php
Disallow: /products-requests.php
Disallow: /przelew.php
Disallow: /rebate-code.php
Disallow: /rma-add.php
Disallow: /rma-list.php
Disallow: /settings.php
Disallow: /checkup.php
Disallow: /rma-open.php
Disallow: /returns-open.php
Disallow: /return.php
Disallow: edi/