robots.txt details

saved: size: 3502 kb md5 checksum: c8a8f90c939fac8c70696f8784361e18

websites using this robots.txt

content


#             _________
#            /         \
#           /           \
#          /             \
#          |  ####  #### |
#          | / @ | / @ | |   .---------------------------------.
#          | \__/  \__/  |  (  YOU'RE DEFINITELY NOT A CRAWLER  )
#          |             |   .---------------------------------.
#          \    ___     /  _/
#           \          /
#            \________/
#                |
#               /|\
#              / | \
#             /  |  |
#            /   |  |
#                |

User-Agent: *
Allow: */community/edits/new
Allow: /search?n=16&r=3
Allow: /original-stories
Disallow: /generate_xml.php
Disallow: /restore.php
Disallow: /ajax/*
Disallow: /api/*
Disallow: /api2/*
Disallow: /assets/*
Disallow: /cron/*
Disallow: /data_vars/*
Disallow: /external/*
Disallow: /files/*
Disallow: /staff/*
Disallow: /pds/*
Disallow: *username=*
Disallow: /offer-redirect/
Disallow: /*?r=
Disallow: /*?q=*
Disallow: */edits/*
Disallow: /*?n=*
Disallow: /admitad/finance/redirect
Disallow: /page/misc/adstat.php*
Disallow: /.well-known/*
Disallow: /search?*
Disallow: /tag/*st=*
Disallow: /tag/*n=*
Disallow: /tag/*et=*
Disallow: /tag/*d=*
Disallow: /tag/*cl=*
Disallow: /tag/*vh=*
Disallow: /@*/*
Disallow: /community/*/*
Disallow: /communities/*
Disallow: /communities?t=*
Disallow: /communities?q=*
Disallow: /add?community=*
Disallow: /information/docs
Disallow: *etm_term=*
Disallow: *ysclid=*
Disallow: *loc=*

User-Agent: Yandex
Allow: */community/edits/new
Allow: /search?n=16&r=3
Allow: /original-stories
Disallow: /generate_xml.php
Disallow: /restore.php
Disallow: /ajax/*
Disallow: /api/*
Disallow: /api2/*
Disallow: /assets/*
Disallow: /cron/*
Disallow: /data_vars/*
Disallow: /external/*
Disallow: /files/*
Disallow: /rabbit_workers/*
Disallow: /staff/*
Disallow: /pds/*
Disallow: *username=*
Disallow: /offer-redirect/
Disallow: /*?r=
Disallow: /*?q=*
Disallow: */edits/*
Disallow: /*?n=*
Disallow: /admitad/finance/redirect
Disallow: /page/misc/adstat.php*
Disallow: /.well-known/*
Disallow: /search?*
Disallow: /tag/*st=*
Disallow: /tag/*n=*
Disallow: /tag/*et=*
Disallow: /tag/*d=*
Disallow: /tag/*cl=*
Disallow: /tag/*vh=*
Disallow: /@*/*
Disallow: /community/*/*
Disallow: /communities/*
Disallow: /communities?t=*
Disallow: /communities?q=*
Disallow: /add?community=*
Clean-param: rsid&story_url&vkmain&andpostshare&tgmain&YandexZenSpecial&linkshare&fbmain&viber&turbo_turbo&vshare&vkgame&iospostshare&andlinkshare&st&dv&d&photo&text&longread&is_hot&from&cpm_token

Crawl-Delay: 3

User-Agent: Mail.RU_Bot
Allow: */community/edits/new
Allow: /search?n=16&r=3
Allow: /original-stories
Disallow: /generate_xml.php
Disallow: /restore.php
Disallow: /WT4fgrvixo2F/*
Disallow: /ajax/*
Disallow: /api/*
Disallow: /api2/*
Disallow: /assets/*
Disallow: /cron/*
Disallow: /data_vars/*
Disallow: /external/*
Disallow: /files/*
Disallow: /rabbit_workers/*
Disallow: /staff/*
Disallow: /pds/*
Disallow: *username=*
Disallow: /offer-redirect/
Disallow: /*?r=
Disallow: /*?q=*
Disallow: */edits/*
Disallow: /*?n=*
Disallow: /admitad/finance/redirect
Disallow: /page/misc/adstat.php*
Disallow: /.well-known/*
Disallow: /search?*
Disallow: /tag/*st=*
Disallow: /tag/*n=*
Disallow: /tag/*et=*
Disallow: /tag/*d=*
Disallow: /tag/*cl=*
Disallow: /tag/*vh=*
Disallow: /@*/*
Disallow: /community/*/*
Disallow: /communities/*
Disallow: /communities?t=*
Disallow: /communities?q=*
Disallow: /add?community=*
Crawl-Delay: 3

Sitemap: https://pikabu.ru/sitemap_index.xml