robots.txt details

saved: size: 997 kb md5 checksum: 4e5e4b220aa513894b6331c13e241313

websites using this robots.txt

content


User-agent: *
Disallow: /search
Disallow: /comments/

User-agent: Mediapartners-Google
Disallow:

User-agent: Google-Extended
Disallow:

User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /css
Disallow: /fonts
Disallow: /media
Disallow: /piano
Disallow: /static

User-agent: grapeshot
Disallow:

User-agent: SiteAuditBot
Disallow:

User-agent: anthropic-ai
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: cohere-ai
User-agent: DataForSeoBot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: GoogleOther
User-agent: GPTBot
User-agent: img2dataset
User-agent: ImagesiftBot
User-agent: magpie-crawler
User-agent: Meltwater
User-Agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: PerplexityBot
User-agent: PiplBot
User-agent: Seekr
User-agent: TruliaBot
User-agent: YouBot
Disallow: /

Sitemap: https://slate.com/sitemaps/sitemap-index.xml