robots.txt details

5d1d705bd1aa6ba61741642eabfed691

saved: | size: 266kb | checksum: 5d1d705bd1aa6ba61741642eabfed691

websites using this robots.txt

content


# robots.txt for https://www.propublica.org/

sitemap: https://www.propublica.org/sitemaps-1-sitemap.xml

# live - don't allow web crawlers to index cpresources/ or vendor/

User-agent: *
Disallow: /cpresources/
Disallow: /vendor/
Disallow: /.env
Disallow: /cache/