robots.txt details

saved: size: 807 kb md5 checksum: 63095a0b067a824dd08f0699ce70b20d

websites using this robots.txt

content


# ROBOTS.TXT FOR PHOTOSHELTER.COM
Sitemap: https://www.photoshelter.com/sitemap-static.xml
Sitemap: https://www.photoshelter.com/users-sitemapindex/0.xml.gz
Sitemap: https://www.photoshelter.com/users-sitemapindex/1.xml.gz
Sitemap: https://www.photoshelter.com/users-sitemapindex/2.xml.gz
Sitemap: https://www.photoshelter.com/users-sitemapindex/3.xml.gz

# Was disallowed because it was overly aggressive
# access re-enabled on May 30, 2013
# User-agent: ia_archiver
# Disallow: /

User-agent: *
Disallow: /adm/
Disallow: /ajax/
Disallow: /com/
Disallow: /ext/
Disallow: /ltr/
Disallow: /mem/
Disallow: /mu/
Disallow: /pp/
Disallow: /ezp/
Disallow: /cart/
Disallow: /c/*/login
Disallow: /c/*/signup*
Disallow: /lattice/unsupported-browser

User-agent: MJ12bot
Disallow: 

User-agent: Twitterbot
Disallow: