robots.txt details

saved: size: 1678 kb md5 checksum: 54885a1d20ca39ee31bf939d4709a004

websites using this robots.txt

content


# See https://secure.smugmug.com/help/contact if you'd like to apply to be allowlisted for crawling SmugMug.

user-agent: Adsbot-Google
user-agent: Alexabot
user-agent: BingPreview
user-agent: CloudflarePrefetch
user-agent: FriendFeedBot
user-agent: FunnelBack
user-agent: Google
user-agent: Google Favicon
user-agent: Google-Site-Verification
user-agent: Google-Sitemaps
user-agent: Googlebot
user-agent: Googlebot-Image
user-agent: Googlebot-Mobile
user-agent: Googlebot-News
user-agent: Googlebot-Video
user-agent: Mediapartners-Google
user-agent: Pingdom
user-agent: Pinterest
user-agent: ScoutJet
user-agent: Slurp
user-agent: Spinn3r
user-agent: Teoma
user-agent: Twitterbot
user-agent: Yandex
user-agent: YandexImages
user-agent: YandexVideoParser
user-agent: Yeti
user-agent: archive.org_bot
user-agent: baiduspider
user-agent: bingbot
user-agent: facebookexternalhit
user-agent: gsa-crawler
user-agent: houzzbot
user-agent: ia_archiver
user-agent: msnbot
user-agent: rogerbot
disallow: /404
disallow: /access-denied
disallow: /admin
disallow: /api
disallow: /cart
disallow: /checkout
disallow: /client
disallow: /date
disallow: /downloads
disallow: /gallery
disallow: /go
disallow: /hack
disallow: /keyword
disallow: /order
disallow: /password
disallow: /popular
disallow: /search
disallow: /services/api/php
disallow: /services/api/rest
disallow: /services/api/xmlrpc
disallow: /test
allow: /api/developer
allow: /api/doc
allow: /api/v2

user-agent: Google
user-agent: Twitterbot
user-agent: facebookexternalhit
user-agent: houzzbot
allow: /date
allow: /keyword
allow: /popular

user-agent: *
disallow: /

sitemap: https://dancestudiopix.smugmug.com/sitemap-index.xml