robots.txt details

da24ca1278753d6f1e577e869f30bdd9

saved: | size: 690kb | checksum: da24ca1278753d6f1e577e869f30bdd9

websites using this robots.txt

content


# www.robotstxt.org/

Sitemap: http://app.wistia.com/sitemaps/62939.xml
Sitemap: https://www.theurbanlist.com/sydney/sitemap
Sitemap: https://www.theurbanlist.com/brisbane/sitemap
Sitemap: https://www.theurbanlist.com/melbourne/sitemap
Sitemap: https://www.theurbanlist.com/perth/sitemap
Sitemap: https://www.theurbanlist.com/goldcoast/sitemap
Sitemap: https://www.theurbanlist.com/sunshinecoast/sitemap
Sitemap: https://www.theurbanlist.com/singapore/sitemap
Sitemap: https://www.theurbanlist.com/nz/sitemap

# Allow crawling of all content
User-agent: *

# Stop SEO Moz from picking up all the empty bookmarking pages.
User-agent: rogerbot
Disallow: /user/save/*
Disallow: /user/unsave/*