# Robots.txt for http://www.dailymail.co.uk/ updated 25/01/2018 # TS-5682 # # All Robots User-agent: * # # Begin Standard Rules Disallow: /travel-destinations* Disallow: /traveldestinations* Disallow: /*readcommentshtml* Disallow: /*jsessionid* Disallow: /*createThread.html Disallow: /*debateSearchResults.html Disallow: /*debateTagSearch.html Disallow: /*debateUserSearch.html Disallow: /*login?redirectPath= Disallow: /*logout?redirectPath= Disallow: /*myStories.html$ Disallow: /*nextThread.html Disallow: /*previousThread.html Disallow: /*questionId* Disallow: /*selectedImage* Disallow: /*textbased/channel Disallow: /*threadIndex=* Disallow: /*topGallery* Disallow: /ce/item.cms* Disallow: /dailymail/ Disallow: /debate/polls/poll.html$ Disallow: /destinations/ Disallow: /guide/* Disallow: /home/sitemaparchive/static$ Disallow: /link/index.html* Disallow: /mailshopQA/* Disallow: /SITE=DM/ Disallow: /sport/columnist-1000543/hatchet-man.rss Disallow: /sport/columnist-1031773/andy-roddick.rss Disallow: /sport/columnist-1046072/glenn-hoddle.rss Disallow: /sport/columnist-1060209/jake-humphrey.rss Disallow: /sport/columnist-353/sir-bobby-robson.rss Disallow: /sport/columnist-354/ian-ridley--mail-sunday-sports-reporter.rss Disallow: /sport/columnist-1043711/mark-jeffreys.rss Disallow: /sport/columnist-423/dan-king.rss Disallow: /sport/columnist-352/monty-panesar.rss Disallow: /sudoku* Disallow: /travel/enjoyengland/index.rss Disallow: /travel/visitcornwall/index.html Disallow: /tvshowbiz/tvlistings/* Disallow: /weather.html* Disallow: /weather/weather_landing_page.html* Disallow: /ukplus/* Disallow: /*/chat/* Disallow: /app/ Disallow: /beta/* Disallow: /home/canneslions/index.html Disallow: /cms/sites/all/modules/ckeditor_link/proxy.php* Disallow: /%20user/view.php* Disallow: /worldhome/* Disallow: /home/search.html* Disallow: /home/article-4252192/* Disallow: /home/article-1388040/Privacy-Policy-Cookies.html Disallow: /news/article-2663638/Mother-two-60-bled-death-surgeon-cut-vein-botched-hip-operation.html Disallow: /news/article-2634104/Transgender-woman-landmark-legal-battle-force-government-forget-identity-man-sex-change.html Disallow: /dailymailtv/justthepics/index.html Disallow: /health/the-opioid-crisis/index.html Disallow: /galleries/index.html # # Disallow Money for Google News User-agent: Googlebot-News Disallow: /tmoney/* # # Allow Adsense User-agent: Mediapartners-Google Disallow: # # Disallow Specific Robots User-agent: MnoGoSearch/* Disallow: / # User-agent: omgilibot/0.3 Disallow: / # User-agent: WebVac Disallow: / # User-agent: WebZip Disallow: / # User-agent: psbot Disallow: / # User-agent: daumoa Disallow: / # user-agent: AhrefsBot disallow: / # User-agent: CrystalSemanticsBot Disallow: / # user-agent: AhrefsBot disallow: / # User-agent: ia_archiver Disallow: /*/article-* # User-agent: Nutch Disallow: / # # Don't crawl profile pages #Disallow: /registration/*/profile.html # # Sitemap Files Sitemap: http://www.dailymail.co.uk/newssitemap.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2018.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2017.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2016.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2015.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2014.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2013.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2012.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2011.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2010.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2009.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2008.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2007.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2006.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2005.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2004.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2003.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2002.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2001.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~2000.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~1999.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~1998.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~1997.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~1996.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~1994.xml Sitemap: http://www.dailymail.co.uk/sitemap-articles-year~1993.xml Sitemap: http://www.dailymail.co.uk/sitemap-channels.xml Sitemap: http://www.dailymail.co.uk/sitemap-columnist.xml Sitemap: http://www.dailymail.co.uk/sitemap-topics.xml Sitemap: http://www.dailymail.co.uk/sitemap-facts-year~2017.xml Sitemap: http://www.dailymail.co.uk/sitemap-facts-year~2018.xml Sitemap: http://www.dailymail.co.uk/sitemap-videos-year~2009.xml Sitemap: http://www.dailymail.co.uk/sitemap-videos-year~2010.xml Sitemap: http://www.dailymail.co.uk/sitemap-videos-year~2011.xml Sitemap: http://www.dailymail.co.uk/sitemap-videos-year~2012.xml Sitemap: http://www.dailymail.co.uk/sitemap-videos-year~2013.xml Sitemap: http://www.dailymail.co.uk/sitemap-videos-year~2014.xml Sitemap: http://www.dailymail.co.uk/sitemap-videos-year~2015.xml Sitemap: http://www.dailymail.co.uk/sitemap-videos-year~2016.xml Sitemap: http://www.dailymail.co.uk/sitemap-videos-year~2017.xml Sitemap: http://www.dailymail.co.uk/sitemap-videos-year~2018.xml