# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file # # To ban all spiders from the entire site uncomment the next two lines: User-agent: * Disallow: /*?page Disallow: /*sign_in Disallow: /*?locale Disallow: /sign_in Disallow: /users/sign_in Disallow: /*&locale <% Course.find_each do |course| %> Disallow: /courses/<%= course.id %> Disallow: /courses/<%= course.slug %> Disallow: /en/courses/<%= course.id %> Disallow: /en/courses/<%= course.slug %> Disallow: /ru/courses/<%= course.id %> Disallow: /ru/courses/<%= course.slug %> <% end %> <% Course.where(visible: false).find_each do |course| %> Disallow: course_show_uk_url(course) Disallow: course_show_en_url(course) Disallow: course_show_ru_url(course) <% end %> Disallow: /courses/faq Disallow: /ru/courses/faq Disallow: /en/courses/faq Disallow: /languages/english/english-for-restaurant-and-catering Disallow: /teachers/faq Disallow: /en/teachers/faq Disallow: /ru/teachers/faq Disallow: /! Disallow: /ru/articles/faq Disallow: /articles/28 Disallow: /articles/5-profesiy-de-neobkhidna-anhliyska-mova Disallow: /ru/articles/5-profesiy-de-neobkhidna-anhliyska-mova Disallow: /en/articles/5-profesiy-de-neobkhidna-anhliyska-mova