#Google Search Engine Robot User-agent: Googlebot Allow: /?_escaped_fragment_ Allow: /*?lang= Allow: /hashtag/*?src= Allow: /search?q=%23 Disallow: /search/realtime Disallow: /search/users Disallow: /search/*/grid Disallow: /*? Disallow: /*/followers Disallow: /*/following Disallow: /account/not_my_account #Yahoo! Search Engine Robot User-Agent: Slurp Allow: /?_escaped_fragment_ Allow: /*?lang= Allow: /hashtag/*?src= Allow: /search?q=%23 Disallow: /search/realtime Disallow: /search/users Disallow: /search/*/grid Disallow: /*? Disallow: /*/followers Disallow: /*/following Disallow: /account/not_my_account #Yandex Search Engine Robot User-agent: Yandex Allow: /?_escaped_fragment_ Allow: /*?lang= Allow: /hashtag/*?src= Allow: /search?q=%23 Disallow: /search/realtime Disallow: /search/users Disallow: /search/*/grid Disallow: /*? Disallow: /*/followers Disallow: /*/following Disallow: /account/not_my_account #Microsoft Search Engine Robot User-Agent: msnbot Allow: /?_escaped_fragment_ Allow: /*?lang= Allow: /hashtag/*?src= Allow: /search?q=%23 Disallow: /search/realtime Disallow: /search/users Disallow: /search/*/grid Disallow: /*? Disallow: /*/followers Disallow: /*/following Disallow: /account/not_my_account # Every bot that might possibly read and respect this file. User-agent: * Allow: /*?lang= Allow: /hashtag/*?src= Allow: /search?q=%23 Disallow: /search/realtime Disallow: /search/users Disallow: /search/*/grid Disallow: /*? Disallow: /*/followers Disallow: /*/following Disallow: /account/not_my_account Disallow: /oauth Disallow: /1/oauth Disallow: /i/streams Disallow: /i/hello # Wait 1 second between successive requests. See ONBOARD-2698 for details. Crawl-delay: 1 # Independent of user agent. Links in the sitemap are full URLs using https:// and need to match # the protocol of the sitemap. Sitemap: https://twitter.com/sitemap.xml