robots.txt details

saved: size: 2451 kb md5 checksum: 3a5faad6dd32c4b0395dfb1727eeef36

websites using this robots.txt

content


#PrivatePages
User-agent: *
 Disallow: /manufacturers
 Disallow: /order
 Disallow: /*supplier
 Disallow: /*login
 Disallow: /*cart
 Disallow: /login?create_account=1
 Disallow: /login?back=address
 Disallow: /login?back=addresses
 Disallow: /login?back=discount
 Disallow: /login?back=identity
 Disallow: /password-recovery
 Disallow: /?supplier_list=0
 Disallow: /?manufacturer_list=0
 Disallow: /guest-tracking
 Disallow: /*search
 Disallow: /*module/productcomments/default
 Disallow: /*module/blockwishlist/mywishlist
 Disallow: /*?s=
 Disallow: /*orderby

 Disallow: /my-account
 Disallow: /search
 Disallow: /quick-order
 Disallow: /order-confirmation
 Disallow: /products-comparison

User-agent: ShopWiki
 Disallow: /

 User-agent: NG/2.0
 Disallow: /

 user-agent: crystalsemantics
 disallow: /

 
user-agent: f-six
 disallow: /

 user-agent: sitebot
 disallow: /

 user-agent: Mediapartners-Google
 Allow: /trouve.php

 User-Agent: trendictionbot
 Disallow: /

 User-Agent: scrapybot
 Disallow: /

 User-Agent: VocusBot
 Disallow: /

 User-Agent: SEOkicks-Robot
 Disallow: /

 User-Agent: ezooms.bot
 Disallow: /

 User-Agent: skimlinks.com
 Disallow: /

User-Agent: SeznamBot
 Disallow: /


 User-Agent: wikiwix-bot
 Disallow: /

 User-Agent: crystalsemantics
 Disallow: /

 User-Agent: semantissimo
 Disallow: /

 User-Agent: MLBot
 Disallow: /

 User-Agent: LemonSources
 Disallow: /

 User-Agent: discobot
 Disallow: /

 
User-Agent: linkfluence
 Disallow: /

 User-Agent: netseer
 Disallow: /

 User-Agent: ltbot
 Disallow: /

 User-Agent: aiHitBot
 Disallow: /

 User-Agent: language-tools.com
 Disallow: /

 User-Agent: TREC-KBA-Bot
 Disallow: /

 User-Agent: checkprivacy.or.kr
 Disallow: /

 User-Agent: yacybot
 Disallow: /

 User-Agent: Socialradarbot
 Disallow: /

 User-Agent: Havij
 Disallow: /

 User-Agent: proximic
 Disallow: /

#Images
User-agent: *
 # CSS, JS, Images
 Allow: /misc/*.css$
 Allow: /misc/*.css?
 Allow: /misc/*.js$
 Allow: /misc/*.js?
 Allow: /misc/*.gif
 Allow: /misc/*.jpg
 Allow: /misc/*.jpeg
 Allow: /misc/*.png
 Allow: /modules/*.css$
 Allow: /modules/*.css?
 Allow: /modules/*.js$
 Allow: /modules/*.js?
 Allow: /modules/*.gif
 Allow: /modules/*.jpg
 Allow: /modules/*.jpeg
 Allow: /modules/*.png
 Allow: /profiles/*.css$
 Allow: /profiles/*.css?
 Allow: /profiles/*.js$
 Allow: /profiles/*.js?
 Allow: /profiles/*.gif
 Allow: /profiles/*.jpg
 Allow: /profiles/*.jpeg
 Allow: /profiles/*.png