robots.txt details

2e2168c1589fc2fe398aa0921eb76672

saved: | size: 1212kb | checksum: 2e2168c1589fc2fe398aa0921eb76672

websites using this robots.txt

content


User-agent: *
Disallow: /wp-admin/
#Disallow: /wp-includes/
Disallow: /wp-includes/certificates
Disallow: /wp-includes/SimplePie
Disallow: /wp-includes/Text
Disallow: /wp-includes/*.php$
Disallow: /eyeblaster/addineyeV2.html
Disallow: /wp-content/plugins
Disallow: /Just-For-Test
Disallow: /test/
Disallow: /recipes/
Disallow: /wine/
Disallow: /wine-jargon/
Disallow: /archive/

#Directories no longer & files exists#
Disallow: /forum/
Disallow: /watchdog/
Disallow: /cronjob/
Disallow: /wp-content/uploads/wpsc/
Disallow: /downloads


#Deny Ads Crawling#
Disallow: /*Insert_Click_Track_URLhttp://ad.wsod.com/

#Deny Attachements #
Disallow: /*attachment/

#Block Bad URLs with "..."#
Disallow: /*/*...


#Block Broken URLs Generated by Martiniad Script#
Disallow: /1
Disallow: /pubclick

#Private Directories#
Disallow: /newsletters/p/
Disallow: /members/p/
Disallow: /premium/p/
Disallow: /members-st/

#Preview Pages#
Disallow: /?p=*

#MCS Pages#
Disallow: /mcs/*

#Allow all user-agents access to ADS
Allow: /ads.txt

User-agent: Mediapartners-Google
Allow: /

User-agent: Googlebot-News
Allow: /

Sitemap: https://www.cheatsheet.com/sitemap.xml