robots.txt details

saved: size: 1646 kb md5 checksum: bbe0c4f0d07736ad3a32a2b5ab0eb8f0

websites using this robots.txt

content


# Robot exclusion file
User-agent: *

# The following pages require registration and login

Disallow: /session-img/
Disallow: /invalid-request/
Disallow: /document/
Disallow: /apply-profile/

Disallow: */searchjobs/*
Disallow: */jobsrss/*
Disallow: /jobsrss/*
Disallow: */jbequicksignup/*
Disallow: /your-jobs/*
Disallow: /external-redirect-registration* 
Disallow: */emailjob/*

# The following pages are not searchable

Disallow: /remindme/
Disallow: /moreterms/
Disallow: /logon/
Disallow: /editprofile/
Disallow: /newalert/
Disallow: /profilecv/
Disallow: /register/
Disallow: /?loggedOff=true/
Disallow: /searchjobs/
Disallow: /twitterfeeds/ 
Disallow: /jobsrss/
Disallow: /privacy-policy/
Disallow: /refund-policy/
Disallow: /callback-request/
Disallow: /terms-and-conditions/
Disallow: /help/
Disallow: /about-us/
Disallow: /accessibility/
Disallow: /jbequicksignup/
Disallow: /confirmjsemailaddress/
Disallow: /resetpassword/
Disallow: /error500.html?aspxerrorpath=/
Disallow: /?keywords=
Disallow: /jobs/*id=
Disallow: /session-img/
Disallow: /invalid-request/
Disallow: /document/
Disallow: /apply-profile/
Disallow: *sort=*

# OCT 18

Disallow: /jobs*/-150-000-or-more/*
Disallow: /jobs*/-10-0000-19-999/*
Disallow: /jobs*/up-to-300/*
Disallow: /your-jobs/*
Disallow: /jobs*/permanent/*
Disallow: /jobs*/recruitment-consultant/*
Disallow: /jobs*/direct-employer/*

Disallow: /*/*/*/*/
Noindex: /*/*/*/*/


# The following URLs are bot crawl delay instructions

User-agent: SemrushBot-SA 
crawl-delay: 10

Sitemap: https://www.energyjobline.com/sitemapindex.xml