# robots.txt file for th.jobsdb.com

# Unrestricted access
User-agent: Mediapartners-Google
User-agent: AdIdxBot
Disallow:

# Default directives
User-agent: *
Disallow: */job/
Disallow: *?returnUrl=
Disallow: *?page=

# Disallowed bots
User-agent: LinkedInBot
User-agent: Baiduspider
User-agent: PetalBot
Disallow: /

# Exceptions
User-agent: anthropic-ai
User-agent: Bytespider
User-agent: CCBot
User-agent: Diffbot
User-agent: Google-Extended
User-agent: omgili
User-agent: GPTBot
Disallow: /companies
Disallow: */job/

User-agent: LinkedInBot
Allow: */job/