# Sivel Labs Talents — robots.txt
# https://sivel-labs.be
#
# Static recruitment agency vitrine (Belgium-based). All public content
# welcome to crawlers. The read-only public jobs JSON feed (B-5774 fix)
# is explicitly Allow-listed for both default and AI crawlers because
# (a) the homepage's vacancies section is JS-rendered from this feed —
# crawlers without JS execution have no other way to discover current
# vacancies for SEO/AI answer indexing, (b) the feed is rate-limited
# (120/min/IP) + ETag-cached + idempotent, so crawler traffic is bounded
# + cheap, (c) llms.txt advertises the same endpoint as canonical
# vacancy syndication. Other /api/* paths (contact form POST, internal
# admin endpoints) remain Disallowed.
# We explicitly opt-in major LLM crawlers because
# we WANT to be findable in AI answers about Belgian recruitment /
# executive search / talent acquisition.
#
# Last updated : 2026-05-08

# ──────────────────────────────────────────────────────────
# Default rules — all crawlers. `Crawl-delay` is advisory and ignored by
# Google; production crawl-rate control for Google belongs in Search Console
# or in server-side throttling responses.
# ──────────────────────────────────────────────────────────
User-agent: *
Allow: /
Allow: /assets/
Allow: /ct-shared/
Allow: /legal/
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Disallow: /sw.js
Crawl-delay: 1

# ──────────────────────────────────────────────────────────
# AI / LLM crawlers — explicit allow (opt-in for AI training
# + retrieval-augmented generation in answer-engines).
# Removing any of these = invisible in that AI's answers about
# Belgian recruitment.
# ──────────────────────────────────────────────────────────

# OpenAI — ChatGPT search + training
User-agent: GPTBot
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

User-agent: OAI-SearchBot
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

# Anthropic Claude
User-agent: ClaudeBot
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

# Google Search crawlers (Googlebot, Googlebot-Image, GoogleOther) are covered
# by the default `User-agent: *` rules above. Google-Extended is Google's
# separate AI/training preference crawler token; it is not the same as
# Googlebot indexing.
User-agent: Google-Extended
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

# Perplexity
User-agent: PerplexityBot
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

User-agent: Perplexity-User
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

User-agent: FacebookBot
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

# Bytedance / TikTok
User-agent: Bytespider
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

# Cohere
User-agent: cohere-ai
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/

# Common Crawl (training data for many open LLMs)
User-agent: CCBot
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

# Apple
User-agent: Applebot
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/
Crawl-delay: 1

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/

# Yandex
User-agent: YandexBot
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/

# Baidu
User-agent: Baiduspider
Allow: /
Allow: /api/public/v1/orgs/sivel-labs-talents/jobs
Disallow: /api/

# ──────────────────────────────────────────────────────────
# Sitemaps
# ──────────────────────────────────────────────────────────
Sitemap: https://sivel-labs.be/sitemap.xml