# ─────────────────────────────────────────────────────────────────────
# Content Signals (contentsignals.org / IETF AIPREF draft)
# Declares how automated systems may USE content they're allowed to access:
#   search   = yes  → OK to index for search results
#   ai-input = yes  → OK to use as input for AI answers (RAG / grounding)
#   ai-train = no   → NOT OK to use for training AI / ML models
# In short: cite us in AI answers and search, but don't train models on us.
# A Content-Signal is scoped to its User-agent group (like Allow), so it is
# repeated per group below. This expresses a preference, not access control.
# ─────────────────────────────────────────────────────────────────────

# Search-engine crawlers: always allowed
User-agent: Googlebot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Bingbot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# AI search + training crawlers: explicit allow so they index us.
# (Some hosts default-block these via WAF rules; explicit Allow tells them
# we want our content visible to AI search engines like ChatGPT, Claude,
# Perplexity, Google AI Overviews. Foundation pillar.)
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Claude-Web
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Applebot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Bytespider
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: cohere-ai
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: meta-externalagent
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Default: everything else may crawl
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Sitemap (Astro generates sitemap-index.xml + sitemap-0.xml automatically)
Sitemap: https://getintel.ai/sitemap-index.xml