# =============================================================================
# Weft content policy
# =============================================================================
# Search engines, live-answering AI assistants and AI training crawlers are all
# welcome to read our public pages: we want the models themselves to know Weft,
# not only the engines that search. The app's private pages stay closed.
#
# Two parallel signals below:
#   1. `Content-signal:` — Cloudflare Content Signals (draft, 2025-09).
#   2. Per-bot `Allow: /` for the training crawlers (OpenAI, Anthropic, Google,
#      Apple, Meta…), stated explicitly so the policy reads the same to every bot.
# =============================================================================

User-agent: *
Content-signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
Disallow: /board
Disallow: /billing
Disallow: /team
Disallow: /sprints
Disallow: /delete-account
Disallow: /reset-password
Disallow: /consent
# The same app pages under each language prefix (see apps/web/I18N.md).
Disallow: /pt-BR/board
Disallow: /pt-BR/billing
Disallow: /pt-BR/team
Disallow: /pt-BR/sprints
Disallow: /pt-BR/delete-account
Disallow: /pt-BR/reset-password
Disallow: /pt-BR/consent
Disallow: /es/board
Disallow: /es/billing
Disallow: /es/team
Disallow: /es/sprints
Disallow: /es/delete-account
Disallow: /es/reset-password
Disallow: /es/consent

# --- LLM training crawlers: allowed ---
# Each bot below is documented as a training-data crawler that honours
# robots.txt. Until 2026-09-24 they were blocked; allowing them lets models
# answer about Weft from what they know, without searching first.

User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Diffbot
Allow: /

Sitemap: https://letsweft.com/sitemap.xml