# Lumify Robots.txt
# https://lumify.ai — Sports Intelligence API for AI agents & developers
#
# POLICY (A4)
# - Public marketing + docs: open to search engines and AI crawlers
#   (both training bots and retrieval/search bots). Lumify is built to be
#   discovered and cited by agents.
# - Authenticated app, private APIs, auth flows, and retired AI-search
#   product URLs: closed.
# - Query-string URLs are disallowed to prevent UTM/ref duplicates (see A1).
#
# Named User-agent blocks replace User-agent: * for that bot (they do not
# merge). Private-path Disallow rules are therefore repeated below.

User-agent: *
Allow: /

# Sitemap index (pages.xml = HTML SEO; agents.xml = llms/OpenAPI/MCP/.md).
Sitemap: https://lumify.ai/sitemap.xml

# LLM-readable documentation (comments only — not robots directives).
# Agent Skill:    https://lumify.ai/SKILL.md
# Capabilities:   https://lumify.ai/capabilities.json
# MCP card:       https://lumify.ai/.well-known/mcp/server-card.json
# LLMs:           https://lumify.ai/llms.txt
# LLMs-full GEO:  https://lumify.ai/llms-full.txt
# LLMs-full docs: https://lumify.ai/docs/llms-full.txt
# OpenAPI dump:   https://lumify.ai/openapi-llms.txt
# Manifest:       https://lumify.ai/.well-known/agent.json
# OpenAPI:        https://lumify.ai/openapi.json
# MCP:            https://lumify.ai/mcp
# AI setup:       https://lumify.ai/docs/ai

# --- Private / non-indexable ---
Disallow: /app/
Disallow: /admin/
Disallow: /dashboard
Disallow: /account
Disallow: /billing
Disallow: /api-keys
Disallow: /usage
Disallow: /analytics
Disallow: /playground
Disallow: /settings
Disallow: /oauth
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /verify-beta
Disallow: /beta-confirmation
Disallow: /api/
Disallow: /v1/
Disallow: /health
Disallow: /health-check
Disallow: /cdn-cgi/
Disallow: /*?*

# Legacy WordPress paths (pre-migration)
Disallow: /knowledgebase-category/
Disallow: /knowledgebase-tag/
Disallow: /tag/
Disallow: /category/
Disallow: /feed/
Disallow: /*/feed/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /wp-admin/

# Retired AI-search product URLs (also 410 in app — A2)
Disallow: /ai-search-for-
Disallow: /blog
Disallow: /blog/
Disallow: /beta-signup
Disallow: /beta-terms
Disallow: /try-it
# Legacy /compare and /compare/ return 410 (retired AI-search). Do not use
# "Disallow: /compare" — that prefix would also hide /compare/* landings.
Disallow: /compare$
Disallow: /compare/$
Disallow: /frequently-asked-questions

# ---------------------------------------------------------------------------
# AI crawlers — explicitly welcomed on public surfaces (same private disallows).
# Includes training + search/retrieval + user-triggered fetch bots.
# ---------------------------------------------------------------------------

User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Amazonbot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: Meta-ExternalAgent
User-agent: CCBot
User-agent: cohere-ai
User-agent: AI2Bot
User-agent: DuckAssistBot
User-agent: MistralAI-User
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /dashboard
Disallow: /account
Disallow: /billing
Disallow: /api-keys
Disallow: /usage
Disallow: /analytics
Disallow: /playground
Disallow: /settings
Disallow: /oauth
Disallow: /login
Disallow: /logout
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /verify-beta
Disallow: /beta-confirmation
Disallow: /api/
Disallow: /v1/
Disallow: /health
Disallow: /health-check
Disallow: /cdn-cgi/
Disallow: /*?*
Disallow: /knowledgebase-category/
Disallow: /knowledgebase-tag/
Disallow: /tag/
Disallow: /category/
Disallow: /feed/
Disallow: /*/feed/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /wp-admin/
Disallow: /ai-search-for-
Disallow: /blog
Disallow: /blog/
Disallow: /beta-signup
Disallow: /beta-terms
Disallow: /try-it
Disallow: /compare$
Disallow: /compare/$
Disallow: /frequently-asked-questions

# Google-Extended is a Gemini-training control token (not an independent crawler).
# Allow = eligible for Gemini grounding/training from Googlebot-fetched content.
User-agent: Google-Extended
Allow: /