# ==============================================================================
# MLVA Global Search Engine & AI Discoverability Directives
# Target: https://mlva.web.app/
# Specification: RFC 9309 Robots Exclusion Protocol
# Last Verified: November 2026
# ==============================================================================

# Global Default: Allow search indexing for public documentation
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /_auth/

# ==============================================================================
# SECTION 1: SEARCH & USER-DIRECTED RETRIEVAL CRAWLERS (Live Discovery)
# ==============================================================================

# 1. Google Search & AI Overviews
# Docs: https://developers.google.com/search/docs/crawling-indexing/googlebot
User-agent: Googlebot
Allow: /

# 2. Microsoft Bing Search & Copilot Grounding
# Docs: https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0
User-agent: bingbot
Allow: /

# 3. OpenAI Search & Live User Fetching
# Docs: https://platform.openai.com/docs/bots
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# 4. Anthropic Search & Live User Fetching
# Docs: https://docs.anthropic.com/en/docs/resources/claude-bot
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# 5. Perplexity AI Search & Live User Fetching
# Docs: https://docs.perplexity.ai/docs/perplexitybot
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# 6. Apple Search & Siri Retrieval
# Docs: https://support.apple.com/en-us/119829
User-agent: Applebot
Allow: /

# 7. Amazon Search & Shopping Assistant Retrieval
# Docs: https://developer.amazon.com/amazonbot
User-agent: Amazonbot
Allow: /

User-agent: Amzn-SearchBot
Allow: /

User-agent: Amzn-User
Allow: /

# 8. Meta User Fetching & Social Previews
# Docs: https://developers.facebook.com/docs/sharing/webmasters/crawler/
User-agent: Meta-ExternalFetcher
Allow: /

User-agent: FacebookBot
Allow: /

# 9. Additional Standard Search Engines
User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /

# ==============================================================================
# SECTION 2: MODEL TRAINING CRAWLERS & ROBOTS.TXT POLICY TOKENS
# Allowed on public documentation to train model awareness of MLVA CLI syntax.
# Handled separately from search indexing. Does not affect private routes.
# ==============================================================================

# Google Generative AI Training Token (Controls Gemini / Vertex AI model training)
# Docs: https://developers.google.com/search/docs/crawling-indexing/google-extended
User-agent: Google-Extended
Allow: /

# Apple Foundation Model Training Token (Controls Apple Intelligence training)
# Docs: https://support.apple.com/en-us/119829
User-agent: Applebot-Extended
Allow: /

# OpenAI Pre-Training Crawler
# Docs: https://platform.openai.com/docs/bots
User-agent: GPTBot
Allow: /

# Anthropic Pre-Training Crawler
# Docs: https://docs.anthropic.com/en/docs/resources/claude-bot
User-agent: ClaudeBot
Allow: /

# Meta AI & Llama Pre-Training Crawler
# Docs: https://developers.facebook.com/docs/sharing/webmasters/crawler/
User-agent: meta-externalagent
Allow: /

# Common Crawl Foundation Data
# Docs: https://commoncrawl.org/big-picture
User-agent: CCBot
Allow: /

# ==============================================================================
# XML Sitemap Reference
# ==============================================================================
Sitemap: https://mlva.web.app/sitemap.xml