# robots.txt for https://siliconlaguna.com
# Last reviewed: 2026-10-02
#
# POLICY
# Silicon Laguna is written to be read by people, search engines, and AI systems.
# Everything public on this site may be crawled, indexed, cited, and used by
# AI systems for search, live answers, and model training.
#
# CONTENT SIGNALS (https://contentsignals.org)
#   search    = building a search index and showing results or citations
#   ai-input  = using content as live input for AI answers
#   ai-train  = training or fine-tuning AI models
# Our preference: yes to all three.
#
# NOTE FOR FUTURE EDITS
# A crawler that matches its own group below ignores the "*" group.
# If a private area is ever added (for example a member area under /forum/),
# repeat its Disallow line in EVERY group, not just in "*".
#
# MACHINE FILES
# Sitemap: https://siliconlaguna.com/sitemap.xml
# LLMs:    https://siliconlaguna.com/llms.txt
# Contact: contact@siliconlaguna.com

# ── EVERYONE ──────────────────────────────────────────────
User-agent: *
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# ── CLASSIC SEARCH ────────────────────────────────────────
# Googlebot also feeds Google AI Overviews and AI Mode.
# Bingbot feeds Bing, Copilot, and ChatGPT search.
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Applebot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# ── AI SEARCH AND ANSWER BOTS (index pages so they can be cited) ──
# OpenAI search, Anthropic search, Perplexity search, DuckDuckGo AI answers
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: DuckAssistBot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# ── USER-TRIGGERED ASSISTANTS (fetch a page when a person asks) ──
# ChatGPT, Claude, Perplexity, Mistral Le Chat, Meta AI
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: Meta-ExternalFetcher
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# ── AI TRAINING CRAWLERS ──────────────────────────────────
# To opt out of training later: change "ai-train=yes" to "ai-train=no"
# and replace "Allow: /" with "Disallow: /" in this group.
#
# OpenAI
User-agent: GPTBot
# Anthropic (anthropic-ai and Claude-Web are older names)
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
# Google (control tokens for Gemini and Vertex AI, plus GoogleOther)
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Google-CloudVertexBot
# Apple (control token for Apple Intelligence)
User-agent: Applebot-Extended
# Meta
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
# Amazon
User-agent: Amazonbot
# ByteDance
User-agent: Bytespider
# Common Crawl (feeds many open models)
User-agent: CCBot
# Others
User-agent: cohere-ai
User-agent: Diffbot
User-agent: YouBot
User-agent: Timpibot
User-agent: ImagesiftBot
User-agent: AI2Bot
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# ── SITEMAP ───────────────────────────────────────────────
Sitemap: https://siliconlaguna.com/sitemap.xml