# robots.txt for bigspaceship.com
# Last updated: 2026-05-01

# Default rule for all bots
User-agent: *
Allow: /
Disallow: /404
Disallow: /401

# Content usage signals (contentsignals.org)
# Search: yes — allow indexing for traditional + AI search
# AI training: no — do not use for model training without agreement
# AI input: yes — allow grounding/citation in AI answers with attribution
Content-Signal: search=yes, ai-train=no, ai-input=yes

# --- AI Search & Answer Engines (allowed: drives visibility) ---

# OpenAI ChatGPT search (user-initiated)
User-agent: OAI-SearchBot
Allow: /

# Perplexity (live answer fetching)
User-agent: PerplexityBot
Allow: /

# Anthropic Claude (live answer fetching)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Google AI Overviews / Gemini grounding
User-agent: Google-Extended
Allow: /

# Microsoft Copilot
User-agent: CCBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# --- AI Training Crawlers (disallowed: no training without agreement) ---

User-agent: GPTBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# Sitemap
Sitemap: https://www.bigspaceship.com/sitemap.xml

Sitemap: https://www.bigspaceship.com/sitemap.xml