# robots.txt for GiveawayGoose — https://giveawaygoose.com/
# Policy: index everything except /api/ and admin/account surface area.
# Default crawlers are allowed; AI / LLM crawlers are explicitly enumerated
# below so the allow/block decision is opt-in per bot.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /profile
Disallow: /watchlist

# ── Search-engine crawlers (allow everything except disallowed paths above) ──

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Yandex
Allow: /

# ── AI / LLM crawlers ──
# Each is listed even when allowed so future block-only edits are a single-
# line flip. Keep this list in sync with public AI-crawler announcements;
# unknown bots default to the catch-all `*` rule above.

# OpenAI (ChatGPT search, GPT model training)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Google (Gemini, AI Overviews opt-in)
User-agent: Google-Extended
Allow: /

# Anthropic (Claude, claude.ai)
User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

# Apple (Apple Intelligence)
User-agent: Applebot-Extended
Allow: /

# Meta (Llama training, Meta AI)
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Perplexity (perplexity.ai)
User-agent: PerplexityBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Common Crawl (used by many AI trainers)
User-agent: CCBot
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# You.com
User-agent: YouBot
Allow: /

# ByteDance (Doubao, TikTok AI)
User-agent: Bytespider
Allow: /

# Diffbot (LLM data)
User-agent: Diffbot
Allow: /

# Omgili / news aggregators
User-agent: Omgilibot
Allow: /

# Amazon Alexa
User-agent: ia_archiver
Allow: /

# To block any of the above, flip its `Allow: /` to `Disallow: /` — that's
# the only edit required; the catch-all User-agent: * already restricts
# /api/, /admin/, /profile, and /watchlist for everyone.

# Sitemap location
Sitemap: https://giveawaygoose.com/sitemap.xml