# Gadiambe.fr — robots.txt

# All well-behaved crawlers
User-agent: *
Allow: /
Disallow: /api/
Disallow: /recherche
Disallow: /cache/
Disallow: /data/
Disallow: /scripts/
Disallow: /config/
Disallow: /controllers/
Disallow: /includes/
Disallow: /templates/

# Aggressive SEO crawlers — block
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: DotBot
User-agent: MJ12bot
User-agent: BLEXBot
User-agent: DataForSeoBot
Disallow: /

# AI retrieval crawlers — allow (they cite sources and send traffic)
# GPTBot: ChatGPT web search, cites sources
# ClaudeBot: Claude web search, cites sources
# PerplexityBot: Perplexity search, cites sources
# YouBot: You.com search, cites sources
# ChatGPT-User: ChatGPT browsing mode
# Google-Extended: Gemini AI, cites sources
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: YouBot
Allow: /
User-agent: cohere-ai
Allow: /

# AI training-only crawlers — block (scrape without citation)
User-agent: CCBot
User-agent: Bytespider
User-agent: Diffbot
User-agent: Omgilibot
User-agent: Timpibot
User-agent: PetalBot
User-agent: ImagesiftBot
User-agent: FacebookBot
Disallow: /

Sitemap: https://gadiambe.fr/sitemap.xml