# Robots.txt for Leo AI
# https://leo-ai.io

# Default rules for all crawlers
User-agent: *
Allow: /

# Important content - explicitly allow
Allow: /blog/
Allow: /events/
Allow: /about-leo-ai/
Allow: /privacy.html
Allow: /data-deletion-request.html
Allow: /index.html

# Block admin and API routes
Disallow: /api/
Disallow: /admin/

# Block internal/development files
Disallow: /.claude/
Disallow: /*.py
Disallow: /ramp-graphics.html
Disallow: /carousels/

# Block raw video files (users should find these through pages, not direct crawl)
Disallow: /*.mp4
Disallow: /*.webm
# Except the launch video pages' own videos, so search can see and show them (plan 2026-10-12).
# The longer rule wins over /*.mp4 for this folder only; every other video stays blocked.
Allow: /use-cases/product-launch-videos/*.mp4

# Block 404 page from indexing
Disallow: /404.html

# Sitemap location
Sitemap: https://leo-ai.io/sitemap.xml

# GPTBot (OpenAI) - allow for AI visibility
User-agent: GPTBot
Allow: /

# Google-Extended (Gemini) - allow for AI visibility
User-agent: Google-Extended
Allow: /

# Anthropic Claude - allow for AI visibility
User-agent: anthropic-ai
Allow: /

# Bing crawler specific (supports crawl-delay)
User-agent: Bingbot
Crawl-delay: 1
Allow: /