# ============================================
# IdeaProof.io - Robots.txt
# Last Updated: January 17, 2026
# ============================================

# Allow all crawlers by default
User-agent: *
Allow: /
Allow: /lists/
Allow: /guides/
Allow: /tools/
Allow: /versus/
Allow: /questions/
Allow: /calculators/
Allow: /blog/
Allow: /assets/
Allow: /fundraising-benchmarks
Allow: /*.js
Allow: /*.css
Crawl-delay: 1

# Protected areas - block from all crawlers
Disallow: /admin/
Disallow: /dashboard/
Disallow: /auth/
Disallow: /api/internal/
Disallow: /*.json$
Disallow: /*?*apikey*
Disallow: /*?*token*
Disallow: /.env


# ============================================
# LLM & AI CRAWLERS (Priority Access)
# ============================================

# OpenAI (GPT, ChatGPT)
User-agent: GPTBot
Allow: /
Allow: /blog/
Allow: /lists/
Allow: /features
Allow: /pricing
Allow: /about

Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 1

# Anthropic (Claude)
User-agent: anthropic-ai
Allow: /
Allow: /lists/

Crawl-delay: 1

User-agent: ClaudeBot
Allow: /
Allow: /blog/
Allow: /lists/

Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Crawl-delay: 1

# Google (Gemini, AI Overviews)
User-agent: Google-Extended
Allow: /
Crawl-delay: 1

User-agent: GoogleOther
Allow: /
Crawl-delay: 2

User-agent: Googlebot
Allow: /
Crawl-delay: 1

# Perplexity
User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

# Meta (Llama AI)
User-agent: FacebookBot
Allow: /
Crawl-delay: 2

User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 2

# Amazon (Alexa)
User-agent: Amazonbot
Allow: /
Crawl-delay: 2

# Apple (Siri, Apple Intelligence)
User-agent: Applebot
Allow: /
Crawl-delay: 1

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Crawl-delay: 1

# Microsoft (Copilot, Bing)
User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: msnbot
Allow: /
Crawl-delay: 2

# Cohere AI
User-agent: cohere-ai
Allow: /
Crawl-delay: 2

# You.com
User-agent: YouBot
Allow: /
Crawl-delay: 2

# Common Crawl
User-agent: CCBot
Allow: /blog/
Allow: /lists/
Allow: /features
Allow: /pricing
Allow: /about

Crawl-delay: 5

# Diffbot
User-agent: Diffbot
Allow: /
Crawl-delay: 3

# xAI (Grok)
User-agent: Grok
Allow: /
Crawl-delay: 2

# DeepSeek
User-agent: DeepSeekBot
Allow: /
Crawl-delay: 2

# Mistral AI
User-agent: MistralBot
Allow: /
Crawl-delay: 2

# ============================================
# SEO CRAWLERS (Rate Limited)
# ============================================

User-agent: SemrushBot
Allow: /
Crawl-delay: 10

User-agent: AhrefsBot
Allow: /
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

# ============================================
# SITEMAPS
# ============================================

# Primary sitemap (served via rewrite from ideaproof.io/sitemap.xml)
Sitemap: https://ideaproof.io/sitemap.xml

# Backup sitemap (direct storage access)
Sitemap: https://aioenivvgvtwyhpdioue.supabase.co/storage/v1/object/public/sitemaps/sitemap.xml