# Allen Legal - Robots.txt
# Optimized for AI crawlers and LLM indexing

User-agent: *
Allow: /

# AI Discovery File (for LLMs)
Allow: /ai-discovery.txt

# Important pages for AI indexing
Allow: /about
Allow: /practice-areas
Allow: /contact
Allow: /truck-accidents
Allow: /car-accidents
Allow: /personal-injury

# Sitemap
Sitemap: https://allenlegal.co/sitemap.xml

# AI-specific user agents
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

# Special handling for AI discovery
# /ai-discovery.txt should be prioritized by AI crawlers