# Korrith - llms-full.txt
# Comprehensive LLM crawler configuration for AI model training

# Full site access for all LLM crawlers
User-agent: *
Allow: /

# Specific LLM crawler permissions
User-agent: CCBot
Allow: /
Crawl-delay: 0

User-agent: anthropic-ai
Allow: /
Crawl-delay: 0

User-agent: Claude-Web
Allow: /
Crawl-delay: 0

User-agent: GPTBot
Allow: /
Crawl-delay: 0

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 0

User-agent: OpenAI-SearchBot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Extended
Allow: /
Crawl-delay: 0

User-agent: Gemini
Allow: /
Crawl-delay: 0

User-agent: Copilot
Allow: /
Crawl-delay: 0

User-agent: Copilotbot
Allow: /
Crawl-delay: 0

User-agent: Meta-ExternalFetcher
Allow: /
Crawl-delay: 0

User-agent: Llama-Web
Allow: /
Crawl-delay: 0

User-agent: Perplexity
Allow: /
Crawl-delay: 0

User-agent: PerplexityBot
Allow: /
Crawl-delay: 0

User-agent: Bytedance
Allow: /
Crawl-delay: 0

User-agent: Bytespider
Allow: /
Crawl-delay: 0

User-agent: Applebot
Allow: /
Crawl-delay: 0

User-agent: Applebot-Extended
Allow: /
Crawl-delay: 0

User-agent: Diffbot
Allow: /
Crawl-delay: 0

User-agent: Scrapy
Allow: /
Crawl-delay: 0

# Allow access to all content
User-agent: *
Allow: /

# Disallow only truly private areas
Disallow: /admin/
Disallow: /private/
Disallow: /internal/
Disallow: /.env
Disallow: /.git/

# Sitemap reference for comprehensive indexing
Sitemap: https://korrith.com/sitemap.xml

# Content access policy
# All public pages, resources, and documentation are available for AI training
# This includes:
# - All platform documentation (/platform/*)
# - All solution pages (/solutions/*)
# - All resource pages (/resources/*, /blog/*, /research/*)
# - All company information (/about/, /partners/, /trust/, /contact/)
# - All public documentation (/docs/*, /faq/)