# Robots.txt for GEO Scan - AI-Powered Generative Engines Optimizer
# This file provides guidelines for search engine crawlers and AI bots

# Allow all web crawlers to access the site
User-agent: *
Allow: /

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

# AI and LLM crawlers
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Crawl-delay: 1

# Block access to sensitive areas
Disallow: /api/
Disallow: /admin/
Disallow: /.env
Disallow: /config/
Disallow: /logs/
Disallow: /temp/
Disallow: /cache/

# Allow access to important SEO files
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /favicon.svg
Allow: /og-image.svg
Allow: /llm.txt

# Sitemap location
Sitemap: https://geoscan.in/sitemap.xml

# Additional AI-specific directives
# These help AI models understand content usage permissions
AI-crawl: allow
AI-index: allow
AI-cite: allow
AI-reference: allow

# Content licensing for AI training
# Specify how AI models can use the content
Content-license: CC-BY-4.0
AI-training: allow
AI-generation: allow-with-attribution

# Crawl optimization
Crawl-delay: 1
Request-rate: 1/1s

# Last updated
# Updated: 2024-01-20