# Robots.txt for www.pingram.io
# Updated for better SEO and crawler management

# Allow all crawlers access to public content
User-agent: *
Allow: /

# Prevent indexing of search results or internal pages
Disallow: /search*
Disallow: /_*
Disallow: /admin*
Disallow: /*.json$

# Allow sitemap files (must come before generic XML disallow)
Allow: /sitemap*.xml

# Block other XML files (API responses, data files, etc.)
Disallow: /*.xml$

# Block malformed docs URLs and component files
Disallow: /docs/*/docs/
Disallow: /docs/components/*.astro
Disallow: /**/NotificationAPI$
Disallow: /docs/*/NotificationAPI$

# Respect crawl delay to avoid overwhelming the server
Crawl-delay: 1

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: facebookexternalhit
Allow: /

# Block aggressive crawlers that don't respect rate limits
User-agent: SemrushBot
Disallow: /

# Allow AhrefsBot for SEO monitoring but with crawl delay
User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: MJ12bot
Disallow: /

# AI Crawlers - Allow all for maximum AEO visibility
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Bytespider
Allow: /

# Host directive for domain canonicalization
Host: https://www.pingram.io

# Sitemap location
Sitemap: https://www.pingram.io/sitemap-index.xml

# LLM-readable content
# llms.txt: https://www.pingram.io/llms.txt
# llms-full.txt: https://www.pingram.io/llms-full.txt

# Note: The app subdomain (app.pingram.io) has its own robots.txt
# that blocks all crawlers since it contains private application content