# Robots.txt for gyld.ai
# Last updated: 2025

# Allow all crawlers
User-agent: *
Allow: /

# Disallow private/API routes
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /oauth/
Disallow: /_next/
Disallow: /test-*
Disallow: /debug-*
Disallow: /fix-*

# Disallow removed/non-existent pages
Disallow: /resources
Disallow: /chatbot
Disallow: /leads

# Explicitly allow all public marketing pages
Allow: /employees
Allow: /employees/*
Allow: /sales
Allow: /support
Allow: /operations
Allow: /finance
Allow: /solo
Allow: /blog
Allow: /blog/*
Allow: /signup
Allow: /login
Allow: /terms
Allow: /privacy
Allow: /security
Allow: /email

# SEO pages
Allow: /vs
Allow: /vs/*
Allow: /company-brain-for
Allow: /company-brain-for/*

# Googlebot specific
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Bingbot specific
User-agent: Bingbot
Allow: /
Crawl-delay: 0

# OpenAI GPTBot
User-agent: GPTBot
Allow: /

# Common AI/ML crawlers
User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Sitemap location
Sitemap: https://gyld.ai/sitemap.xml

# LLM context file
# See https://llmstxt.org for specification
Allow: /llms.txt