# WebDevSuite — robots.txt
# https://webdevsuite.dev

# ── Allow legitimate search engines and LLM agents ────────────────────────────
User-agent: *
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /tools/
Allow: /pricing
Allow: /about
Allow: /faq
Allow: /contact
Allow: /privacy-policy
Allow: /terms-of-service
Allow: /refund-policy

# ── Block Bot Traps & Sensitive Paths ───────────────────────────────────────
Disallow: /api/
Disallow: /profile/
Disallow: /settings/
Disallow: /admin/
Disallow: /analytics/
Disallow: /tickets/
Disallow: /login
Disallow: /signup
Disallow: /reset-password
Disallow: /logout
Disallow: /onboard

# ── Explicitly configure major LLM agents ───────────────────────────────────
# Allow access to LLM context files and public tool descriptions, but block everything else.
User-agent: GPTBot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /tools/
Disallow: /

User-agent: ChatGPT-User
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /tools/
Disallow: /

User-agent: Claude-Web
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /tools/
Disallow: /

User-agent: anthropic-ai
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /tools/
Disallow: /

User-agent: Google-Extended
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /tools/
Disallow: /

User-agent: PerplexityBot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /tools/
Disallow: /

User-agent: cohere-ai
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /tools/
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# ── Sitemap ───────────────────────────────────────────────────────────────────
Sitemap: https://webdevsuite.dev/sitemap.xml