# ==============================================================================
# robots.txt — www.optimizegeo.ai
# Last updated: 2026-09-10   Spec: RFC 9309
#
# BEFORE YOU EDIT THIS FILE, READ THIS:
#
# A crawler obeys ONLY the one group that names it and ignores every other group
# in the file, including "User-agent: *". Two rules follow from that:
#
#   1. Every crawler must be named explicitly. Anything unnamed falls through to
#      the wildcard group, and AI-visibility auditors score that fall-through as
#      "partial access", which surfaces as a Crawlability warning.
#   2. Every group must repeat the full Disallow list. A rule that is not
#      repeated inside a group does not apply to that group.
#
# Consecutive "User-agent:" lines with no rule between them form ONE shared
# group. That is valid and is what keeps this file readable.
#
# Do not delete groups to shorten the file. Do not remove the repeated Disallow
# lines. Both changes silently alter behaviour.
#
# Disallow paths have no trailing slash so they also match the section root
# (/auth, /onboarding, /dashboard, /setup). Public docs at /docs/onboarding
# are a different prefix and stay crawlable.
# ==============================================================================


# ------------------------------------------------------------------------------
# GROUP 1 — Search engines and AI answer engines. Full access to public content.
# These decide whether we appear in Google, ChatGPT, Gemini, Perplexity, Claude.
# ------------------------------------------------------------------------------

# Google
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: Google-InspectionTool
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: Google-CloudVertexBot
User-agent: Google-Agent
User-agent: AdsBot-Google

# Microsoft / Bing (also feeds Copilot retrieval)
User-agent: Bingbot
User-agent: msnbot

# Apple
User-agent: Applebot
User-agent: Applebot-Extended

# OpenAI
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: OAI-AdsBot

# Anthropic — current tokens
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot

# Anthropic — legacy tokens. Anthropic no longer documents these; they are kept
# only because several third-party auditing tools still test for them.
User-agent: anthropic-ai
User-agent: Claude-Web

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User

# Other AI crawlers and assistants
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Bytespider
User-agent: CCBot
User-agent: cohere-ai
User-agent: MistralAI-User
User-agent: YouBot
User-agent: Diffbot
User-agent: DuckDuckBot
User-agent: DuckAssistBot
User-agent: GrokBot
User-agent: Yandex
User-agent: Baiduspider

Allow: /
Disallow: /auth
Disallow: /onboarding
Disallow: /dashboard
Disallow: /setup
Disallow: /api/
Disallow: /ui-api/


# ------------------------------------------------------------------------------
# GROUP 2 — SEO and monitoring tools. Same access, named so their own audits
# report full access rather than partial.
# ------------------------------------------------------------------------------

User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: SemrushBot-SI
User-agent: SiteAuditBot
User-agent: rogerbot
User-agent: dotbot
User-agent: Screaming Frog SEO Spider

Allow: /
Disallow: /auth
Disallow: /onboarding
Disallow: /dashboard
Disallow: /setup
Disallow: /api/
Disallow: /ui-api/


# ------------------------------------------------------------------------------
# GROUP 3 — Every crawler not named above. Identical rules.
# ------------------------------------------------------------------------------

User-agent: *
Allow: /
Disallow: /auth
Disallow: /onboarding
Disallow: /dashboard
Disallow: /setup
Disallow: /api/
Disallow: /ui-api/


# ------------------------------------------------------------------------------
# Sitemap — a file-level directive, applies regardless of group.
# AI content guidance lives at /llms.txt (fetched separately; not a robots rule).
# ------------------------------------------------------------------------------

Sitemap: https://www.optimizegeo.ai/sitemap.xml