#     T___
#     |q p|
#     \_`_/
#  (m9\:::/\
#     /___\6
#     // \\
#    _\\ //_
#
# We wrote a whole post about how this file should look.
# https://believe.digital/thinking/robots-txt-ai-visibility/
#
# ----------------------------------------------------------------------
# robots.txt : maximum AI visibility
# Site: www.believe.digital
# Last reviewed: 2026-09-15
# ----------------------------------------------------------------------

# 1. BASELINE FOR EVERY CRAWLER
#    Our own paths: admin UI, internal API, team publishing flow, booking
#    confirmation page, and the homepage A/B variant (duplicate of /).
#    Keep this list as short as you can defend. Anything disallowed here
#    is also invisible to every AI answer engine.
#    Whatever you put here, repeat it in group 2. See note 2.
User-agent: *
Disallow: /admin/
Disallow: /api/
Disallow: /post/
Disallow: /call-scheduled/
Disallow: /variant-b/
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# 2. AI-SEARCH INDEX BOTS. These decide whether you are eligible to be
#    cited at all. Named explicitly because Applebot and Amzn-SearchBot
#    otherwise inherit whatever rules you wrote for Googlebot.
User-agent: Googlebot
User-agent: bingbot
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Applebot
User-agent: Amzn-SearchBot
User-agent: meta-webindexer
#    The next five lines are NOT a duplication error. They are the same
#    rules as group 1, repeated, because this group does not inherit them.
#    If you edit group 1 and not this, you have just opened those paths to
#    every bot named above. Keep the two lists identical.
Disallow: /admin/
Disallow: /api/
Disallow: /post/
Disallow: /call-scheduled/
Disallow: /variant-b/

# 3. LIVE USER-TRIGGERED FETCHERS. A human is waiting on an answer right
#    now. These are the hits that correlate with citations and clicks.
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: Amzn-User
User-agent: Google-GeminiNotebook
User-agent: Google-Agent
User-agent: meta-externalfetcher
Allow: /

# 4. LINK PREVIEWS AND SOCIAL UNFURLS. Blocking these breaks your share
#    cards and gains you nothing on the AI front.
User-agent: facebookexternalhit
Allow: /

# 5. TRAINING CRAWLERS AND TRAINING CONTROL TOKENS. Allowed in this
#    variant. Listed explicitly so the decision is on the record rather
#    than an accident of omission. See Variant B to reverse it.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
Allow: /

Sitemap: https://believe.digital/sitemap.xml