# robots.txt for https://docs.banxa.com
# Format: RFC 9309 (https://www.rfc-editor.org/rfc/rfc9309)
#
# This site is the public developer documentation for the Banxa APIs.
# Everything published here is intended to be read, indexed, and cited —
# by search engines and by AI agents alike. There is no private content.
#
# Machine-readable entry points:
#   https://docs.banxa.com/llms.txt                 - index of the documentation
#   https://docs.banxa.com/llms-full.txt            - full documentation text
#   https://docs.banxa.com/.well-known/api-catalog  - API catalog (RFC 9727)
#   https://docs.banxa.com/auth.md                  - how to obtain API credentials
#   https://docs.banxa.com/sitemap.xml              - canonical page list
#   https://docs.banxa.com/mcp                      - documentation MCP server
#
# Content-Signal (https://contentsignals.org/) states how Banxa prefers this
# content to be used. It expresses a preference, not a grant of rights, and it
# does not change the terms under which this documentation is published:
#   search    - build a search index and link back to these pages
#   ai-input  - retrieve these pages to answer a user's question (RAG, agents)
#   ai-train  - train or fine-tune a generative model on this content
# All three are yes for crawlers that feed answer engines: this is public
# integration documentation, and the purpose of publishing it is for developers
# and their agents to find and use it. Bulk training-corpus crawlers that never
# return a citation or a visitor are disallowed further down.

# Default group: all crawlers, full access.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:
# Non-standard, honoured by Bing and others. The origin rate-limits bursts,
# so pacing requests avoids 429 responses.
Crawl-delay: 1

# ---------------------------------------------------------------------------
# AI crawlers and assistants, listed individually so the permission is explicit
# rather than inherited from the wildcard group above. Same policy for every
# one of them: full access to the whole site.
# ---------------------------------------------------------------------------

# OpenAI - model training
User-agent: GPTBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# OpenAI - ChatGPT search index
User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# OpenAI - user-initiated browsing
User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# Anthropic - crawler
User-agent: ClaudeBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# Anthropic - legacy crawler identifier
User-agent: Claude-Web
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# Anthropic - user-initiated fetch
User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# Anthropic - search index
User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# Anthropic - legacy identifier
User-agent: anthropic-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# Google - Gemini and Vertex AI grounding
User-agent: Google-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# Apple - Apple Intelligence
User-agent: Applebot-Extended
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# Perplexity - crawler
User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# Perplexity - user-initiated fetch
User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# Meta - AI crawler
User-agent: Meta-ExternalAgent
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# Amazon
User-agent: Amazonbot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# Cohere
User-agent: cohere-ai
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# DuckDuckGo - DuckAssist
User-agent: DuckAssistBot
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow:

# ---------------------------------------------------------------------------
# Training-corpus crawlers. These collect content in bulk to build model
# training sets; they do not power an answer engine, so they never cite this
# documentation or send a developer here. Banxa does not consent to that use.
# ---------------------------------------------------------------------------

# Common Crawl
User-agent: CCBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

# ByteDance
User-agent: Bytespider
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

Sitemap: https://docs.banxa.com/sitemap.xml