# Mana — mana.am
# Search engines may index marketing, documentation, creator profiles, and
# descriptive creation pages. Executable creations, source/bundle assets, and
# machine/bulk data surfaces are not crawl targets.

User-agent: *
Allow: /
# Content Signals — search indexing is allowed, and so is using this content to
# answer a question someone just asked (RAG / grounding / citation). Training a
# model on it is not. See contentsignals.org.
#
# `ai-input=yes` is the deliberate half of that pair. Mana publishes an MCP
# server that hands agents the public catalogue on request; declaring "do not
# use our content in answers" while doing so would be incoherent. The
# permission covers citation, not reproduction — republishing a creation
# elsewhere is still prohibited, see the "Content use" section of
# https://mana.am/llms.txt.
Content-Signal: search=yes, ai-input=yes, ai-train=no, use=reference
# Allow social-card scrapers to fetch the dynamic OG generator. This MUST
# come before `Disallow: /api/` because longest-match-wins in the robots.txt
# spec — without it, Twitterbot / Facebookbot / Slackbot / etc. would refuse
# to fetch the og:image and shares would render as broken-image placeholders.
Allow: /api/og
# The fixture sandbox is documentation that happens to answer JSON: fixed fake
# records, no production read, no rate-limit cost. It must sit above
# `Disallow: /api/` (longest match wins) or an agent scanner cannot verify that
# the safe test surface we advertise actually exists.
Allow: /api/sandbox
Allow: /api/sandbox/
# Apex Hub-and-Spoke surfaces (Phase 2). Explicit Allow lines are redundant
# given the umbrella `Allow: /`, but spelling them out makes the SEO surface
# intent legible to anyone auditing the file later — and reassures crawlers
# that don't fully understand longest-match precedence.
Allow: /apps
Allow: /apps/
Allow: /u/
Allow: /community
Allow: /categories/
Allow: /trending
Allow: /new

# Runtime, creation payload, and bulk/data surfaces. Descriptive `/apps/*`
# pages above remain indexable; these paths either expose non-HTML data, bypass
# the descriptive shell, or can enumerate public creations at scale.
Disallow: /public/share/
Disallow: /batch
Disallow: /ask
Disallow: /ask/
Disallow: /*/ask
Disallow: /nlweb/ask
Disallow: /mcp
Disallow: /mcp/
Disallow: /webmcp
Disallow: /webmcp/
Disallow: /docs/mcp
Disallow: /.well-known/mcp
Disallow: /.well-known/mcp/
Disallow: /.well-known/webmcp
Disallow: /c/
Disallow: /share/
# Direct player/bundle modes used by the browser shell, never search results.
Disallow: /*?*embed=shell*
Disallow: /*?*toolbar=1*

Disallow: /admin
Disallow: /points
Disallow: /invite/
Disallow: /api/
# Build metadata consumed by the daily prerender refresh workflow
# (diff/skip/IndexNow). Not user-facing content.
Disallow: /_seo-manifest.json
Disallow: /_apex-meta-cache.json

# AI / LLM crawlers — training is fully blocked. Search/citation agents use the
# generic group above, so they may index descriptive pages but inherit every
# runtime/API/bulk-path prohibition. Do not add agent-specific `Allow: /`
# groups: a specific group would stop inheriting those protected-path rules.
#
# Every vendor runs separate user-agents for the two jobs, and the rules are
# NOT inherited: blocking ClaudeBot does not block Claude-SearchBot, and
# blocking GPTBot does not block OAI-SearchBot. Getting this backwards is the
# common failure — you either feed training pipelines you meant to opt out of,
# or you vanish from the AI answers you wanted to appear in.
#
# Fetching and using are separate levers, and the split runs along the
# citation / training line rather than the crawler / no-crawler line.
#
# Retrieval and citation agents are open on both levers: open at the crawl
# level because they are reading only because a person just asked about Mana,
# and open at the use level (`ai-input=yes`) because being the answer to "how do
# I build an iPhone app without coding" is the point. The public MCP server at
# /mcp hands those same agents the public catalogue on request, so any other
# setting here would contradict the product.
#
# Training stays closed on both levers: `ai-train=no` above, the TDM
# reservation at /.well-known/tdmrep.json under Art. 4 of EU Directive
# 2019/790, and a crawl-level block for each training agent below — for those,
# fetching has no user-facing purpose to preserve.
#
# Citation is not republication. Quoting Mana, describing a creation, and
# linking to it are all fine; copying a creation's files to another host is not,
# and that prohibition is unaffected by anything in this file. See the
# "Content use" section of https://mana.am/llms.txt.
#
# Two of these are not crawlers at all: Google-Extended and Applebot-Extended
# are opt-out directives for Googlebot / Applebot. They never appear in access
# logs, and blocking them costs nothing in Search / Siri / Spotlight ranking.
#
# Cloudflare's MANAGED robots.txt (Security Settings → Bot traffic → "Set your
# preference to block training in robots.txt") prepends the same training
# blocks above this section. That is consistent with the policy here, so the
# two can coexist — but if you ever flip an agent below back to `Allow`, check
# the SERVED file, because the managed block will contradict it silently
# (`curl https://mana.am/robots.txt` — a byte count above this source means
# something was injected).

# ── Training: closed ────────────────────────────────────────────────
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ByteSpider
Disallow: /

User-agent: meta-externalagent
Disallow: /

Sitemap: https://mana.am/sitemap.xml

# AI-facing context index (the robots.txt of LLMs).
# See https://mana.am/llms.txt
# Structured feed map contains indexable documentation only.
Schemamap: https://mana.am/schema-map.xml