# robots.txt for coffeejson.org
#
# Crawler policy: retrieval-only. Search indexing and AI retrieval/grounding are
# welcome; AI *training* crawlers are not. The training / bulk-scraping bots
# below are disallowed by name. ClaudeBot, Google-Extended, OAI-SearchBot,
# ChatGPT-User, PerplexityBot and Applebot are allowed by omission: with no group
# of their own they fall under `User-agent: *`.
#
# IMPORTANT — this is an access policy for this website, NOT a license term. The
# spec prose and the schema are CC0: once you have them, the license places no
# restriction on any use, training included. Disallowing a crawler here says
# "don't take it from this server in bulk"; it does not, and cannot, add a
# condition to a CC0 artifact. Any page describing this policy must say the same.

User-agent: *
# `/r` renders whatever document is in the URL. The `?d=` payload space is
# unbounded and every URL in it is thin, duplicate-shaped content — a crawl trap,
# not a privacy matter. Keep the bare `/r/` page and future `?s=<slug>` corpus
# links crawlable; keep the payload space out. Both shapes are spelled out
# because GitHub Pages serves the directory form and redirects the bare one to
# it, so a crawler can meet either.
Disallow: /r?d=
Disallow: /r/?d=
# Content Signals stance: allow search + AI input (retrieval/grounding), refuse
# AI training. See contentsignals.org.
Content-Signal: search=yes,ai-input=yes,ai-train=no

# --- AI training / bulk-scraping crawlers: disallowed (the ai-train=no stance) ---
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

Sitemap: https://coffeejson.org/sitemap.xml