# Policy Pal is a public product site. Everything here is meant to be read,
# indexed, and quoted — by search engines and by AI assistants alike. The named
# groups below are not exceptions to the wildcard; they exist because several
# crawlers only honour a group that names them, and because an explicit Allow
# is a clearer statement of intent than silence.
#
# `/cdn-cgi/` is the one exclusion, and it is not Policy Pal's to publish: it is
# Cloudflare's own namespace on this hostname. Its paths answer browsers rather
# than readers, and one of them cannot answer at all — Email Address
# Obfuscation rewrites every `mailto:` into a link to
# `/cdn-cgi/l/email-protection#<encoded>`, where the address lives in the
# fragment that a request never carries, so a crawler following it can only be
# given a 404. Search Console reported exactly that. The rule is repeated in
# every group on purpose: a crawler obeys only the most specific group that
# names it, so one placed under the wildcard alone would bind nobody that the
# groups below name — starting with Googlebot.

User-agent: *
Allow: /
Disallow: /cdn-cgi/

# Search indexing
User-agent: Googlebot
Allow: /
Disallow: /cdn-cgi/
User-agent: Bingbot
Allow: /
Disallow: /cdn-cgi/
User-agent: DuckDuckBot
Allow: /
Disallow: /cdn-cgi/
User-agent: Applebot
Allow: /
Disallow: /cdn-cgi/

# AI training and retrieval
User-agent: GPTBot
Allow: /
Disallow: /cdn-cgi/
User-agent: OAI-SearchBot
Allow: /
Disallow: /cdn-cgi/
User-agent: ChatGPT-User
Allow: /
Disallow: /cdn-cgi/
User-agent: ClaudeBot
Allow: /
Disallow: /cdn-cgi/
User-agent: Claude-User
Allow: /
Disallow: /cdn-cgi/
User-agent: Claude-SearchBot
Allow: /
Disallow: /cdn-cgi/
User-agent: PerplexityBot
Allow: /
Disallow: /cdn-cgi/
User-agent: Perplexity-User
Allow: /
Disallow: /cdn-cgi/
User-agent: Google-Extended
Allow: /
Disallow: /cdn-cgi/
User-agent: Applebot-Extended
Allow: /
Disallow: /cdn-cgi/
User-agent: DuckAssistBot
Allow: /
Disallow: /cdn-cgi/
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /cdn-cgi/
User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /cdn-cgi/
User-agent: MistralAI-User
Allow: /
Disallow: /cdn-cgi/
User-agent: Amazonbot
Allow: /
Disallow: /cdn-cgi/
User-agent: Bytespider
Allow: /
Disallow: /cdn-cgi/
User-agent: cohere-ai
Allow: /
Disallow: /cdn-cgi/
User-agent: CCBot
Allow: /
Disallow: /cdn-cgi/

Sitemap: https://policypal.link/sitemap.xml