# robots.txt — https://neme-learning.pages.dev/
#
# Crawler policy (two separate decisions, recorded separately):
#   1. search_crawl: ALLOWED. Search and answer-engine crawlers may crawl this
#      site; the pages are public and intended to be indexable.
#   2. model_training: NOT OPTED IN. Known model-training crawlers are asked to
#      stay out (disallowed below).
#
# Honest limitation, stated plainly: on pages.dev, robots.txt is a voluntary
# convention. Cloudflare does not enforce these rules for the listed user
# agents at the edge — the exclusion is not enforced; compliance is up to each
# crawler operator. This file is a published request, not a technical guarantee.

# General policy: everything is allowed for search and answer retrieval.
User-agent: *
Disallow:

# Known model-training crawlers (distinct from search/answer crawlers):
# GPTBot          — OpenAI model training
User-agent: GPTBot
Disallow: /

# CCBot           — Common Crawl (training datasets)
User-agent: CCBot
Disallow: /

# ClaudeBot       — Anthropic model training
User-agent: ClaudeBot
Disallow: /

# anthropic-ai    — Anthropic model training (legacy agent token)
User-agent: anthropic-ai
Disallow: /

# Claude-Web      — Anthropic research crawling
User-agent: Claude-Web
Disallow: /

# Google-Extended — Gemini model training (does not affect Google Search)
User-agent: Google-Extended
Disallow: /

# Bytespider      — ByteDance model training
User-agent: Bytespider
Disallow: /

# Applebot-Extended — Apple AI model training
User-agent: Applebot-Extended
Disallow: /

# cohere-ai       — Cohere model training
User-agent: cohere-ai
Disallow: /

Sitemap: https://neme-learning.pages.dev/sitemap.xml