# What may be done with the content of this site.
#
# Short version: search it, quote it, cite it. Do not train on the
# individual reviews. Aggregate figures — scores, counts, the
# statistics published on /methodology/ — carry no such reservation
# and may be used freely with attribution.
#
# Full position: https://studentreviews.com/ai/
# Machine-readable reservation: https://studentreviews.com/.well-known/tdmrep.json

# Content Signals (https://contentsignals.org). These express what may
# be done with content after it is fetched, which robots.txt alone
# cannot say. They are preferences, not access control.
#   search    build a search index, show a link and a snippet
#   ai-input  use as input to a model answering a question, with a link
#   ai-train  use to train or fine-tune a model

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Search, retrieval and answering. Explicitly welcome: a review nobody
# can find helps nobody.
User-agent: Googlebot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Bingbot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Slurp
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: DuckDuckBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Applebot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: OAI-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: ChatGPT-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Claude-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Claude-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: PerplexityBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Perplexity-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: YandexBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

User-agent: Baiduspider
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Training collection. Refused for the review corpus.
#
# Google-Extended and Applebot-Extended are opt-out tokens rather than
# crawlers: these lines withhold the content from Gemini and Apple
# Intelligence training without touching Google Search or Siri.
User-agent: GPTBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: ClaudeBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: anthropic-ai
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: Google-Extended
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: Applebot-Extended
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: CCBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: meta-externalagent
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: FacebookBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: Bytespider
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: Amazonbot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: cohere-ai
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: cohere-training-data-crawler
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: Diffbot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: AI2Bot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: omgili
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: omgilibot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: ImagesiftBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: PanguBot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: Timpibot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: Webzio-Extended
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: Kangaroo Bot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: Scrapy
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: Novellum AI Crawl
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

User-agent: Terracotta Bot
Content-Signal: search=no, ai-input=no, ai-train=no
Disallow: /

Sitemap: https://studentreviews.com/sitemap.xml