#
# robots.txt for https://www.darwinproject.ac.uk
#
# This file is only honoured at the root of the host:
# Used:    https://www.darwinproject.ac.uk/robots.txt
# Ignored: https://www.darwinproject.ac.uk/view/robots.txt
#
# For more information about the robots.txt standard, see:
# https://www.robotstxt.org/robotstxt.html

Sitemap: https://www.darwinproject.ac.uk/sitemap.xml

User-agent: PetalBot
Disallow: /

# a crawler obeys only the most specific group naming it and skips *, so each
# named group must repeat every rule it should follow
User-agent: Baiduspider
Disallow: /search

User-agent: GPTBot
Disallow: /search

User-agent: OAI-SearchBot
Disallow: /search

User-agent: ChatGPT-User
Disallow: /search

User-agent: ClaudeBot
Disallow: /search

User-agent: Claude-SearchBot
Disallow: /search

User-agent: Claude-User
Disallow: /search

User-agent: PerplexityBot
Disallow: /search

User-agent: Perplexity-User
Disallow: /search

User-agent: CCBot
Disallow: /search

User-agent: Bytespider
Disallow: /search

User-agent: Amazonbot
Disallow: /search

User-agent: Applebot
Disallow: /search

User-agent: meta-externalagent
Disallow: /search

User-agent: meta-externalfetcher
Disallow: /search

User-agent: FacebookBot
Disallow: /search

User-agent: cohere-ai
Disallow: /search

User-agent: cohere-training-data-crawler
Disallow: /search

User-agent: Diffbot
Disallow: /search

User-agent: AI2Bot
Disallow: /search

User-agent: DuckAssistBot
Disallow: /search

User-agent: MistralAI-User
Disallow: /search

User-agent: PanguBot
Disallow: /search

User-agent: YouBot
Disallow: /search

User-agent: Timpibot
Disallow: /search

User-agent: *
# search renders its results client-side, and every letter page links several
# parameter combinations of it, so crawling it yields empty pages without limit
Disallow: /search