# hatch-capital.com robots.txt
#
# Every crawler may read the site's pages. None may read the API or a prospect's own pages: a Top 5 report (/r/...)
# or a report in progress (/top-5/<id>). The public Top 5 page, /top-5, stays open.
#
# The AI crawlers are named one by one, so each knows it is welcome. A crawler obeys only the most specific group that
# names it, so every named group repeats the Disallow lines of the * group (src/test/robots-txt.test.ts checks it).

User-agent: *
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

# Meta AI: search and citations, training, fetches a person asks for
User-agent: meta-webindexer
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

User-agent: meta-externalagent
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

User-agent: meta-externalfetcher
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

# OpenAI: training, ChatGPT search, fetches a person asks for
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

# Anthropic: training, Claude search, fetches a person asks for
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

# Perplexity: search, fetches a person asks for
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

# Google (Gemini) and Apple (Apple Intelligence): the tokens that cover use in their AI models
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

# Common Crawl, Amazon, DuckDuckGo's DuckAssist, Mistral's Le Chat
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

User-agent: DuckAssistBot
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

User-agent: MistralAI-User
Allow: /
Disallow: /api/
Disallow: /r/
Disallow: /top-5/

Sitemap: https://hatch-capital.com/sitemap.xml