blob: 10f4a16958349f83632d3b37f00ff861236f8833 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
|
# OpenAI’s web crawler: GPT3.5, GPT4, ChatGPT
User-agent: GPTBot
# ChatGPT plugins
User-agent: ChatGPT-User
# Google's web crawler: Bard, VertexAI, Gemini
User-agent: Google-Extended
# Claude
User-agent: anthropic-ai
# Common Crawl
User-agent: CCBot
# Censorship area
Disallow: /
|