1 files changed, 14 insertions, 0 deletions
diff --git a/robots.txt b/robots.txt
index 8bf1a58..10f4a16 100644
--- a/robots.txt
+++ b/robots.txt
@@ -1,3 +1,17 @@
+# OpenAI’s web crawler: GPT3.5, GPT4, ChatGPT
+User-agent: GPTBot
+
+# ChatGPT plugins
 User-agent: ChatGPT-User
 
+# Google's web crawler: Bard, VertexAI, Gemini
+User-agent: Google-Extended
+
+# Claude
+User-agent: anthropic-ai
+
+# Common Crawl
+User-agent: CCBot
+
+# Censorship area
 Disallow: /