From a954dfb0a52c4ea58e3f263d9beafab385a5b1a7 Mon Sep 17 00:00:00 2001
From: Marc Coquand <marc@mccd.space>
Date: Wed, 14 Feb 2024 07:12:57 -0600
Subject: Update robots.txt

---
 robots.txt | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

(limited to 'robots.txt')

diff --git a/robots.txt b/robots.txt
index 8bf1a58..10f4a16 100644
--- a/robots.txt
+++ b/robots.txt
@@ -1,3 +1,17 @@
+# OpenAI’s web crawler: GPT3.5, GPT4, ChatGPT
+User-agent: GPTBot
+
+# ChatGPT plugins
 User-agent: ChatGPT-User
 
+# Google's web crawler: Bard, VertexAI, Gemini
+User-agent: Google-Extended
+
+# Claude
+User-agent: anthropic-ai
+
+# Common Crawl
+User-agent: CCBot
+
+# Censorship area
 Disallow: /
-- 
cgit v1.2.3