aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMarc Coquand <marc@mccd.space>2024-02-14 07:12:57 -0600
committerMarc Coquand <marc@mccd.space>2024-02-14 07:12:57 -0600
commita954dfb0a52c4ea58e3f263d9beafab385a5b1a7 (patch)
tree9c06054f38d7b05b690b069836ec4fa79f50c554
parentebbdacce30ee511e1133aeed5842941a35959c44 (diff)
downloadmccd.space-a954dfb0a52c4ea58e3f263d9beafab385a5b1a7.tar.gz
mccd.space-a954dfb0a52c4ea58e3f263d9beafab385a5b1a7.tar.bz2
mccd.space-a954dfb0a52c4ea58e3f263d9beafab385a5b1a7.zip
Update robots.txt
Diffstat (limited to '')
-rw-r--r--robots.txt14
1 files changed, 14 insertions, 0 deletions
diff --git a/robots.txt b/robots.txt
index 8bf1a58..10f4a16 100644
--- a/robots.txt
+++ b/robots.txt
@@ -1,3 +1,17 @@
+# OpenAI’s web crawler: GPT3.5, GPT4, ChatGPT
+User-agent: GPTBot
+
+# ChatGPT plugins
User-agent: ChatGPT-User
+# Google's web crawler: Bard, VertexAI, Gemini
+User-agent: Google-Extended
+
+# Claude
+User-agent: anthropic-ai
+
+# Common Crawl
+User-agent: CCBot
+
+# Censorship area
Disallow: /