Update robots.txt

NeuromatchAcademy · May 12, 2024 · b0b689b · b0b689b
1 parent 93eca10
commit b0b689b
Showing 1 changed file with 14 additions and 2 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -26,7 +26,7 @@ Disallow: /interact/
 User-agent: Amazonbot
 Disallow: /
 
-# AI Data Scraper
+# Undocumented AI Agent
 # https://darkvisitors.com/agents/anthropic-ai
 
 User-agent: anthropic-ai
@@ -56,7 +56,19 @@ Disallow: /
 User-agent: ChatGPT-User
 Disallow: /
 
-# AI Assistant
+# Undocumented AI Agent
+# https://darkvisitors.com/agents/claude-web
+
+User-agent: Claude-Web
+Disallow: /
+
+# AI Data Scraper
+# https://darkvisitors.com/agents/claudebot
+
+User-agent: ClaudeBot
+Disallow: /
+
+# Undocumented AI Agent
 # https://darkvisitors.com/agents/cohere-ai
 
 User-agent: cohere-ai