accidental robots.txt blockage.

Defacto2 · Sep 10, 2024 · fbaa83c · fbaa83c
1 parent 55554c0
commit fbaa83c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/public/text/robots.txt b/public/text/robots.txt
@@ -1,4 +1,3 @@
-User-agent: *
 # How to manually block common AI crawlers
 # https://help.raptive.com/hc/en-us/articles/25756415800987-How-to-manually-block-common-AI-crawlers
 User-agent: anthropic-ai
@@ -15,6 +14,7 @@ User-agent: GPTBot
 Disallow: /
 User-agent: PiplBot
 Disallow: /
+User-agent: *
 Allow: /
 Disallow: /d/
 Disallow: /v/