From fbaa83c951e3696779fd971de767870053cf51dc Mon Sep 17 00:00:00 2001 From: Code by Ben Date: Tue, 10 Sep 2024 19:05:20 +1000 Subject: [PATCH] accidental robots.txt blockage. --- public/text/robots.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/text/robots.txt b/public/text/robots.txt index 2515a7a0..1dfe5401 100644 --- a/public/text/robots.txt +++ b/public/text/robots.txt @@ -1,4 +1,3 @@ -User-agent: * # How to manually block common AI crawlers # https://help.raptive.com/hc/en-us/articles/25756415800987-How-to-manually-block-common-AI-crawlers User-agent: anthropic-ai @@ -15,6 +14,7 @@ User-agent: GPTBot Disallow: / User-agent: PiplBot Disallow: / +User-agent: * Allow: / Disallow: /d/ Disallow: /v/