From af2c48520e76497db88ae37c2a3c5b3360ff9313 Mon Sep 17 00:00:00 2001 From: hobgoblina <20880695+hobgoblina@users.noreply.github.com> Date: Sun, 24 Nov 2024 00:19:58 +0000 Subject: [PATCH] Update robots.txt --- public/robots.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/public/robots.txt b/public/robots.txt index f7de331f64284f..dc22e36ba18e48 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -146,6 +146,12 @@ Disallow: / User-agent: omgili Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/pangubot + +User-agent: PanguBot +Disallow: / + # AI Search Crawler # https://darkvisitors.com/agents/perplexitybot