diff --git a/public/robots.txt b/public/robots.txt index f8bbfa4c9f707d..39c9657a8b674a 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -104,6 +104,18 @@ Disallow: / User-agent: GPTBot Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/meta-externalagent + +User-agent: Meta-ExternalAgent +Disallow: / + +# AI Assistant +# https://darkvisitors.com/agents/meta-externalfetcher + +User-agent: Meta-ExternalFetcher +Disallow: / + # AI Data Scraper # https://darkvisitors.com/agents/omgili @@ -116,6 +128,12 @@ Disallow: / User-agent: PerplexityBot Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/timpibot + +User-agent: Timpibot +Disallow: / + # AI Search Crawler # https://darkvisitors.com/agents/youbot