diff --git a/public/robots.txt b/public/robots.txt index a955b370f20618..f8bbfa4c9f707d 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -38,6 +38,12 @@ Disallow: / User-agent: Applebot Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/applebot-extended + +User-agent: Applebot-Extended +Disallow: / + # AI Data Scraper # https://darkvisitors.com/agents/bytespider