specified search engines in robots.txt

SJSUCSClub · Oct 21, 2024 · 95d6247 · 95d6247
1 parent b0779c8
commit 95d6247
Showing 1 changed file with 97 additions and 5 deletions.
diff --git a/app/robots.ts b/app/robots.ts
@@ -12,11 +12,103 @@ export default async function robots(): Promise<MetadataRoute.Robots> {
   const bridgeSitemaps = coursesSitemaps.concat(professorsSitemaps);
 
   return {
-    rules: {
-      userAgent: '*',
-      allow: '/',
-      disallow: '/profile',
-    },
+    rules: [
+      {
+        userAgent: 'baiduspider',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'baiduspider-image',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'baiduspider-mobile',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'baiduspider-news',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'baiduspider-video',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'bingbot',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'msnbot',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'msnbot-media',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'adidxbot',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'Googlebot',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'Googlebot-Image',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'Googlebot-Mobile',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'Googlebot-News',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'Googlebot-Video',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'Storebot-Google',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'Mediapartners-Google',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'AdsBot-Google',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'slurp',
+        allow: '/',
+        disallow: '/profile',
+      },
+      {
+        userAgent: 'yandex',
+        allow: '/',
+        disallow: '/profile',
+      },
+    ],
     sitemap: [`${process.env.NEXT_PUBLIC_BASE_URL}/sitemap.xml`].concat(
       bridgeSitemaps,
     ),