v3 - basic sitemap generation (#580)

* feat: add NEXT_PUBLIC_URL env variable

* feat: basic sitemap generation

* feat(sitemap): generate robots.txt with sitemap reference

* feat(robots): disallow crawling of studio and api paths

* fix(sitemap): fetch Sanity data with token

.env.development

@@ -0,0 +1 @@
+NEXT_PUBLIC_URL=http://localhost:3000

.env.production

@@ -0,0 +1 @@
+NEXT_PUBLIC_URL=https://$NEXT_PUBLIC_VERCEL_URL

src/app/robots.ts

@@ -0,0 +1,11 @@
+import type { MetadataRoute } from "next";
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: {
+      userAgent: "*",
+      disallow: ["/studio", "/shared", "/api"],
+    },
+    sitemap: new URL("sitemap.xml", process.env.NEXT_PUBLIC_URL).toString(),
+  };
+}

src/app/sitemap.ts

@@ -0,0 +1,21 @@
+import type { MetadataRoute } from "next";
+import { client } from "../../studio/lib/client";
+import { Slug } from "../../studio/lib/payloads/global";
+import { token } from "../../studio/lib/token";
+
+interface SitemapDocument {
+  slug: Slug;
+  _updatedAt: string;
+}
+
+const clientWithToken = client.withConfig({ token });
+
+export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
+  const documents =
+    await clientWithToken.fetch<SitemapDocument[]>(`*[defined(slug)]`);
+
+  return documents.map((s) => ({
+    url: new URL(s.slug.current, process.env.NEXT_PUBLIC_URL).toString(),
+    lastModified: new Date(s._updatedAt),
+  }));
+}