From 4e6d0c5157d9d49f8b742ebcef9ec8d364742212 Mon Sep 17 00:00:00 2001 From: luisdralves Date: Tue, 12 Dec 2023 16:01:29 +0000 Subject: [PATCH] Add sitemap generator --- .gitignore | 1 + workspaces/cms-scripts/package.json | 3 +- workspaces/cms-scripts/src/sitemap.ts | 208 ++++++++++++++++++++++ workspaces/website/functions/[[route]].ts | 1 + 4 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 workspaces/cms-scripts/src/sitemap.ts diff --git a/.gitignore b/.gitignore index fd1f2a4c7c..6de30f375d 100644 --- a/.gitignore +++ b/.gitignore @@ -64,5 +64,6 @@ _crowdin *.__* dist public/data +public/sitemap.xml !_crowdin/translation .wrangler/ diff --git a/workspaces/cms-scripts/package.json b/workspaces/cms-scripts/package.json index 04fd441d9c..85715fc911 100644 --- a/workspaces/cms-scripts/package.json +++ b/workspaces/cms-scripts/package.json @@ -4,9 +4,10 @@ "private": true, "scripts": { "update-algolia-index": "vite-node src/algolia.ts", - "update-dynamic-data": "vite-node src/index.ts", + "update-dynamic-data": "vite-node src/index.ts && yarn sitemap", "update-jobs": "vite-node src/jobs-update.ts", "pre-crowdin": "vite-node src/pre-crowdin.ts", + "sitemap": "vite-node src/sitemap.ts", "build": "tsc" }, "dependencies": { diff --git a/workspaces/cms-scripts/src/sitemap.ts b/workspaces/cms-scripts/src/sitemap.ts new file mode 100644 index 0000000000..6c8458a006 --- /dev/null +++ b/workspaces/cms-scripts/src/sitemap.ts @@ -0,0 +1,208 @@ +/** + * Module dependencies. + */ + +import fs from 'fs/promises'; +import path from 'path'; + +/** + * Change directory to project root. + */ + +process.chdir(path.resolve(__dirname, '../../..')); + +import { locales } from '@starknet-io/cms-data/src/i18n/config'; +import { getPosts } from './data'; + +/** + * Config constants. + */ + +const domain = 'https://www.starknet.io'; +const changefreqByDepth = ['daily', 'weekly', 'monthly', 'yearly']; +const priorityByDepth = [1, 0.8, 0.6, 0.4]; +const customPages = ['announcements', 'events', 'jobs', 'posts', 'roadmap', 'tutorials']; + +/** + * `SitemapUrl` type. + */ + +type SitemapUrl = { + changefreq: string; + lastmod?: string; + priority: number; + url: string; +}; + +/** + * Initialize `sitemapUrls` constant. + */ + +const sitemapUrls: SitemapUrl[] = locales.flatMap(locale => [ + { + changefreq: 'weekly', + priority: 1, + url: `/${locale}` + }, + ...customPages.map(url => ({ + changefreq: 'weekly', + priority: 0.8, + url: `/${locale}/${url}` + })) +]); + +/** + * `parsePageDirectory` function. + */ + +const parsePageDirectory = async (dir: string, depth = 0) => { + const files = await fs.readdir(dir); + + await Promise.all( + files.map(async filepath => { + const file = path.join(dir, filepath); + + if ((await fs.stat(file))?.isDirectory()) { + await parsePageDirectory(file, depth + 1); + + return; + } + + const page = await fs.readFile(file, 'utf8'); + + try { + const { hidden_page, link } = JSON.parse(page); + + if (!hidden_page && link) { + sitemapUrls.push({ + url: link, + changefreq: changefreqByDepth[depth], + priority: priorityByDepth[depth] + }); + } + } catch { + console.error('Error parsing page', file); + } + }) + ); +}; + +/** + * `parsePosts` function. + */ + +const parsePosts = async () => { + const { filenameMap } = await getPosts(); + const categories: string[] = []; + + filenameMap.forEach(({ locale, category, slug, published_date }) => { + if (!categories.includes(category)) { + categories.push(category); + + sitemapUrls.push({ + url: `/${locale}/posts/${category}`, + changefreq: 'weekly', + priority: 0.8 + }); + } + + sitemapUrls.push({ + url: `/${locale}/posts/${category}/${slug}`, + changefreq: 'monthly', + priority: 0.6, + lastmod: published_date?.split('T')?.[0] + }); + }); +}; + +/** + * `parseTutorialsFile` function. + */ + +const parseTutorialsFile = async (locale: string, filename: string) => { + try { + const file = await fs.readFile(`./public/data/tutorials/${locale}/${filename}`, 'utf8'); + const tutorial = JSON.parse(file); + + if (tutorial.type === 'youtube') { + sitemapUrls.push({ + url: `/${locale}/tutorials/video/${tutorial.id}`, + changefreq: 'monthly', + priority: 0.6 + }); + } + } catch { + console.error(`Error parsing tutorial`, filename); + } +}; + +/** + * `parseTutorials` function. + */ + +const parseTutorials = async () => { + for (const locale of locales) { + const files = await fs.readdir(`./public/data/tutorials/${locale}`); + + await Promise.all( + files.map(async filename => { + await parseTutorialsFile(locale, filename); + }) + ); + } +}; + +/** + * `parseDetails` function. + */ + +const parseDetails = async (name: string) => { + for (const locale of locales) { + const file = await fs.readFile(`./public/data/${name}-details/${locale}.json`, 'utf8'); + + try { + const roadmaps = JSON.parse(file); + sitemapUrls.push( + ...roadmaps.map((roadmap: { slug: string }) => ({ + url: `/${locale}/${name}/${roadmap.slug}`, + changefreq: 'monthly', + priority: 0.6 + })) + ); + } catch { + console.error(`Error parsing ${name} for locale`, locale); + } + } +}; + +/** + * Parse all urls. + */ + +await parsePageDirectory('./public/data/pages'); +await parsePosts(); +await parseDetails('announcements'); +await parseDetails('roadmap'); +await parseTutorials(); + +let sitemap = ` +`; + +sitemapUrls.forEach(({ url, changefreq, priority, lastmod }) => { + sitemap += ` + + ${domain}${url} + ${changefreq} + ${priority}${lastmod ? ` + ${lastmod}` : ''} + `; +}); + +sitemap += ` +`; + +/** + * Write sitemap file + */ + +await fs.writeFile('./public/sitemap.xml', sitemap); diff --git a/workspaces/website/functions/[[route]].ts b/workspaces/website/functions/[[route]].ts index 076b9998e6..6f65f56109 100644 --- a/workspaces/website/functions/[[route]].ts +++ b/workspaces/website/functions/[[route]].ts @@ -34,6 +34,7 @@ router.get("/*.svg", ittyAssetshandler); router.get("/*.ico", ittyAssetshandler); router.get("/*.txt", ittyAssetshandler); router.get("/assets/*", ittyAssetshandler); +router.get("/sitemap.xml", ittyAssetshandler); router.all("/data/*", preflight); router.get("/data/*", async (req, context: EventContext<{}, any, Record>) => {