feat: Download the data directly from the migration site

website/package.json

@@ -13,6 +13,7 @@
     "@turf/turf": "^7.1.0",
     "@types/cors": "^2.8.17",
     "@types/d3": "^7.4.0",
+    "@types/jsdom": "^21.1.7",
     "@types/node": "^20.11.30",
     "@types/react": "^18.2.67",
     "@types/topojson": "^3.2.6",
@@ -23,6 +24,7 @@
     "fp-ts": "^2.16.0",
     "immer": "^9.0.21",
     "io-ts": "^2.2.20",
+    "jsdom": "^25.0.0",
     "jss": "^10.10.0",
     "mapshaper": "^0.6.25",
     "next": "^13.2.4",

website/src/domain/municipality-migrations.ts

@@ -0,0 +1,81 @@
+import { groupBy } from "fp-ts/lib/NonEmptyArray";
+import { z } from "zod";
+
+const mutationTypes = {
+  1: "INCLUSION",
+  2: "FUSION",
+};
+export const MunicipalityMigrationData = z
+  .object({
+    ds: z.array(
+      z
+        .tuple([
+          z.number(),
+          z.number(),
+          z.string(),
+          z.number(),
+          z.string(),
+          z.number(),
+          z.string(),
+          z.string(),
+          z.string(),
+        ])
+        .transform((val) => ({
+          migrationNumber: val[0],
+          histNumber: val[1],
+          canton: val[2],
+          bezirkNumber: val[3],
+          bezirkName: val[4],
+          ofsNumber: val[5],
+          municipalityName: val[6],
+          radiationReason: val[7],
+          inscriptionReason: val[8],
+        }))
+    ),
+    mutations: z.record(
+      z
+        .object({
+          t: z.number(),
+          d: z.string(),
+        })
+        .transform((val) => ({
+          type: val.t,
+          date: val.d,
+        }))
+    ),
+  })
+  .transform((val) => {
+    const ds = val.ds;
+    const items = ds.map((row) => {
+      const mutation = val.mutations[row.migrationNumber];
+      return {
+        ...row,
+        date: mutation.date,
+        year: Number(mutation.date.split(".")[2]),
+        type: mutationTypes[mutation.type as keyof typeof mutationTypes],
+      };
+    });
+    const grouped = groupBy<(typeof items)[number]>(
+      (x) => `${x.migrationNumber}`
+    )(items);
+    return Object.entries(grouped).map(([migrationNumber, items]) => {
+      const added = items.filter((x) => x.inscriptionReason);
+      const removed = items.filter((x) => x.radiationReason);
+      return {
+        added,
+        removed,
+        year: items[0].year,
+        migrationNumber: Number(migrationNumber),
+        type: items[0].type,
+        label: `+${added.map((x) => x.municipalityName).join(", ")} / -${removed
+          .map((x) => x.municipalityName)
+          .join(", ")}`,
+      };
+    });
+  });
+
+export type MunicipalityMigrationData = z.infer<
+  typeof MunicipalityMigrationData
+>;
+
+export type MunicipalityMigrationDataItem = MunicipalityMigrationData[number];

website/src/pages/api/mutations.ts

@@ -0,0 +1,45 @@
+import { JSDOM } from "jsdom";
+import { NextApiHandler } from "next";
+import { MunicipalityMigrationData } from "src/domain/municipality-migrations";
+
+async function fetchAndParse(dateFrom: string, dateTo: string) {
+  const url = `https://www.agvchapp.bfs.admin.ch/fr/mutations/results?EntriesFrom=${dateFrom}&EntriesTo=${dateTo}`;
+
+  const response = await (await fetch(url)).text();
+  const html = response;
+
+  const dom = new JSDOM(html);
+  const document = dom.window.document;
+
+  const scriptsWithDs = document.querySelectorAll("script");
+  let dsValue;
+
+  scriptsWithDs.forEach((script) => {
+    const content = script.textContent;
+    if (content && content.includes("var ds")) {
+      const trimmed = content.slice(0, content.indexOf("var getGroupLabel"));
+      dsValue = eval(`${trimmed}; ({ ds: ds, mutations: mutations })`);
+      return;
+    }
+  });
+
+  if (dsValue) {
+    return MunicipalityMigrationData.parse(dsValue);
+  } else {
+    throw new Error("Script with var ds not found");
+  }
+}
+
+const handler: NextApiHandler = async (req, res) => {
+  const { from: dateFrom, to: dateTo } = req.query;
+  if (!dateFrom || !dateTo) {
+    return res
+      .status(400)
+      .send('Please provide "from" and "to" query parameters');
+  }
+  const content = await fetchAndParse(dateFrom as string, dateTo as string);
+
+  return res.send(content);
+};
+
+export default handler;