diff --git a/website/package.json b/website/package.json index ee1f871a..d6cad625 100644 --- a/website/package.json +++ b/website/package.json @@ -13,6 +13,7 @@ "@turf/turf": "^7.1.0", "@types/cors": "^2.8.17", "@types/d3": "^7.4.0", + "@types/jsdom": "^21.1.7", "@types/node": "^20.11.30", "@types/react": "^18.2.67", "@types/topojson": "^3.2.6", @@ -23,6 +24,7 @@ "fp-ts": "^2.16.0", "immer": "^9.0.21", "io-ts": "^2.2.20", + "jsdom": "^25.0.0", "jss": "^10.10.0", "mapshaper": "^0.6.25", "next": "^13.2.4", diff --git a/website/src/domain/municipality-migrations.ts b/website/src/domain/municipality-migrations.ts new file mode 100644 index 00000000..97b24478 --- /dev/null +++ b/website/src/domain/municipality-migrations.ts @@ -0,0 +1,81 @@ +import { groupBy } from "fp-ts/lib/NonEmptyArray"; +import { z } from "zod"; + +const mutationTypes = { + 1: "INCLUSION", + 2: "FUSION", +}; +export const MunicipalityMigrationData = z + .object({ + ds: z.array( + z + .tuple([ + z.number(), + z.number(), + z.string(), + z.number(), + z.string(), + z.number(), + z.string(), + z.string(), + z.string(), + ]) + .transform((val) => ({ + migrationNumber: val[0], + histNumber: val[1], + canton: val[2], + bezirkNumber: val[3], + bezirkName: val[4], + ofsNumber: val[5], + municipalityName: val[6], + radiationReason: val[7], + inscriptionReason: val[8], + })) + ), + mutations: z.record( + z + .object({ + t: z.number(), + d: z.string(), + }) + .transform((val) => ({ + type: val.t, + date: val.d, + })) + ), + }) + .transform((val) => { + const ds = val.ds; + const items = ds.map((row) => { + const mutation = val.mutations[row.migrationNumber]; + return { + ...row, + date: mutation.date, + year: Number(mutation.date.split(".")[2]), + type: mutationTypes[mutation.type as keyof typeof mutationTypes], + }; + }); + const grouped = groupBy<(typeof items)[number]>( + (x) => `${x.migrationNumber}` + )(items); + return Object.entries(grouped).map(([migrationNumber, items]) => { + const added = items.filter((x) => x.inscriptionReason); + const removed = items.filter((x) => x.radiationReason); + return { + added, + removed, + year: items[0].year, + migrationNumber: Number(migrationNumber), + type: items[0].type, + label: `+${added.map((x) => x.municipalityName).join(", ")} / -${removed + .map((x) => x.municipalityName) + .join(", ")}`, + }; + }); + }); + +export type MunicipalityMigrationData = z.infer< + typeof MunicipalityMigrationData +>; + +export type MunicipalityMigrationDataItem = MunicipalityMigrationData[number]; diff --git a/website/src/pages/api/mutations.ts b/website/src/pages/api/mutations.ts new file mode 100644 index 00000000..7b677a43 --- /dev/null +++ b/website/src/pages/api/mutations.ts @@ -0,0 +1,45 @@ +import { JSDOM } from "jsdom"; +import { NextApiHandler } from "next"; +import { MunicipalityMigrationData } from "src/domain/municipality-migrations"; + +async function fetchAndParse(dateFrom: string, dateTo: string) { + const url = `https://www.agvchapp.bfs.admin.ch/fr/mutations/results?EntriesFrom=${dateFrom}&EntriesTo=${dateTo}`; + + const response = await (await fetch(url)).text(); + const html = response; + + const dom = new JSDOM(html); + const document = dom.window.document; + + const scriptsWithDs = document.querySelectorAll("script"); + let dsValue; + + scriptsWithDs.forEach((script) => { + const content = script.textContent; + if (content && content.includes("var ds")) { + const trimmed = content.slice(0, content.indexOf("var getGroupLabel")); + dsValue = eval(`${trimmed}; ({ ds: ds, mutations: mutations })`); + return; + } + }); + + if (dsValue) { + return MunicipalityMigrationData.parse(dsValue); + } else { + throw new Error("Script with var ds not found"); + } +} + +const handler: NextApiHandler = async (req, res) => { + const { from: dateFrom, to: dateTo } = req.query; + if (!dateFrom || !dateTo) { + return res + .status(400) + .send('Please provide "from" and "to" query parameters'); + } + const content = await fetchAndParse(dateFrom as string, dateTo as string); + + return res.send(content); +}; + +export default handler; diff --git a/website/src/pages/mutations.tsx b/website/src/pages/mutations.tsx index 8933a0da..a5c99c1c 100644 --- a/website/src/pages/mutations.tsx +++ b/website/src/pages/mutations.tsx @@ -7,135 +7,25 @@ import { List, ListItem, ListItemText, + TextField, Typography, } from "@material-ui/core"; import dynamic from "next/dynamic"; import { groupBy } from "fp-ts/lib/NonEmptyArray"; import { GeoDataFeature, useGeoData } from "src/domain/geodata"; +import { + MunicipalityMigrationData, + MunicipalityMigrationDataItem, +} from "src/domain/municipality-migrations"; import * as turf from "@turf/turf"; import { FlyToInterpolator } from "@deck.gl/core"; import { parse } from "path"; - -const row = z.object({ - "N° d'hist.": z.string(), - Canton: z.string(), - "N° du district": z.string(), - "Nom du district": z.string(), - "N° OFS commune": z.string().transform((x) => Number(x)), - "Nom de la commune": z.string(), - "Raison de la radiation": z.string(), - "Raison de l'inscription": z.string(), - - "Numéro de mutation": z.string(), - "Type de mutation": z.string(), - "Entrée en vigueur": z.string(), -}); - -interface CsvRow { - "N° d'hist.": string; - Canton: string; - "N° du district": string; - "Nom du district": string; - "N° OFS commune": string; - "Nom de la commune": string; - "Raison de la radiation": string; - "Raison de l'inscription": string; - "Numéro de mutation": string; - "Type de mutation": string; - "Entrée en vigueur": string; -} - -const parseHTML = (htmlContent: string) => { - const parser = new DOMParser(); - const doc = parser.parseFromString( - `${htmlContent}`, - "text/html" - ); - const table = doc.querySelector("table"); - if (!table) { - throw new Error("Could not find table"); - } - const rows = table.querySelectorAll("tr"); - const headers = Array.from(rows[0].querySelectorAll("th")).map( - (header) => header.textContent - ); - const data: CsvRow[] = []; - for (let i = 1; i < rows.length; i++) { - const cells = rows[i].querySelectorAll("td"); - const rowData: CsvRow = {} as CsvRow; - for (let j = 0; j < cells.length; j++) { - const header = headers[j]; - if (!header) { - continue; - } - rowData[header as NonNullable] = cells[j] - .textContent as string; - } - data.push(rowData); - } - - // Filter out special lines and extract mutation information - const filteredData: CsvRow[] = []; - let mutationInfo: { num: string; type: string; date: string } | null = null; - - data.forEach((row) => { - if (row["N° d'hist."].includes("Numéro de mutation :")) { - mutationInfo = { - num: "", - type: "", - date: "", - }; - const rx = - /Numéro de mutation : (.*), Type de mutation : (.*), Entrée en vigueur : (.*)/; - const match = row["N° d'hist."].match(rx); - if (match) { - mutationInfo.num = match[1]; - mutationInfo.type = match[2]; - mutationInfo.date = match[3]; - } - } else { - if (mutationInfo) { - row["Numéro de mutation"] = mutationInfo.num; - row["Type de mutation"] = mutationInfo.type; - row["Entrée en vigueur"] = mutationInfo.date; - } - filteredData.push(row); - } - }); - - const mutations = z.array(row).parse(filteredData); - const groupedMutations = groupBy( - (mutation) => mutation["Numéro de mutation"] - )(mutations); - - return groupedMutations; -}; - -type Row = z.infer; +import { useQuery } from "react-query"; const MutationsMap = dynamic(() => import("../components/Mutations/Map"), { ssr: false, }); -const parseMutationRows = (rows: Row[]) => { - const removed = rows.filter((r) => r["Raison de la radiation"]); - const added = rows.filter((r) => r["Raison de l'inscription"]); - return { - label: `+ ${added - .map((x) => x["Nom de la commune"]) - .join(", ")} / - ${removed - .map((x) => x["Nom de la commune"]) - .join(", ")}`, - added, - removed, - year: Number(rows[0]?.["Entrée en vigueur"].split(".")[2]), - "Entrée en vigueur": rows[0]?.["Entrée en vigueur"], - "Numéro de mutation": rows[0]?.["Numéro de mutation"], - }; -}; - -type Parsed = ReturnType; - const INITIAL_VIEW_STATE = { latitude: 46.8182, longitude: 8.2275, @@ -149,22 +39,27 @@ const INITIAL_VIEW_STATE = { }; export default function Page() { - const [groupedMutations, setGroupedMutations] = useState< - ReturnType[] - >([]); + const [year1, setYear1] = useState("2022"); + const [year2, setYear2] = useState("2024"); + const { data: groupedMutations } = useQuery({ + queryKey: ["mutations", year1, year2], + queryFn: async () => { + const mutations = (await ( + await fetch(`/api/mutations?from=01.01.${year1}&to=01.01.${year2}`) + ).json()) as MunicipalityMigrationData; + console.log({ mutations }); + return mutations; + }, + }); const [highlightedMunicipalities, setHighlightedMunicipalities] = - useState(); + useState(); - const handleMutationSelect = (parsed: Parsed) => { + const handleMutationSelect = (parsed: MunicipalityMigrationDataItem) => { setHighlightedMunicipalities(parsed); - setYear1(`${parsed.year - 1}`); - setYear2(`${parsed.year}`); }; const [viewState, setViewState] = useState(INITIAL_VIEW_STATE); - const [year1, setYear1] = useState("2022"); - const [year2, setYear2] = useState("2024"); const { data: geoData1 } = useGeoData({ year: year1, format: "topojson", @@ -188,7 +83,7 @@ export default function Page() { }); useEffect(() => { const { added = [], removed = [] } = highlightedMunicipalities ?? {}; - const all = [...added, ...removed].map((x) => x["N° OFS commune"]); + const all = [...added, ...removed].map((x) => x.ofsNumber); const findFeature = (x: GeoDataFeature) => all.includes(x.properties?.id); const municipality = geoData1.municipalities?.features.find(findFeature) ?? @@ -213,17 +108,6 @@ export default function Page() { } }, [highlightedMunicipalities, geoData1, geoData2]); - const handleChangeContent = (html: string) => { - try { - const data = Object.values(parseHTML(html)); - const groupedMutations = data.map(parseMutationRows); - setGroupedMutations(groupedMutations); - setHighlightedMunicipalities(groupedMutations[0]); - } catch (e) { - alert(e instanceof Error ? e.message : `${e}`); - } - }; - return ( - - Copy/paste here the HTML content of the mutation table, see for{" "} - - here - - . Choose display "100" elements to be able to copy all mutations. . - -