diff --git a/packages/core/src/datasets.ts b/packages/core/src/datasets.ts index 5f98a5b..41daf31 100644 --- a/packages/core/src/datasets.ts +++ b/packages/core/src/datasets.ts @@ -6,24 +6,36 @@ import * as d3 from "d3"; import db from "../../config/db"; export const updater = async () => { - const tempDir = "/tmp"; + const extractToPath = "/tmp"; const zipFileName = `Monthly New Registration of Cars by Make.zip`; - const zipFilePath = `${tempDir}/${zipFileName}`; + const zipFilePath = `${extractToPath}/${zipFileName}`; const csvFileName = `M03-Car_Regn_by_make.csv`; - const csvFilePath = `${tempDir}/${csvFileName}`; + const csvFilePath = `${extractToPath}/${csvFileName}`; const zipUrl = `https://datamall.lta.gov.sg/content/dam/datamall/datasets/Facts_Figures/Vehicle Registration/${zipFileName}`; const response = await fetch(zipUrl); if (!response.ok) { throw new Error(`Failed to download the ZIP file: ${response.statusText}`); } - const data = await response.buffer(); - fs.writeFileSync(zipFilePath, data); const zip = new AdmZip(zipFilePath); - zip.extractAllTo(`${tempDir}`, true); + zip.extractAllTo(`${extractToPath}`, true); + const zipEntries = zip.getEntries(); - const csvData = fs.readFileSync(csvFilePath, "utf-8"); + let destinationPath = extractToPath; + zipEntries.forEach((entry) => { + if (!entry.isDirectory) { + const entryName = entry.entryName; + + destinationPath = `${extractToPath}/${entryName}`; + + const content = entry.getData(); + fs.writeFileSync(destinationPath, content); + zip.deleteFile(entryName); + } + }); + + const csvData = fs.readFileSync(destinationPath, "utf-8"); const parsedData = d3.csvParse(csvData); const existingData = await db.collection("cars").find().toArray(); diff --git a/packages/functions/src/datasets.ts b/packages/functions/src/datasets.ts index 2823472..18bc7b2 100644 --- a/packages/functions/src/datasets.ts +++ b/packages/functions/src/datasets.ts @@ -6,6 +6,6 @@ export const updater = ApiHandler(async (event, context) => { return { statusCode: 200, - body: JSON.stringify("Data has been successfully updated"), + body: JSON.stringify({ message: "Data has been successfully updated" }), }; });