Merge pull request #6 from ruchernchong/5-fix-files-extracted-path

Fix reading of file from path
sgcarstrends · Dec 12, 2023 · d0940bc · d0940bc
2 parents 02bef4a + 60f557c
commit d0940bc
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 8 deletions.
diff --git a/packages/core/src/datasets.ts b/packages/core/src/datasets.ts
@@ -6,24 +6,36 @@ import * as d3 from "d3";
 import db from "../../config/db";
 
 export const updater = async () => {
-  const tempDir = "/tmp";
+  const extractToPath = "/tmp";
   const zipFileName = `Monthly New Registration of Cars by Make.zip`;
-  const zipFilePath = `${tempDir}/${zipFileName}`;
+  const zipFilePath = `${extractToPath}/${zipFileName}`;
   const csvFileName = `M03-Car_Regn_by_make.csv`;
-  const csvFilePath = `${tempDir}/${csvFileName}`;
+  const csvFilePath = `${extractToPath}/${csvFileName}`;
   const zipUrl = `https://datamall.lta.gov.sg/content/dam/datamall/datasets/Facts_Figures/Vehicle Registration/${zipFileName}`;
 
   const response = await fetch(zipUrl);
   if (!response.ok) {
     throw new Error(`Failed to download the ZIP file: ${response.statusText}`);
   }
-  const data = await response.buffer();
-  fs.writeFileSync(zipFilePath, data);
 
   const zip = new AdmZip(zipFilePath);
-  zip.extractAllTo(`${tempDir}`, true);
+  zip.extractAllTo(`${extractToPath}`, true);
+  const zipEntries = zip.getEntries();
 
-  const csvData = fs.readFileSync(csvFilePath, "utf-8");
+  let destinationPath = extractToPath;
+  zipEntries.forEach((entry) => {
+    if (!entry.isDirectory) {
+      const entryName = entry.entryName;
+
+      destinationPath = `${extractToPath}/${entryName}`;
+
+      const content = entry.getData();
+      fs.writeFileSync(destinationPath, content);
+      zip.deleteFile(entryName);
+    }
+  });
+
+  const csvData = fs.readFileSync(destinationPath, "utf-8");
   const parsedData = d3.csvParse(csvData);
 
   const existingData = await db.collection("cars").find().toArray();

diff --git a/packages/functions/src/datasets.ts b/packages/functions/src/datasets.ts
@@ -6,6 +6,6 @@ export const updater = ApiHandler(async (event, context) => {
 
   return {
     statusCode: 200,
-    body: JSON.stringify("Data has been successfully updated"),
+    body: JSON.stringify({ message: "Data has been successfully updated" }),
   };
 });