From d8b2e6cae75f708089820d205abb468b4e597232 Mon Sep 17 00:00:00 2001 From: Ru Chern Chong Date: Sat, 4 May 2024 04:30:00 +0800 Subject: [PATCH] Fix errors --- src/config/db.ts | 40 +++---- src/index.ts | 286 ++++++++++++++++++++++++++--------------------- sst-env.d.ts | 7 ++ sst.config.ts | 4 +- 4 files changed, 188 insertions(+), 149 deletions(-) create mode 100644 sst-env.d.ts diff --git a/src/config/db.ts b/src/config/db.ts index d4e2200..1fda6b1 100644 --- a/src/config/db.ts +++ b/src/config/db.ts @@ -1,20 +1,20 @@ -import * as mongodb from "mongodb"; - -const MongoClient = mongodb.MongoClient; - -let cachedDb: mongodb.Db | null = null; - -const connectToDatabase = async (): Promise => { - if (cachedDb) { - return cachedDb; - } - - const client = await MongoClient.connect(process.env.MONGODB_URI); - cachedDb = await client.db("main"); - - return cachedDb; -}; - -const db = await connectToDatabase(); - -export default db; +// import * as mongodb from "mongodb"; +// +// const MongoClient = mongodb.MongoClient; +// +// let cachedDb: mongodb.Db | null = null; +// +// const connectToDatabase = async (): Promise => { +// if (cachedDb) { +// return cachedDb; +// } +// +// const client = await MongoClient.connect(process.env.MONGODB_URI); +// cachedDb = await client.db("main"); +// +// return cachedDb; +// }; +// +// const db = await connectToDatabase(); +// +// export default db; diff --git a/src/index.ts b/src/index.ts index 4bcbf84..e627831 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,8 +2,8 @@ import { Hono } from "hono"; import { Document, WithId } from "mongodb"; import { format, subMonths } from "date-fns"; import * as d3 from "d3"; -import fs from "fs/promises"; -import db from "./config/db"; +// import fs from "fs/promises"; +// import db from "./config/db"; import { downloadFile } from "./utils/downloadFile"; import { extractZipFile } from "./utils/extractZipFile"; import { Car, COEResult, FUEL_TYPE, UpdateParams } from "./types"; @@ -21,10 +21,40 @@ const getCarsByFuelType = async ( month: month ?? { $gte: trailingTwelveMonths }, }; - const cars: WithId[] = await db - .collection("cars") - .find(filter) - .toArray(); + // const cars: WithId[] = await db + // .collection("cars") + // .find(filter) + // .toArray(); + + const cars = [ + { + _id: { $oid: "65f3fb10fa9ed39b7d6e73d5" }, + month: "2024-02", + make: "AUDI", + importer_type: "AMD", + fuel_type: "Electric", + vehicle_type: "Hatchback", + number: "1", + }, + { + _id: { $oid: "65f3fb10fa9ed39b7d6e73d9" }, + month: "2024-02", + make: "B.M.W.", + importer_type: "AMD", + fuel_type: "Electric", + vehicle_type: "Hatchback", + number: "", + }, + { + _id: { $oid: "65f3fb10fa9ed39b7d6e73dc" }, + month: "2024-02", + make: "BYD", + importer_type: "AMD", + fuel_type: "Electric", + vehicle_type: "Hatchback", + number: "6", + }, + ]; return cars.reduce( (result: WithId[], { _id, month, make, fuel_type, number }) => { @@ -50,75 +80,75 @@ const getCarsByFuelType = async ( ); }; -const getLatestMonth = async (): Promise => { - const months = await db.collection("coe").distinct("month"); - return months[months.length - 1]; -}; - -export const getCOEResultByMonth = async ( - month?: string, -): Promise[]> => { - const selectedMonth = month || (await getLatestMonth()); - return db - .collection("coe") - .find({ month: selectedMonth }) - .sort({ bidding_no: 1, vehicle_class: 1 }) - .toArray(); -}; - -const EXTRACT_PATH: string = "/tmp"; - -export const update = async ({ - collectionName, - zipFileName, - zipUrl, - keyFields, -}: UpdateParams): Promise<{ message: string }> => { - const collection = db.collection(collectionName); - - try { - const zipFilePath = `${EXTRACT_PATH}/${zipFileName}`; - await downloadFile({ url: zipUrl, destination: zipFilePath }); - - const extractedFileName = await extractZipFile(zipFilePath, EXTRACT_PATH); - const destinationPath = `${EXTRACT_PATH}/${extractedFileName}`; - console.log(`Destination path:`, destinationPath); - - const csvData = await fs.readFile(destinationPath, "utf-8"); - const parsedData = d3.csvParse(csvData); - - const existingData: WithId[] = await collection.find().toArray(); - - const createUniqueKey = ( - item: T, - keyFields: Array, - ): string => - keyFields - .filter((field) => item[field]) - .map((field) => item[field]) - .join("-"); - - const existingDataMap: Map> = new Map( - existingData.map((item) => [createUniqueKey(item, keyFields), item]), - ); - const newDataToInsert = parsedData.filter( - (newItem) => !existingDataMap.has(createUniqueKey(newItem, keyFields)), - ); - - let message: string; - if (newDataToInsert.length > 0) { - const result = await collection.insertMany(newDataToInsert); - message = `${result.insertedCount} document(s) inserted`; - } else { - message = `No new data to insert. The provided data matches the existing records.`; - } - - return { message }; - } catch (error) { - console.error(`An error has occurred:`, error); - throw error; - } -}; +// const getLatestMonth = async (): Promise => { +// const months = await db.collection("coe").distinct("month"); +// return months[months.length - 1]; +// }; + +// export const getCOEResultByMonth = async ( +// month?: string, +// ): Promise[]> => { +// const selectedMonth = month || (await getLatestMonth()); +// return db +// .collection("coe") +// .find({ month: selectedMonth }) +// .sort({ bidding_no: 1, vehicle_class: 1 }) +// .toArray(); +// }; + +// const EXTRACT_PATH: string = "/tmp"; + +// export const update = async ({ +// collectionName, +// zipFileName, +// zipUrl, +// keyFields, +// }: UpdateParams): Promise<{ message: string }> => { +// const collection = db.collection(collectionName); +// +// try { +// const zipFilePath = `${EXTRACT_PATH}/${zipFileName}`; +// await downloadFile({ url: zipUrl, destination: zipFilePath }); +// +// const extractedFileName = await extractZipFile(zipFilePath, EXTRACT_PATH); +// const destinationPath = `${EXTRACT_PATH}/${extractedFileName}`; +// console.log(`Destination path:`, destinationPath); +// +// const csvData = await fs.readFile(destinationPath, "utf-8"); +// const parsedData = d3.csvParse(csvData); +// +// const existingData: WithId[] = await collection.find().toArray(); +// +// const createUniqueKey = ( +// item: T, +// keyFields: Array, +// ): string => +// keyFields +// .filter((field) => item[field]) +// .map((field) => item[field]) +// .join("-"); +// +// const existingDataMap: Map> = new Map( +// existingData.map((item) => [createUniqueKey(item, keyFields), item]), +// ); +// const newDataToInsert = parsedData.filter( +// (newItem) => !existingDataMap.has(createUniqueKey(newItem, keyFields)), +// ); +// +// let message: string; +// if (newDataToInsert.length > 0) { +// const result = await collection.insertMany(newDataToInsert); +// message = `${result.insertedCount} document(s) inserted`; +// } else { +// message = `No new data to insert. The provided data matches the existing records.`; +// } +// +// return { message }; +// } catch (error) { +// console.error(`An error has occurred:`, error); +// throw error; +// } +// }; app.get("/", async (c) => { const month = c.req.query("month"); @@ -150,59 +180,59 @@ app.get("/petrol", async (c) => { return c.json(cars); }); -app.get("/coe", (c) => { - return c.json(db.collection("coe").find().toArray()); -}); - -app.get("/coe/latest", async (c) => { - const month = c.req.query("month"); - const result: WithId[] = await getCOEResultByMonth(month); - return c.json(result); -}); - -app.get("/updater/cars", async (c) => { - const COLLECTION_NAME: string = "cars"; - const ZIP_FILE_NAME: string = "Monthly New Registration of Cars by Make.zip"; - const ZIP_URL: string = `https://datamall.lta.gov.sg/content/dam/datamall/datasets/Facts_Figures/Vehicle Registration/${ZIP_FILE_NAME}`; - - const { message } = await update({ - collectionName: COLLECTION_NAME, - zipFileName: ZIP_FILE_NAME, - zipUrl: ZIP_URL, - keyFields: ["month"], - }); - - console.log(`Message:`, message); - - return c.json({ - status: 200, - collection: COLLECTION_NAME, - message, - timestamp: new Date().toISOString(), - }); -}); - -app.get("/updater/coe", async (c) => { - const COLLECTION_NAME: string = "coe"; - const ZIP_FILE_NAME: string = "COE Bidding Results.zip"; - const ZIP_URL: string = `https://datamall.lta.gov.sg/content/dam/datamall/datasets/Facts_Figures/Vehicle Registration/${ZIP_FILE_NAME}`; - - const { message } = await update({ - collectionName: COLLECTION_NAME, - zipFileName: ZIP_FILE_NAME, - zipUrl: ZIP_URL, - keyFields: ["month", "bidding_no"], - }); - - console.log(`Message:`, message); - - return c.json({ - status: 200, - collection: COLLECTION_NAME, - message, - timestamp: new Date().toISOString(), - }); -}); +// app.get("/coe", (c) => { +// return c.json(db.collection("coe").find().toArray()); +// }); +// +// app.get("/coe/latest", async (c) => { +// const month = c.req.query("month"); +// const result: WithId[] = await getCOEResultByMonth(month); +// return c.json(result); +// }); + +// app.get("/updater/cars", async (c) => { +// const COLLECTION_NAME: string = "cars"; +// const ZIP_FILE_NAME: string = "Monthly New Registration of Cars by Make.zip"; +// const ZIP_URL: string = `https://datamall.lta.gov.sg/content/dam/datamall/datasets/Facts_Figures/Vehicle Registration/${ZIP_FILE_NAME}`; +// +// const { message } = await update({ +// collectionName: COLLECTION_NAME, +// zipFileName: ZIP_FILE_NAME, +// zipUrl: ZIP_URL, +// keyFields: ["month"], +// }); +// +// console.log(`Message:`, message); +// +// return c.json({ +// status: 200, +// collection: COLLECTION_NAME, +// message, +// timestamp: new Date().toISOString(), +// }); +// }); +// +// app.get("/updater/coe", async (c) => { +// const COLLECTION_NAME: string = "coe"; +// const ZIP_FILE_NAME: string = "COE Bidding Results.zip"; +// const ZIP_URL: string = `https://datamall.lta.gov.sg/content/dam/datamall/datasets/Facts_Figures/Vehicle Registration/${ZIP_FILE_NAME}`; +// +// const { message } = await update({ +// collectionName: COLLECTION_NAME, +// zipFileName: ZIP_FILE_NAME, +// zipUrl: ZIP_URL, +// keyFields: ["month", "bidding_no"], +// }); +// +// console.log(`Message:`, message); +// +// return c.json({ +// status: 200, +// collection: COLLECTION_NAME, +// message, +// timestamp: new Date().toISOString(), +// }); +// }); app.get("/vehicle-make", (c) => { return c.json(db.collection("cars").distinct("make")); diff --git a/sst-env.d.ts b/sst-env.d.ts new file mode 100644 index 0000000..8f3af7b --- /dev/null +++ b/sst-env.d.ts @@ -0,0 +1,7 @@ +/* tslint:disable *//* eslint-disable */import "sst" +declare module "sst" { + export interface Resource { + Database: import("@cloudflare/workers-types").D1Database + } +} +export {} \ No newline at end of file diff --git a/sst.config.ts b/sst.config.ts index a58d4c8..6956837 100644 --- a/sst.config.ts +++ b/sst.config.ts @@ -9,9 +9,11 @@ export default $config({ }; }, async run() { + const database = new sst.cloudflare.D1("Database"); const hono = new sst.cloudflare.Worker("Api", { - url: true, handler: "src/index.ts", + link: [database], + url: true, }); return {