From 4dcd6d7084a99101f236f5340174973e53fe4b9e Mon Sep 17 00:00:00 2001 From: Francis Duvivier Date: Wed, 27 Nov 2024 23:07:54 +0100 Subject: [PATCH] Fix populate --- src/db/BadgeHubDataPostgresAdapter.ts | 62 ++++++--- src/db/models/DBProjectStatusOnBadge.ts | 12 +- src/db/models/README.md | 1 + src/db/sqlHelpers/objectToSQL.ts | 13 +- src/generated/routes.ts | 4 +- src/setupPopulateDBApi.ts | 169 +++++++++++++----------- 6 files changed, 160 insertions(+), 101 deletions(-) diff --git a/src/db/BadgeHubDataPostgresAdapter.ts b/src/db/BadgeHubDataPostgresAdapter.ts index 17108fb..2ce587c 100644 --- a/src/db/BadgeHubDataPostgresAdapter.ts +++ b/src/db/BadgeHubDataPostgresAdapter.ts @@ -13,7 +13,7 @@ import { getPool } from "@db/connectionPool"; import { DBProject as DBProject } from "@db/models/app/DBProject"; import sql, { join, raw } from "sql-template-tag"; import { DBInsertUser } from "@db/models/app/DBUser"; -import { Entry, getEntriesWithDefinedValues } from "@util/objectEntries"; +import { getEntriesWithDefinedValues } from "@util/objectEntries"; import { DBBadge } from "@db/models/DBBadge"; import { getBaseSelectProjectQuery, @@ -33,19 +33,19 @@ import { DBInsertAppMetadataJSON, } from "@db/models/app/DBAppMetadataJSON"; import { DBCategory } from "@db/models/app/DBCategory"; -import { getInsertKeysAndValuesSql } from "@db/sqlHelpers/objectToSQL"; +import { + assertValidColumKey, + getInsertKeysAndValuesSql, +} from "@db/sqlHelpers/objectToSQL"; -function getUpdateAssigmentsSql( - definedEntries: Entry>>[] -) { +function getUpdateAssigmentsSql(changes: Object) { + const changeEntries = getEntriesWithDefinedValues(changes); + if (!changeEntries.length) { + return; + } return join( - definedEntries.map( - ([ - key, - value, - ]) => sql`${raw(key)} // raw is ok here because these keys are checked against our typescript definitions by tsoa - = - ${value}` + changeEntries.map( + ([key, value]) => sql`${raw(assertValidColumKey(key))} = ${value}` ) ); } @@ -91,13 +91,13 @@ export class BadgeHubDataPostgresAdapter implements BadgeHubDataPort { projectSlug: ProjectSlug, changes: Partial> ): Promise { - const definedEntries = getEntriesWithDefinedValues(changes); - const setters = getUpdateAssigmentsSql(definedEntries); - if (definedEntries.length !== 0) { - await this.pool.query(sql`update projects - set ${setters} - where slug = ${projectSlug}`); + const setters = getUpdateAssigmentsSql(changes); + if (!setters) { + return; } + await this.pool.query(sql`update projects + set ${setters} + where slug = ${projectSlug}`); } async deleteProject(projectSlug: ProjectSlug): Promise { @@ -106,12 +106,34 @@ export class BadgeHubDataPostgresAdapter implements BadgeHubDataPort { where slug = ${projectSlug}`); } - writeFile( + async writeFile( projectSlug: ProjectSlug, filePath: string, contents: string | Uint8Array ): Promise { - throw new Error("Method not implemented."); + if (filePath === "metadata.json") { + const appMetadata: DBAppMetadataJSON = JSON.parse( + typeof contents === "string" + ? contents + : new TextDecoder().decode(contents) + ); + const setters = getUpdateAssigmentsSql(appMetadata); + if (!setters) { + return; + } + + const appMetadataUpdateQuery = sql`update app_metadata_jsons + set ${setters} + where id = (select app_metadata_json_id + from versions v + where v.id = + (select projects.version_id from projects where slug = ${projectSlug}))`; + await this.pool.query(appMetadataUpdateQuery); + } else { + throw new Error( + "Method not implemented for files other than the metadata.json file yet." + ); + } } updateDraftMetadata( diff --git a/src/db/models/DBProjectStatusOnBadge.ts b/src/db/models/DBProjectStatusOnBadge.ts index 7b47772..6d5446e 100644 --- a/src/db/models/DBProjectStatusOnBadge.ts +++ b/src/db/models/DBProjectStatusOnBadge.ts @@ -3,11 +3,15 @@ import { BadgeSlugRelation } from "./DBBadge"; import { ProjectStatusName } from "@domain/readModels/app/Project"; import { ProjectSlugRelation } from "@db/models/app/DBProject"; +export interface DBInsertProjectStatusOnBadge + extends BadgeSlugRelation, + ProjectSlugRelation { + status?: ProjectStatusName; // Status for this project for this particular badge +} + // table name: project_statuses_on_badges export interface DBProjectStatusOnBadge - extends DBDatedData, - BadgeSlugRelation, - ProjectSlugRelation { + extends DBInsertProjectStatusOnBadge, + DBDatedData { id: number; - status: ProjectStatusName; // Status for this project for this particular badge } diff --git a/src/db/models/README.md b/src/db/models/README.md index 451756b..ba84b84 100644 --- a/src/db/models/README.md +++ b/src/db/models/README.md @@ -5,3 +5,4 @@ In the db models, we try to stick to 2 principles - DRY, no data duplication, unless it's accidental duplication - No arrays. Arrays make querying and switching to another DB more difficult. So instead of arrays, we try to work with relation tables. - There is currently an exception here for the MetaDataFileContents, there the mappings are records and arrays and records with arrays, this is because the data comes from the json file and we could also just store it as stringified JSON. +- Prefer slugs as primary keys where possible, because they are more human-readable which makes the api calls and db queries easier to read. diff --git a/src/db/sqlHelpers/objectToSQL.ts b/src/db/sqlHelpers/objectToSQL.ts index 00e32ea..fea19e8 100644 --- a/src/db/sqlHelpers/objectToSQL.ts +++ b/src/db/sqlHelpers/objectToSQL.ts @@ -1,9 +1,20 @@ import { getEntriesWithDefinedValues } from "@util/objectEntries"; import { join, raw } from "sql-template-tag"; +const COLUMN_KEY_REGEX = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + +export function assertValidColumKey(key: string): string { + if (!COLUMN_KEY_REGEX.test(key)) { + throw new Error("Invalid column key: " + key); + } + return key; +} + export function getInsertKeysAndValuesSql(user: Object) { const definedEntries = getEntriesWithDefinedValues(user); - const keys = join(definedEntries.map(([key]) => raw(key))); // raw is ok here because these keys are checked against our typescript definitions by tsoa + const keys = join( + definedEntries.map(([key]) => raw(assertValidColumKey(key))) + ); const values = join(definedEntries.map(([, value]) => value)); return { keys, values }; } diff --git a/src/generated/routes.ts b/src/generated/routes.ts index 4e56852..25f3fcd 100644 --- a/src/generated/routes.ts +++ b/src/generated/routes.ts @@ -2,15 +2,15 @@ /* eslint-disable */ // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa import type { TsoaRoute } from "@tsoa/runtime"; -import { fetchMiddlewares, ExpressTemplateService } from "@tsoa/runtime"; +import { ExpressTemplateService, fetchMiddlewares } from "@tsoa/runtime"; // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa import { PublicRestController } from "./../controllers/public-rest.js"; // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa import { PrivateRestController } from "./../controllers/private-rest.js"; import type { Request as ExRequest, - Response as ExResponse, RequestHandler, + Response as ExResponse, Router, } from "express"; diff --git a/src/setupPopulateDBApi.ts b/src/setupPopulateDBApi.ts index 7d5a2b1..f6c3409 100644 --- a/src/setupPopulateDBApi.ts +++ b/src/setupPopulateDBApi.ts @@ -7,9 +7,33 @@ import { POSTGRES_PORT, POSTGRES_USER, } from "@config"; +import sql from "sql-template-tag"; +import { DBInsertUser } from "@db/models/app/DBUser"; +import { DBDatedData } from "@db/models/app/DBDatedData"; +import { DBInsertProject } from "@db/models/app/DBProject"; +import { BadgeHubDataPostgresAdapter } from "@db/BadgeHubDataPostgresAdapter"; +import { DBInsertAppMetadataJSON } from "@db/models/app/DBAppMetadataJSON"; +import { getInsertKeysAndValuesSql } from "@db/sqlHelpers/objectToSQL"; +import { DBInsertProjectStatusOnBadge } from "@db/models/DBProjectStatusOnBadge"; -const CATEGORIES_COUNT = 15; - +const CATEGORY_NAMES = [ + "Uncategorised", + "Event related", + "Games", + "Graphics", + "Hardware", + "Utility", + "Wearable", + "Data", + "Silly", + "Hacking", + "Troll", + "Unusable", + "Adult", + "Virus", + // "Interpreter", // TODO add Interpreter to mock data? +] as const; +const CATEGORIES_COUNT = CATEGORY_NAMES.length; export default async function setupPopulateDBApi(app: Express) { const router = Router(); @@ -25,27 +49,29 @@ export default async function setupPopulateDBApi(app: Express) { port: POSTGRES_PORT, }); - router.get("/populate", async (req, res) => { - await deleteDatabases(pool); + router.post("/populate", async (req, res) => { + await cleanDatabases(pool); await populateDatabases(pool); return res.status(200).send("Population done."); }); } -async function deleteDatabases(pool: pg.Pool) { +async function cleanDatabases(pool: pg.Pool) { const client = await pool.connect(); - await client.query("DELETE FROM badgehub.badge_project"); - await client.query("DELETE FROM badgehub.dependencies"); - await client.query("DELETE FROM badgehub.users"); - await client.query("DELETE FROM badgehub.projects"); + await client.query(sql`delete from badgehub.users`); + await client.query(sql`delete from badgehub.projects`); + await client.query(sql`delete from badgehub.app_metadata_jsons`); + await client.query(sql`delete from badgehub.versions`); + await client.query(sql`delete from badgehub.versioned_dependencies`); + await client.query(sql`delete from badgehub.project_statuses_on_badges`); client.release(); } async function populateDatabases(pool: pg.Pool) { const client = await pool.connect(); const userCount = await insertUsers(client); - const projectCount = await insertProjects(client, userCount); - await badgeProjectCrossTable(client, projectCount); + const projectSlugs = await insertProjects(client, userCount); + await badgeProjectCrossTable(client, projectSlugs); client.release(); } @@ -156,7 +182,7 @@ async function insertUsers(client: pg.PoolClient) { "hotmail.com", ]; - for (const id in users) { + for (let id = 0; id < users.length; id++) { const isAdmin = random(10) == 0; const name = users[id]!; const email = `${name.toLowerCase()}@${domains[random(domains.length)]}`; @@ -168,30 +194,26 @@ async function insertUsers(client: pg.PoolClient) { const updatedAt = date(createDate + random(100)); console.log(`insert into users ${name}`); - - await client.query( - `INSERT INTO badgehub.users - (id, admin, name, email, password, public, show_projects, created_at, updated_at) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, - [ - id, - isAdmin, - name, - email, - password, - isPublic, - showProjects, - createdAt, - updatedAt, - ] - ); + const toInsert: DBInsertUser & DBDatedData = { + id, + admin: isAdmin, + name, + email, + password, + public: isPublic, + show_projects: showProjects, + created_at: createdAt, + updated_at: updatedAt, + }; + const badgeHubAdapter = new BadgeHubDataPostgresAdapter(); + await badgeHubAdapter.insertUser(toInsert); } return users.length; } async function insertProjects(client: pg.PoolClient, userCount: number) { - const apps = [ + const projectSlugs = [ "CodeCraft", "PixelPulse", "BitBlast", @@ -280,74 +302,73 @@ async function insertProjects(client: pg.PoolClient, userCount: number) { "HackQuest", "SecureSphere", ]; + const badgeHubAdapter = new BadgeHubDataPostgresAdapter(); - for (const id in apps) { - const name = apps[id]!; + for (let id = 0; id < projectSlugs.length; id++) { + const name = projectSlugs[id]!; const slug = name.toLowerCase(); const description = getDescription(name); const userId = random(userCount); - const categoryId = random(CATEGORIES_COUNT) + 1; + const categoryId = random(CATEGORIES_COUNT); const createDate = -random(600); const createdAt = date(createDate); const updatedAt = date(createDate + random(100)); console.log(`insert into projects ${name} (${description})`); - await client.query( - `INSERT INTO badgehub.projects - (id, name, slug, description, user_id, category_id, created_at, updated_at) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8)`, - [id, name, slug, description, userId, categoryId, createdAt, updatedAt] + const inserted: DBInsertProject & DBDatedData = { + slug, + user_id: userId, + created_at: createdAt, + updated_at: updatedAt, + }; + + await badgeHubAdapter.insertProject(inserted); + const appMetadata: DBInsertAppMetadataJSON & DBDatedData = { + name, + description, + category: CATEGORY_NAMES[categoryId], + created_at: createdAt, + updated_at: updatedAt, + }; + + await badgeHubAdapter.writeFile( + inserted.slug, + "metadata.json", + JSON.stringify(appMetadata) ); } - return apps.length; + return projectSlugs.map((slug) => slug.toLowerCase()); } async function badgeProjectCrossTable( client: pg.PoolClient, - projectCount: number + projectSlugs: string[] ) { - const badgeIds = [1, 2, 5]; // Hardcoded! Update by hand - for (let index = 0; index < projectCount; index++) { - const badgeId = badgeIds[random(3)]; + const badgeSlugs = ["mch2022", "troopers23", "why2025"] as const; // Hardcoded! Update by hand + for (let index = 0; index < projectSlugs.length; index++) { + const badgeSlug = badgeSlugs[random(3)]!; + let insertObject1: DBInsertProjectStatusOnBadge = { + badge_slug: badgeSlug, + project_slug: projectSlugs[index]!, + }; + const insert1 = getInsertKeysAndValuesSql(insertObject1); await client.query( - `INSERT INTO badgehub.badge_project - (badge_id, project_id) VALUES - ($1, $2)`, - [badgeId, index] + sql`insert into badgehub.project_statuses_on_badges (${insert1.keys}) values (${insert1.values})` ); // Some project support two badges - const badgeId2 = badgeIds[random(3)]; - if (badgeId2 != badgeId && random(3) == 1) { + const badgeId2 = badgeSlugs[random(3)]!; + if (badgeId2 != badgeSlug && random(3) == 1) { + const insertObject2: DBInsertProjectStatusOnBadge = { + badge_slug: badgeId2, + project_slug: projectSlugs[index]!, + }; + const insert2 = getInsertKeysAndValuesSql(insertObject2); await client.query( - `INSERT INTO badgehub.badge_project - (badge_id, project_id) VALUES - ($1, $2)`, - [badgeId2, index] + sql`insert into badgehub.project_statuses_on_badges (${insert2.keys}) values (${insert2.values})` ); } } } - -// Not in use right now -async function userProjectsCrossTable( - client: pg.PoolClient, - userCount: number, - projectCount: number -) { - for (let index = 0; index < 300; index++) { - const userId = random(userCount); - const projectId = random(projectCount); - const createDate = -random(600); - const createdAt = date(createDate); - const updatedAt = date(createDate + random(100)); - await client.query( - `INSERT INTO badgehub.project_user - (id, user_id, project_id, created_at, updated_at) VALUES - ($1, $2, $3, $4, $5)`, - [index, userId, projectId, createdAt, updatedAt] - ); - } -}