From 2b5643f6cf9a7613213f5fb8ec95f0d75778f6df Mon Sep 17 00:00:00 2001 From: Francis Duvivier Date: Wed, 27 Nov 2024 23:07:54 +0100 Subject: [PATCH] Fix populateDB --- public/swagger.json | 18 ++- src/db/BadgeHubDataPostgresAdapter.ts | 62 ++++++--- src/db/models/DBProjectStatusOnBadge.ts | 12 +- src/db/models/README.md | 1 + src/db/sqlHelpers/objectToSQL.ts | 13 +- src/generated/routes.ts | 20 ++- src/setupPopulateDBApi.ts | 169 +++++++++++++----------- 7 files changed, 179 insertions(+), 116 deletions(-) diff --git a/public/swagger.json b/public/swagger.json index c4639b5..5baca7d 100644 --- a/public/swagger.json +++ b/public/swagger.json @@ -148,7 +148,8 @@ "format": "date-time" }, "id": { - "type": "string" + "type": "number", + "format": "double" }, "email": { "type": "string" @@ -447,7 +448,8 @@ "type": "string" }, "user_id": { - "type": "string" + "type": "number", + "format": "double" }, "git": { "type": "string" @@ -656,7 +658,8 @@ "type": "boolean" }, "user_id": { - "type": "string" + "type": "number", + "format": "double" } }, "required": ["user_id"], @@ -676,7 +679,8 @@ "type": "boolean" }, "user_id": { - "type": "string" + "type": "number", + "format": "double" } }, "required": ["user_id"], @@ -696,7 +700,8 @@ "type": "boolean" }, "user_id": { - "type": "string" + "type": "number", + "format": "double" } }, "type": "object", @@ -1210,7 +1215,8 @@ "name": "userId", "required": true, "schema": { - "type": "string" + "format": "double", + "type": "number" } } ], diff --git a/src/db/BadgeHubDataPostgresAdapter.ts b/src/db/BadgeHubDataPostgresAdapter.ts index e6d6b72..ed791b2 100644 --- a/src/db/BadgeHubDataPostgresAdapter.ts +++ b/src/db/BadgeHubDataPostgresAdapter.ts @@ -13,7 +13,7 @@ import { getPool } from "@db/connectionPool"; import { DBProject as DBProject } from "@db/models/app/DBProject"; import sql, { join, raw } from "sql-template-tag"; import { DBInsertUser } from "@db/models/app/DBUser"; -import { Entry, getEntriesWithDefinedValues } from "@util/objectEntries"; +import { getEntriesWithDefinedValues } from "@util/objectEntries"; import { DBBadge } from "@db/models/DBBadge"; import { getBaseSelectProjectQuery, @@ -33,19 +33,19 @@ import { DBInsertAppMetadataJSON, } from "@db/models/app/DBAppMetadataJSON"; import { DBCategory } from "@db/models/app/DBCategory"; -import { getInsertKeysAndValuesSql } from "@db/sqlHelpers/objectToSQL"; +import { + assertValidColumKey, + getInsertKeysAndValuesSql, +} from "@db/sqlHelpers/objectToSQL"; -function getUpdateAssigmentsSql( - definedEntries: Entry>>[] -) { +function getUpdateAssigmentsSql(changes: Object) { + const changeEntries = getEntriesWithDefinedValues(changes); + if (!changeEntries.length) { + return; + } return join( - definedEntries.map( - ([ - key, - value, - ]) => sql`${raw(key)} // raw is ok here because these keys are checked against our typescript definitions by tsoa - = - ${value}` + changeEntries.map( + ([key, value]) => sql`${raw(assertValidColumKey(key))} = ${value}` ) ); } @@ -91,13 +91,13 @@ export class BadgeHubDataPostgresAdapter implements BadgeHubDataPort { projectSlug: ProjectSlug, changes: Partial> ): Promise { - const definedEntries = getEntriesWithDefinedValues(changes); - const setters = getUpdateAssigmentsSql(definedEntries); - if (definedEntries.length !== 0) { - await this.pool.query(sql`update projects - set ${setters} - where slug = ${projectSlug}`); + const setters = getUpdateAssigmentsSql(changes); + if (!setters) { + return; } + await this.pool.query(sql`update projects + set ${setters} + where slug = ${projectSlug}`); } // TODO test @@ -107,12 +107,34 @@ export class BadgeHubDataPostgresAdapter implements BadgeHubDataPort { where slug = ${projectSlug}`); } - writeFile( + async writeFile( projectSlug: ProjectSlug, filePath: string, contents: string | Uint8Array ): Promise { - throw new Error("Method not implemented."); + if (filePath === "metadata.json") { + const appMetadata: DBAppMetadataJSON = JSON.parse( + typeof contents === "string" + ? contents + : new TextDecoder().decode(contents) + ); + const setters = getUpdateAssigmentsSql(appMetadata); + if (!setters) { + return; + } + + const appMetadataUpdateQuery = sql`update app_metadata_jsons + set ${setters} + where id = (select app_metadata_json_id + from versions v + where v.id = + (select projects.version_id from projects where slug = ${projectSlug}))`; + await this.pool.query(appMetadataUpdateQuery); + } else { + throw new Error( + "Method not implemented for files other than the metadata.json file yet." + ); + } } updateDraftMetadata( diff --git a/src/db/models/DBProjectStatusOnBadge.ts b/src/db/models/DBProjectStatusOnBadge.ts index 7b47772..6d5446e 100644 --- a/src/db/models/DBProjectStatusOnBadge.ts +++ b/src/db/models/DBProjectStatusOnBadge.ts @@ -3,11 +3,15 @@ import { BadgeSlugRelation } from "./DBBadge"; import { ProjectStatusName } from "@domain/readModels/app/Project"; import { ProjectSlugRelation } from "@db/models/app/DBProject"; +export interface DBInsertProjectStatusOnBadge + extends BadgeSlugRelation, + ProjectSlugRelation { + status?: ProjectStatusName; // Status for this project for this particular badge +} + // table name: project_statuses_on_badges export interface DBProjectStatusOnBadge - extends DBDatedData, - BadgeSlugRelation, - ProjectSlugRelation { + extends DBInsertProjectStatusOnBadge, + DBDatedData { id: number; - status: ProjectStatusName; // Status for this project for this particular badge } diff --git a/src/db/models/README.md b/src/db/models/README.md index 451756b..ba84b84 100644 --- a/src/db/models/README.md +++ b/src/db/models/README.md @@ -5,3 +5,4 @@ In the db models, we try to stick to 2 principles - DRY, no data duplication, unless it's accidental duplication - No arrays. Arrays make querying and switching to another DB more difficult. So instead of arrays, we try to work with relation tables. - There is currently an exception here for the MetaDataFileContents, there the mappings are records and arrays and records with arrays, this is because the data comes from the json file and we could also just store it as stringified JSON. +- Prefer slugs as primary keys where possible, because they are more human-readable which makes the api calls and db queries easier to read. diff --git a/src/db/sqlHelpers/objectToSQL.ts b/src/db/sqlHelpers/objectToSQL.ts index 00e32ea..fea19e8 100644 --- a/src/db/sqlHelpers/objectToSQL.ts +++ b/src/db/sqlHelpers/objectToSQL.ts @@ -1,9 +1,20 @@ import { getEntriesWithDefinedValues } from "@util/objectEntries"; import { join, raw } from "sql-template-tag"; +const COLUMN_KEY_REGEX = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + +export function assertValidColumKey(key: string): string { + if (!COLUMN_KEY_REGEX.test(key)) { + throw new Error("Invalid column key: " + key); + } + return key; +} + export function getInsertKeysAndValuesSql(user: Object) { const definedEntries = getEntriesWithDefinedValues(user); - const keys = join(definedEntries.map(([key]) => raw(key))); // raw is ok here because these keys are checked against our typescript definitions by tsoa + const keys = join( + definedEntries.map(([key]) => raw(assertValidColumKey(key))) + ); const values = join(definedEntries.map(([, value]) => value)); return { keys, values }; } diff --git a/src/generated/routes.ts b/src/generated/routes.ts index 8eaef8c..faa014d 100644 --- a/src/generated/routes.ts +++ b/src/generated/routes.ts @@ -1,11 +1,8 @@ /* tslint:disable */ /* eslint-disable */ // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa -import { - TsoaRoute, - fetchMiddlewares, - ExpressTemplateService, -} from "@tsoa/runtime"; +import type { TsoaRoute } from "@tsoa/runtime"; +import { fetchMiddlewares, ExpressTemplateService } from "@tsoa/runtime"; // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa import { PublicRestController } from "./../controllers/public-rest.js"; // WARNING: This file was auto-generated with tsoa. Please do not modify it. Re-run tsoa to re-generate this file: https://github.com/lukeautry/tsoa @@ -110,7 +107,7 @@ const models: TsoaRoute.Models = { created_at: { dataType: "datetime", required: true }, updated_at: { dataType: "datetime", required: true }, deleted_at: { dataType: "datetime" }, - id: { dataType: "string", required: true }, + id: { dataType: "double", required: true }, email: { dataType: "string", required: true }, admin: { dataType: "boolean", required: true }, name: { dataType: "string", required: true }, @@ -269,7 +266,7 @@ const models: TsoaRoute.Models = { dataType: "refObject", properties: { slug: { dataType: "string", required: true }, - user_id: { dataType: "string", required: true }, + user_id: { dataType: "double", required: true }, git: { dataType: "string" }, allow_team_fixes: { dataType: "boolean" }, created_at: { dataType: "datetime", required: true }, @@ -369,7 +366,7 @@ const models: TsoaRoute.Models = { version_id: { dataType: "double" }, git: { dataType: "string" }, allow_team_fixes: { dataType: "boolean" }, - user_id: { dataType: "string", required: true }, + user_id: { dataType: "double", required: true }, }, validators: {}, }, @@ -381,7 +378,7 @@ const models: TsoaRoute.Models = { version_id: { dataType: "double" }, git: { dataType: "string" }, allow_team_fixes: { dataType: "boolean" }, - user_id: { dataType: "string", required: true }, + user_id: { dataType: "double", required: true }, }, additionalProperties: false, }, @@ -392,7 +389,7 @@ const models: TsoaRoute.Models = { version_id: { dataType: "double" }, git: { dataType: "string" }, allow_team_fixes: { dataType: "boolean" }, - user_id: { dataType: "string" }, + user_id: { dataType: "double" }, }, additionalProperties: false, }, @@ -471,6 +468,7 @@ export function RegisterRoutes(app: Router) { // NOTE: If you do not see routes for all of your controllers in this file, then you might not have informed tsoa of where to look // Please look into the "controllerPathGlobs" config option described in the readme: https://github.com/lukeautry/tsoa // ########################################################################################################### + app.get( "/api/v3/devices", ...fetchMiddlewares(PublicRestController), @@ -850,7 +848,7 @@ export function RegisterRoutes(app: Router) { in: "path", name: "userId", required: true, - dataType: "string", + dataType: "double", }, props: { in: "body", name: "props", required: true, ref: "UserProps" }, }; diff --git a/src/setupPopulateDBApi.ts b/src/setupPopulateDBApi.ts index 7d5a2b1..f6c3409 100644 --- a/src/setupPopulateDBApi.ts +++ b/src/setupPopulateDBApi.ts @@ -7,9 +7,33 @@ import { POSTGRES_PORT, POSTGRES_USER, } from "@config"; +import sql from "sql-template-tag"; +import { DBInsertUser } from "@db/models/app/DBUser"; +import { DBDatedData } from "@db/models/app/DBDatedData"; +import { DBInsertProject } from "@db/models/app/DBProject"; +import { BadgeHubDataPostgresAdapter } from "@db/BadgeHubDataPostgresAdapter"; +import { DBInsertAppMetadataJSON } from "@db/models/app/DBAppMetadataJSON"; +import { getInsertKeysAndValuesSql } from "@db/sqlHelpers/objectToSQL"; +import { DBInsertProjectStatusOnBadge } from "@db/models/DBProjectStatusOnBadge"; -const CATEGORIES_COUNT = 15; - +const CATEGORY_NAMES = [ + "Uncategorised", + "Event related", + "Games", + "Graphics", + "Hardware", + "Utility", + "Wearable", + "Data", + "Silly", + "Hacking", + "Troll", + "Unusable", + "Adult", + "Virus", + // "Interpreter", // TODO add Interpreter to mock data? +] as const; +const CATEGORIES_COUNT = CATEGORY_NAMES.length; export default async function setupPopulateDBApi(app: Express) { const router = Router(); @@ -25,27 +49,29 @@ export default async function setupPopulateDBApi(app: Express) { port: POSTGRES_PORT, }); - router.get("/populate", async (req, res) => { - await deleteDatabases(pool); + router.post("/populate", async (req, res) => { + await cleanDatabases(pool); await populateDatabases(pool); return res.status(200).send("Population done."); }); } -async function deleteDatabases(pool: pg.Pool) { +async function cleanDatabases(pool: pg.Pool) { const client = await pool.connect(); - await client.query("DELETE FROM badgehub.badge_project"); - await client.query("DELETE FROM badgehub.dependencies"); - await client.query("DELETE FROM badgehub.users"); - await client.query("DELETE FROM badgehub.projects"); + await client.query(sql`delete from badgehub.users`); + await client.query(sql`delete from badgehub.projects`); + await client.query(sql`delete from badgehub.app_metadata_jsons`); + await client.query(sql`delete from badgehub.versions`); + await client.query(sql`delete from badgehub.versioned_dependencies`); + await client.query(sql`delete from badgehub.project_statuses_on_badges`); client.release(); } async function populateDatabases(pool: pg.Pool) { const client = await pool.connect(); const userCount = await insertUsers(client); - const projectCount = await insertProjects(client, userCount); - await badgeProjectCrossTable(client, projectCount); + const projectSlugs = await insertProjects(client, userCount); + await badgeProjectCrossTable(client, projectSlugs); client.release(); } @@ -156,7 +182,7 @@ async function insertUsers(client: pg.PoolClient) { "hotmail.com", ]; - for (const id in users) { + for (let id = 0; id < users.length; id++) { const isAdmin = random(10) == 0; const name = users[id]!; const email = `${name.toLowerCase()}@${domains[random(domains.length)]}`; @@ -168,30 +194,26 @@ async function insertUsers(client: pg.PoolClient) { const updatedAt = date(createDate + random(100)); console.log(`insert into users ${name}`); - - await client.query( - `INSERT INTO badgehub.users - (id, admin, name, email, password, public, show_projects, created_at, updated_at) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, - [ - id, - isAdmin, - name, - email, - password, - isPublic, - showProjects, - createdAt, - updatedAt, - ] - ); + const toInsert: DBInsertUser & DBDatedData = { + id, + admin: isAdmin, + name, + email, + password, + public: isPublic, + show_projects: showProjects, + created_at: createdAt, + updated_at: updatedAt, + }; + const badgeHubAdapter = new BadgeHubDataPostgresAdapter(); + await badgeHubAdapter.insertUser(toInsert); } return users.length; } async function insertProjects(client: pg.PoolClient, userCount: number) { - const apps = [ + const projectSlugs = [ "CodeCraft", "PixelPulse", "BitBlast", @@ -280,74 +302,73 @@ async function insertProjects(client: pg.PoolClient, userCount: number) { "HackQuest", "SecureSphere", ]; + const badgeHubAdapter = new BadgeHubDataPostgresAdapter(); - for (const id in apps) { - const name = apps[id]!; + for (let id = 0; id < projectSlugs.length; id++) { + const name = projectSlugs[id]!; const slug = name.toLowerCase(); const description = getDescription(name); const userId = random(userCount); - const categoryId = random(CATEGORIES_COUNT) + 1; + const categoryId = random(CATEGORIES_COUNT); const createDate = -random(600); const createdAt = date(createDate); const updatedAt = date(createDate + random(100)); console.log(`insert into projects ${name} (${description})`); - await client.query( - `INSERT INTO badgehub.projects - (id, name, slug, description, user_id, category_id, created_at, updated_at) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8)`, - [id, name, slug, description, userId, categoryId, createdAt, updatedAt] + const inserted: DBInsertProject & DBDatedData = { + slug, + user_id: userId, + created_at: createdAt, + updated_at: updatedAt, + }; + + await badgeHubAdapter.insertProject(inserted); + const appMetadata: DBInsertAppMetadataJSON & DBDatedData = { + name, + description, + category: CATEGORY_NAMES[categoryId], + created_at: createdAt, + updated_at: updatedAt, + }; + + await badgeHubAdapter.writeFile( + inserted.slug, + "metadata.json", + JSON.stringify(appMetadata) ); } - return apps.length; + return projectSlugs.map((slug) => slug.toLowerCase()); } async function badgeProjectCrossTable( client: pg.PoolClient, - projectCount: number + projectSlugs: string[] ) { - const badgeIds = [1, 2, 5]; // Hardcoded! Update by hand - for (let index = 0; index < projectCount; index++) { - const badgeId = badgeIds[random(3)]; + const badgeSlugs = ["mch2022", "troopers23", "why2025"] as const; // Hardcoded! Update by hand + for (let index = 0; index < projectSlugs.length; index++) { + const badgeSlug = badgeSlugs[random(3)]!; + let insertObject1: DBInsertProjectStatusOnBadge = { + badge_slug: badgeSlug, + project_slug: projectSlugs[index]!, + }; + const insert1 = getInsertKeysAndValuesSql(insertObject1); await client.query( - `INSERT INTO badgehub.badge_project - (badge_id, project_id) VALUES - ($1, $2)`, - [badgeId, index] + sql`insert into badgehub.project_statuses_on_badges (${insert1.keys}) values (${insert1.values})` ); // Some project support two badges - const badgeId2 = badgeIds[random(3)]; - if (badgeId2 != badgeId && random(3) == 1) { + const badgeId2 = badgeSlugs[random(3)]!; + if (badgeId2 != badgeSlug && random(3) == 1) { + const insertObject2: DBInsertProjectStatusOnBadge = { + badge_slug: badgeId2, + project_slug: projectSlugs[index]!, + }; + const insert2 = getInsertKeysAndValuesSql(insertObject2); await client.query( - `INSERT INTO badgehub.badge_project - (badge_id, project_id) VALUES - ($1, $2)`, - [badgeId2, index] + sql`insert into badgehub.project_statuses_on_badges (${insert2.keys}) values (${insert2.values})` ); } } } - -// Not in use right now -async function userProjectsCrossTable( - client: pg.PoolClient, - userCount: number, - projectCount: number -) { - for (let index = 0; index < 300; index++) { - const userId = random(userCount); - const projectId = random(projectCount); - const createDate = -random(600); - const createdAt = date(createDate); - const updatedAt = date(createDate + random(100)); - await client.query( - `INSERT INTO badgehub.project_user - (id, user_id, project_id, created_at, updated_at) VALUES - ($1, $2, $3, $4, $5)`, - [index, userId, projectId, createdAt, updatedAt] - ); - } -}