26 db migration, type safety, domain layer and insert project api

.env.example

@@ -5,4 +5,6 @@ POSTGRES_PASSWORD=badgehub
 PGADMIN_DEFAULT_PASSWORD=badgehub
 [email protected]
 
-JWT_SIGNING_KEY="!!HackThePlanet!!"
+NODE_ENV=development
+
+JWT_SIGNING_KEY="!!HackThePlanet!!"

README.md

@@ -4,6 +4,56 @@
 
 ## - Development -
 
+### Database Migrations
+
+In order to make sure that we can keep track of the database schema, we use [db-migrate](https://db-migrate.readthedocs.io/en/latest/).
+This allows us to track the database changes along with git and provide a framework for migrating data and rolling back changes.
+So when a change to the database schema is needed, a new migration should be created instead of manually changing the database.
+To create a new migration, follow the steps below.
+
+#### Create a new migration
+
+```bash
+npm run db-migrate:create -- <migration-name>
+```
+
+This will create a new migration file in the `migrations` directory with the name `<timestamp>-<migration-name>.js` as well as 2 sql files, one for the up migration and one for the down migration.
+
+#### Fill in the down and up migration sql files with the necessary changes to the database schema.
+
+These sql commands should take care of changing the database schema as well as migrating the data if necessary.
+
+#### Run the migration to test it.
+
+```bash
+npm run db-migrate:up
+```
+
+#### Create updated mock.sql
+
+For the mock data, we always want up to date tables, so after you have done the migration on the mock data, you should re-export the database.
+You can do this with the pg_dump command in the postgres container:
+
+```bash
+npm run overwrite-mockup-data
+```
+
+Note: you might have to change the db container name in the script. Eg on mac with podman its with underscores instead of dashes which makes the command:
+
+```bash
+docker exec -it badgehub-api_db_1 /usr/bin/pg_dump --username badgehub --schema badgehub badgehub > mockup-data.sql
+```
+
+#### Run the down migration to test it.
+
+```bash
+npm run db-migrate:down
+```
+
+#### Commit the migration files to git.
+
+When the code is deployed, the up migrations will be run automatically before starting the server.
+
 ## Install
 
 Make sure [Docker](https://www.docker.com/get-started/) is installed and running.
@@ -99,5 +149,7 @@ docker compose --file docker-compose-production.yml down
 
 - [Express](https://expressjs.com/), a framework for Node.js
 - [tsoa](https://tsoa-community.github.io/docs/) for generating a swagger file from code
+- [sql-template-tag](https://github.com/blakeembrey/sql-template-tag) for more easily writing SQL queries
+- [db-migrate](https://db-migrate.readthedocs.io/en/latest/) for database migrations
 - [tsx](https://tsx.is/) for watching TypeScript files in Node.js
 - [PM2](https://pm2.keymetrics.io/) for managing Node.js processes

database.json

@@ -0,0 +1,12 @@
+{
+  "defaultEnv": "pg-from-env",
+  "pg-from-env": {
+    "driver": "pg",
+    "host": { "ENV": "POSTGRES_HOST" },
+    "database": { "ENV": "POSTGRES_DB" },
+    "user": { "ENV": "POSTGRES_USER" },
+    "password": { "ENV": "POSTGRES_PASSWORD" },
+    "port": "5432",
+    "schema": "badgehub"
+  }
+}

migrations/20241116085102-initialize.js

@@ -0,0 +1,59 @@
+"use strict";
+
+var dbm;
+var type;
+var seed;
+var fs = require("fs");
+var path = require("path");
+var Promise;
+
+/**
+ * We receive the dbmigrate dependency from dbmigrate initially.
+ * This enables us to not have to rely on NODE_PATH.
+ */
+exports.setup = function (options, seedLink) {
+  dbm = options.dbmigrate;
+  type = dbm.dataType;
+  seed = seedLink;
+  Promise = options.Promise;
+};
+
+exports.up = function (db) {
+  var filePath = path.join(
+    __dirname,
+    "sqls",
+    "20241116085102-initialize-up.sql"
+  );
+  return new Promise(function (resolve, reject) {
+    fs.readFile(filePath, { encoding: "utf-8" }, function (err, data) {
+      if (err) return reject(err);
+      console.log("received data: " + data);
+
+      resolve(data);
+    });
+  }).then(function (data) {
+    return db.runSql(data);
+  });
+};
+
+exports.down = function (db) {
+  var filePath = path.join(
+    __dirname,
+    "sqls",
+    "20241116085102-initialize-down.sql"
+  );
+  return new Promise(function (resolve, reject) {
+    fs.readFile(filePath, { encoding: "utf-8" }, function (err, data) {
+      if (err) return reject(err);
+      console.log("received data: " + data);
+
+      resolve(data);
+    });
+  }).then(function (data) {
+    return db.runSql(data);
+  });
+};
+
+exports._meta = {
+  version: 1,
+};

migrations/package.json

@@ -0,0 +1,3 @@
+{
+  "type": "commonjs"
+}

migrations/sqls/20241116085102-initialize-down.sql

@@ -0,0 +1,4 @@
+-- restore old schema
+insert into badgehub_old.migrations select * from badgehub.migrations;
+drop schema badgehub cascade;
+alter schema badgehub_old rename to badgehub;