Merge pull request #2113 from IDEMSInternational/fix/api-dbs

Fix/api dbs

documentation/docs/developers/server-development.md

@@ -37,6 +37,22 @@ Ensure `API_BASE_PATH=""` to allow running standalone (as opposed to part of ful
 4. Interact with app.
 By default any app running locally via `yarn start` will target the localhost api, so in-app operations such as contact field syncing can be tested in the same way as production
 
+### API Tests
+E2E tests can be run via
+```sh
+yarn workspace api test:e2e
+```
+If developing tests an option `--watch` flag can be added at the end to live-reload
+
+An additional admin user will need to be created on the local db as an alternative to the postgres admin. This will also require creation of a database to support initial bootstrap connection
+
+```
+username: test_admin
+password: test_admin
+
+database: test_admin
+```
+
 
 ## Stack development
 Requires [docker desktop](https://www.docker.com/products/docker-desktop/) installed locally

packages/api/jest.config-e2e.ts

@@ -0,0 +1,16 @@
+import type { JestConfigWithTsJest } from "ts-jest";
+
+import baseConfig from "./jest.config";
+
+const config: JestConfigWithTsJest = {
+  ...baseConfig,
+  rootDir: ".",
+  testEnvironment: "node",
+  testRegex: ".e2e.spec.ts$",
+  moduleNameMapper: {
+    "^src/(.*)$": "<rootDir>/src/$1",
+  },
+  // setupFiles: ["<rootDir>/test/e2e-setup.ts"],
+};
+
+export default config;

packages/api/jest.config.ts

@@ -0,0 +1,15 @@
+import type { JestConfigWithTsJest } from "ts-jest";
+
+const config: JestConfigWithTsJest = {
+  moduleFileExtensions: ["js", "json", "ts"],
+  rootDir: "src",
+  testRegex: ".*\\.spec\\.ts$",
+  transform: {
+    "^.+\\.(t|j)s$": "ts-jest",
+  },
+  collectCoverageFrom: ["**/*.(t|j)s"],
+  coverageDirectory: "../coverage",
+  testEnvironment: "node",
+};
+
+export default config;

packages/api/package.json

@@ -14,68 +14,52 @@
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
-    "test:e2e": "jest --config ./test/jest-e2e.json",
+    "test:e2e": "jest --config jest.config-e2e.ts",
     "docker:build": "echo $npm_package_version && DOCKER_BUILDKIT=1 docker build -f Dockerfile --tag idems/apps-api:$npm_package_version .",
     "docker:run": "docker run -p 3000:3000 --name apps-api --rm idems/apps-api:$npm_package_version"
   },
   "dependencies": {
-    "@nestjs/common": "9.4.0",
-    "@nestjs/config": "^2.3.1",
-    "@nestjs/core": "9.4.0",
-    "@nestjs/event-emitter": "^1.4.1",
-    "@nestjs/platform-express": "9.4.0",
-    "@nestjs/sequelize": "9.0.2",
+    "@nestjs/common": "^9.4.3",
+    "@nestjs/config": "^2.3.4",
+    "@nestjs/core": "^9.4.3",
+    "@nestjs/event-emitter": "^1.4.2",
+    "@nestjs/platform-express": "^9.4.3",
+    "@nestjs/sequelize": "^9.0.2",
     "@nestjs/swagger": "^6.3.0",
-    "dotenv": "^16.0.0",
-    "fs-extra": "^10.0.0",
-    "pg": "^8.6.0",
+    "dotenv": "^16.3.1",
+    "fs-extra": "^10.1.0",
+    "pg": "^8.11.3",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1",
-    "sequelize": "6.31.0",
-    "sequelize-typescript": "2.1.5",
-    "swagger-ui-express": "^4.6.2",
-    "ts-node": "^10.8.0",
+    "sequelize": "^6.33.0",
+    "sequelize-typescript": "^2.1.5",
+    "swagger-ui-express": "^4.6.3",
+    "ts-node": "^10.9.1",
     "typescript": "4.3.5",
-    "umzug": "3.2.1"
+    "umzug": "^3.3.1"
   },
   "devDependencies": {
-    "@nestjs/cli": "^7.6.0",
-    "@nestjs/schematics": "^7.3.0",
-    "@nestjs/testing": "7.6.15",
-    "@types/express": "^4.17.11",
-    "@types/jest": "^26.0.22",
-    "@types/node": "^16.11.26",
-    "@types/sequelize": "4.28.9",
-    "@types/supertest": "^2.0.10",
-    "@types/ws": "^7.4.6",
-    "@typescript-eslint/eslint-plugin": "^5.57.1",
+    "@nestjs/cli": "^9.5.0",
+    "@nestjs/schematics": "^9.2.0",
+    "@nestjs/testing": "9.4.3",
+    "@types/express": "^4.17.20",
+    "@types/jest": "^29.5.6",
+    "@types/node": "^18.18.6",
+    "@types/pg": "^8.10.7",
+    "@types/sequelize": "4.28.17",
+    "@types/supertest": "^2.0.15",
+    "@types/ws": "^7.4.7",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
     "@typescript-eslint/parser": "5.57.1",
-    "eslint": "^8.12.0",
-    "eslint-config-prettier": "^8.1.0",
-    "eslint-plugin-prettier": "^4.0.0",
-    "jest": "^26.6.3",
-    "prettier": "^2.4.1",
+    "eslint": "^8.52.0",
+    "eslint-config-prettier": "^8.10.0",
+    "eslint-plugin-prettier": "^4.2.1",
+    "jest": "^29.7.0",
+    "prettier": "^2.8.8",
     "rimraf": "^5.0.0",
-    "supertest": "^6.1.3",
-    "ts-jest": "^26.5.4",
-    "ts-loader": "^8.0.18",
-    "tsconfig-paths": "^3.9.0"
-  },
-  "jest": {
-    "moduleFileExtensions": [
-      "js",
-      "json",
-      "ts"
-    ],
-    "rootDir": "src",
-    "testRegex": ".*\\.spec\\.ts$",
-    "transform": {
-      "^.+\\.(t|j)s$": "ts-jest"
-    },
-    "collectCoverageFrom": [
-      "**/*.(t|j)s"
-    ],
-    "coverageDirectory": "../coverage",
-    "testEnvironment": "node"
+    "supertest": "^6.3.3",
+    "ts-jest": "^29.1.1",
+    "ts-loader": "^8.4.0",
+    "tsconfig-paths": "^3.14.2"
   }
 }

packages/api/src/db/index.ts

@@ -9,8 +9,7 @@ import { Umzug, SequelizeStorage } from "umzug";
 import { ADMIN_CLIENT_CONFIG, USER_DB_CONFIG } from "./config";
 
 const MIGRATIONS_DIR = path.resolve(__dirname, "migrations");
-const APP_DB_PASSWORD = environment.APP_DB_PASSWORD;
-const APP_DB_USER = environment.APP_DB_USER;
+const { APP_DB_USER, APP_DB_PASSWORD } = environment;
 
 export class DBInstance {
   constructor(private dbName = environment.APP_DB_NAME) {}
@@ -28,7 +27,12 @@ export class DBInstance {
       await this.setupUsers(adminClient);
       await adminClient.end();
       // create additional client on target db to perform migrations
-      const migrationClient = new Sequelize({ ...USER_DB_CONFIG, database: this.dbName });
+      const migrationClient = new Sequelize({
+        ...USER_DB_CONFIG,
+        database: this.dbName,
+        // disable verbose migration logs in test
+        logging: process.env.NODE_ENV === "test" ? false : true,
+      });
       await this.runMigrations(migrationClient);
       await migrationClient.close();
     } catch (error) {
@@ -77,7 +81,8 @@ export class DBInstance {
       },
       context: client.getQueryInterface(),
       storage: new SequelizeStorage({ sequelize: client }),
-      logger: console,
+      // Limit migrator logs only to error and warn
+      logger: { error: console.error, warn: console.warn, info: () => null, debug: () => null },
     });
     const pending = await migrator.pending();
     console.log("[Migrations] pending", pending);

packages/api/src/endpoints/app_users/app_user.controller.ts

@@ -4,15 +4,11 @@ import { AppUser } from "./app_user.model";
 import { AppUsersService } from "./app_user.service";
 import { ApiBody, ApiOperation, ApiParam, ApiResponse, ApiTags } from "@nestjs/swagger";
 import { DeploymentHeaders } from "src/modules/deployment.decorators";
-import { DeploymentService } from "src/modules/deployment.service";
 
 @ApiTags("Users")
 @Controller("app_users")
 export class AppUsersController {
-  constructor(
-    private readonly appUsersService: AppUsersService,
-    private dbService: DeploymentService
-  ) {}
+  constructor(private readonly appUsersService: AppUsersService) {}
 
   // @Post()
   // create(@Body() createUserDto: CreateAppUserDto): Promise<AppUser> {
@@ -23,7 +19,7 @@ export class AppUsersController {
   @ApiOperation({ summary: "List users" })
   @DeploymentHeaders()
   findAll() {
-    return this.dbService.model(AppUser).findAll();
+    return this.appUsersService.model.findAll();
   }
 
   // @Get(":app_user_id")

packages/api/src/endpoints/app_users/app_user.service.ts

@@ -27,15 +27,17 @@ export class AppUsersService {
     });
   }
 
-  getUserData(app_user_id: string): any {
-    return { app_user_id, contact_fields: { example: "hellow" } };
-  }
   async setUserData(app_user_id: string, data: ContactFieldDto) {
-    let user = await this.model.findOne({ where: { app_user_id } });
+    const user = await this.model.findOne({ where: { app_user_id } });
     if (!user) {
-      user = new AppUser();
+      return await this.model.create({ ...data, app_user_id });
+      // HACK - instantiating as appUser object does not generate correct sequelize queryInterface
+      // Alternatively could try to manually re bind
+
+      // user = new AppUser();
       user.app_user_id = app_user_id;
     }
+    // await this.model.upsert({ ...user, ...data, app_user_id });
     return user.update({ ...data, app_user_id });
   }
 }

packages/api/src/environment/index.ts

@@ -1,4 +1,5 @@
 import * as dotenv from "dotenv";
+import { resolve } from "path";
 
 /** Environment variables set from `.env` file */
 interface IParsedEnvironment {
@@ -23,7 +24,18 @@ interface IEnvironment extends IParsedEnvironment {
 let parsedEnv: IParsedEnvironment = {} as any;
 
 try {
-  const { error, parsed } = dotenv.config();
+  const { NODE_ENV } = process.env;
+
+  let envFilePath = resolve(__dirname, "../../.env");
+  if (NODE_ENV === "test") {
+    envFilePath = resolve(__dirname, "../../test/.test.env");
+  }
+  console.log(`Loading environment: [${NODE_ENV}]\n`, envFilePath);
+  const { error, parsed } = dotenv.config({
+    override: false,
+    debug: NODE_ENV === "test" ? true : false,
+    path: envFilePath,
+  });
   if (parsed) {
     parsedEnv = parsed as any;
   }
@@ -37,8 +49,8 @@ try {
 }
 
 const environment: IEnvironment = {
-  ...process.env,
   ...parsedEnv,
+  ...process.env,
   production: parsedEnv.NODE_ENV !== "development",
 };
 export { environment };

packages/api/src/modules/deployment.middleware.ts

@@ -1,36 +1,22 @@
-import { Injectable, NestMiddleware } from "@nestjs/common";
+import { Injectable, NestMiddleware, Scope } from "@nestjs/common";
 import { NextFunction, Request } from "express";
 import { environment } from "src/environment";
 
 import { DeploymentService } from "./deployment.service";
 
 /**
- * Intercept all requests and use request headers to set the active connection to use
- * as part of db operations
+ * Intercept all requests and use request headers ensure target DB has been bootstrapped
  */
 
-@Injectable()
+@Injectable({ scope: Scope.REQUEST })
 export class DeploymentMiddleware implements NestMiddleware {
   constructor(private service: DeploymentService) {}
 
   async use(req: Request, res: Response, next: NextFunction) {
-    // ensure all requests include a db_name parameter
-    if (!req.headers["x-deployment-db-name"]) {
-      req.headers["x-deployment-db-name"] = environment.APP_DB_NAME;
-    }
-    const dbName = req.headers["x-deployment-db-name"] as string;
+    const dbName = (req.headers["x-deployment-db-name"] as string) || environment.APP_DB_NAME;
 
     // Assign the correct sequelize client based on target deployment db
-    // Additionally update request itself to allow access to client
-    // either via service or request handler
-    await this.service.setDeploymentDB(dbName);
-
-    /**
-     * DEPRECATED CC 2023-29-04 - prefer use service instead of parsing from body
-     * client returned from setDeploymentDB, also requires removing from any method
-     * that tries to convert full body to json
-     */
-    // req.body.sequelize = client;
+    await this.service.ensureDeploymentClient(dbName);
 
     next();
   }

packages/api/src/modules/deployment.module.ts

@@ -1,10 +1,11 @@
 import { Global, MiddlewareConsumer, Module, NestModule, RequestMethod } from "@nestjs/common";
+import { ConnectionManagerService } from "src/services/connection-manager";
 import { DeploymentMiddleware } from "./deployment.middleware";
 import { DeploymentService } from "./deployment.service";
 
 @Global()
 @Module({
-  providers: [DeploymentService, DeploymentMiddleware],
+  providers: [DeploymentService, DeploymentMiddleware, ConnectionManagerService],
   exports: [DeploymentService, DeploymentMiddleware],
 })
 /**