TechDebt: API: Environment Variables Validation (#1425)

- ENV schema
- loadEnvironmentVariables()

api/src/app.ts

@@ -13,20 +13,24 @@ import {
 } from './middleware/critterbase-proxy';
 import { rootAPIDoc } from './openapi/root-api-doc';
 import { authenticateRequest, authenticateRequestOptional } from './request-handlers/security/authentication';
+import { loadEvironmentVariables } from './utils/env-config';
 import { scanFileForVirus } from './utils/file-utils';
 import { getLogger } from './utils/logger';
 
+// Load and validate the environment variables
+loadEvironmentVariables();
+
 const defaultLog = getLogger('app');
 
 const HOST = process.env.API_HOST;
-const PORT = Number(process.env.API_PORT);
+const PORT = process.env.API_PORT;
 
 // Max size of the body of the request (bytes)
-const MAX_REQ_BODY_SIZE = Number(process.env.MAX_REQ_BODY_SIZE) || 52428800;
+const MAX_REQ_BODY_SIZE = process.env.MAX_REQ_BODY_SIZE;
 // Max number of files in a single request
-const MAX_UPLOAD_NUM_FILES = Number(process.env.MAX_UPLOAD_NUM_FILES) || 10;
+const MAX_UPLOAD_NUM_FILES = process.env.MAX_UPLOAD_NUM_FILES;
 // Max size of a single file (bytes)
-const MAX_UPLOAD_FILE_SIZE = Number(process.env.MAX_UPLOAD_FILE_SIZE) || 52428800;
+const MAX_UPLOAD_FILE_SIZE = process.env.MAX_UPLOAD_FILE_SIZE;
 
 // Get initial express app
 const app: express.Express = express();

api/src/database/db.ts

@@ -18,14 +18,14 @@ import { asyncErrorWrapper, getGenericizedKeycloakUserInformation, syncErrorWrap
 const defaultLog = getLogger('database/db');
 
 const getDbHost = () => process.env.DB_HOST;
-const getDbPort = () => Number(process.env.DB_PORT);
+const getDbPort = () => process.env.DB_PORT;
 const getDbUsername = () => process.env.DB_USER_API;
 const getDbPassword = () => process.env.DB_USER_API_PASS;
 const getDbDatabase = () => process.env.DB_DATABASE;
 
-const DB_POOL_SIZE: number = Number(process.env.DB_POOL_SIZE) || 20;
-const DB_CONNECTION_TIMEOUT: number = Number(process.env.DB_CONNECTION_TIMEOUT) || 0;
-const DB_IDLE_TIMEOUT: number = Number(process.env.DB_IDLE_TIMEOUT) || 10000;
+const DB_POOL_SIZE = 20;
+const DB_CONNECTION_TIMEOUT = 0;
+const DB_IDLE_TIMEOUT = 10000;
 
 export const DB_CLIENT = 'pg';
 

api/src/utils/env-config.ts

@@ -0,0 +1,122 @@
+import { z } from 'zod';
+import { getLogger } from './logger';
+
+const defaultLog = getLogger('src/utils/env-config.ts');
+
+const ZodEnvString = z.string().trim().min(1, { message: 'Required' }); // '' or ' ' are invalid
+const ZodEnvNumber = z.coerce.number().min(1, { message: 'Required and must be a positive value.' }); // -1 is invalid
+
+// Schema for environment configuration
+export const EnvSchema = z.object({
+  // Environment
+  NODE_ENV: z.enum(['development', 'test', 'production']),
+  NODE_OPTIONS: ZodEnvString,
+  TZ: z.literal('America/Vancouver'),
+
+  // API server
+  API_HOST: ZodEnvString,
+  API_PORT: ZodEnvNumber,
+
+  // Database
+  DB_HOST: ZodEnvString,
+  DB_PORT: ZodEnvNumber,
+  DB_USER_API: ZodEnvString,
+  DB_USER_API_PASS: ZodEnvString,
+  DB_DATABASE: ZodEnvString,
+
+  // Keycloak
+  KEYCLOAK_HOST: ZodEnvString,
+  KEYCLOAK_REALM: ZodEnvString,
+  KEYCLOAK_ADMIN_USERNAME: ZodEnvString,
+  KEYCLOAK_ADMIN_PASSWORD: ZodEnvString,
+  KEYCLOAK_API_TOKEN_URL: ZodEnvString,
+  KEYCLOAK_API_CLIENT_ID: ZodEnvString,
+  KEYCLOAK_API_CLIENT_SECRET: ZodEnvString,
+  KEYCLOAK_API_HOST: ZodEnvString,
+  KEYCLOAK_API_ENVIRONMENT: ZodEnvString,
+
+  // Logging
+  LOG_LEVEL: z.enum(['silent', 'error', 'warn', 'info', 'debug', 'silly']),
+  LOG_LEVEL_FILE: z.enum(['silent', 'error', 'warn', 'info', 'debug', 'silly']),
+  LOG_FILE_DIR: ZodEnvString,
+  LOG_FILE_NAME: ZodEnvString,
+  LOG_FILE_DATE_PATTERN: ZodEnvString,
+  LOG_FILE_MAX_SIZE: ZodEnvString,
+  LOG_FILE_MAX_FILES: ZodEnvString,
+
+  // Validation
+  API_RESPONSE_VALIDATION_ENABLED: z.enum(['true', 'false']),
+  DATABASE_RESPONSE_VALIDATION_ENABLED: z.enum(['true', 'false']),
+
+  // File upload limits
+  MAX_REQ_BODY_SIZE: ZodEnvNumber,
+  MAX_UPLOAD_NUM_FILES: ZodEnvNumber,
+  MAX_UPLOAD_FILE_SIZE: ZodEnvNumber,
+
+  // External Services
+  CB_API_HOST: ZodEnvString,
+  APP_HOST: ZodEnvString,
+
+  // Biohub
+  BACKBONE_INTERNAL_API_HOST: ZodEnvString,
+  BACKBONE_PUBLIC_API_HOST: ZodEnvString,
+  BACKBONE_INTAKE_PATH: ZodEnvString,
+  BACKBONE_ARTIFACT_INTAKE_PATH: ZodEnvString,
+  BIOHUB_TAXON_PATH: ZodEnvString,
+  BIOHUB_TAXON_TSN_PATH: ZodEnvString,
+
+  // Object Store
+  OBJECT_STORE_URL: ZodEnvString,
+  OBJECT_STORE_ACCESS_KEY_ID: ZodEnvString,
+  OBJECT_STORE_SECRET_KEY_ID: ZodEnvString,
+  OBJECT_STORE_BUCKET_NAME: ZodEnvString,
+  S3_KEY_PREFIX: ZodEnvString,
+
+  // GCNotify
+  GCNOTIFY_SECRET_API_KEY: ZodEnvString,
+  GCNOTIFY_ADMIN_EMAIL: ZodEnvString,
+  GCNOTIFY_ONBOARDING_REQUEST_EMAIL_TEMPLATE: z.string().uuid(),
+  GCNOTIFY_ONBOARDING_REQUEST_SMS_TEMPLATE: z.string().uuid(),
+  GCNOTIFY_REQUEST_RESUBMIT_TEMPLATE: z.string().uuid(),
+  GCNOTIFY_EMAIL_URL: ZodEnvString,
+  GCNOTIFY_SMS_URL: ZodEnvString,
+
+  // ClamAV
+  CLAMAV_PORT: ZodEnvNumber,
+  CLAMAV_HOST: ZodEnvString,
+  ENABLE_FILE_VIRUS_SCAN: z.enum(['true', 'false']),
+
+  // Extra
+  FEATURE_FLAGS: z.string().trim().optional() // flagA,flagB,flagC
+});
+
+type Env = z.infer<typeof EnvSchema>;
+
+/**
+ * Load Environment Variables and validate them against the Zod schema.
+ *
+ * @returns {*} {Env} Validated environment variables
+ */
+export const loadEvironmentVariables = (): Env => {
+  const parsed = EnvSchema.safeParse(process.env);
+
+  if (!parsed.success) {
+    defaultLog.error({
+      label: 'loadEvironmentVariables',
+      message: 'Environment variables validation check failed',
+      errors: parsed.error.flatten().fieldErrors
+    });
+
+    process.exit(1);
+  }
+
+  return parsed.data;
+};
+
+// Extend NodeJS ProcessEnv to include the EnvSchema
+declare global {
+  // eslint-disable-next-line @typescript-eslint/no-namespace
+  namespace NodeJS {
+    interface ProcessEnv extends Env {}
+  }
+}

api/src/utils/file-utils.test.ts

@@ -136,8 +136,7 @@ describe('getS3HostUrl', () => {
   });
 
   it('should yield a default S3 host url', () => {
-    delete process.env.OBJECT_STORE_URL;
-    delete process.env.OBJECT_STORE_BUCKET_NAME;
+    Object.assign(process.env, { OBJECT_STORE_URL: undefined, OBJECT_STORE_BUCKET_NAME: undefined });
 
     const result = getS3HostUrl();
 
@@ -193,7 +192,7 @@ describe('_getClamAvScanner', () => {
   it('should return a clamAv scanner client', () => {
     process.env.ENABLE_FILE_VIRUS_SCAN = 'true';
     process.env.CLAMAV_HOST = 'host';
-    process.env.CLAMAV_PORT = '1111';
+    process.env.CLAMAV_PORT = 1111;
 
     const result = _getClamAvScanner();
     expect(result).to.not.be.null;
@@ -215,7 +214,7 @@ describe('_getObjectStoreBucketName', () => {
   });
 
   it('should return its default value', () => {
-    delete process.env.OBJECT_STORE_BUCKET_NAME;
+    Object.assign(process.env, { OBJECT_STORE_BUCKET_NAME: undefined });
 
     const result = _getObjectStoreBucketName();
     expect(result).to.equal('');
@@ -251,7 +250,7 @@ describe('_getObjectStoreUrl', () => {
   });
 
   it('should return its default value', () => {
-    delete process.env.OBJECT_STORE_URL;
+    Object.assign(process.env, { OBJECT_STORE_URL: undefined });
 
     const result = _getObjectStoreUrl();
     expect(result).to.equal('https://nrs.objectstore.gov.bc.ca');
@@ -273,7 +272,7 @@ describe('getS3KeyPrefix', () => {
   });
 
   it('should return its default value', () => {
-    delete process.env.S3_KEY_PREFIX;
+    Object.assign(process.env, { S3_KEY_PREFIX: undefined });
 
     const result = getS3KeyPrefix();
     expect(result).to.equal('sims');

api/src/utils/file-utils.ts

@@ -31,7 +31,7 @@ export const _getClamAvScanner = async (): Promise<NodeClam> => {
   return new NodeClam().init({
     clamdscan: {
       host: process.env.CLAMAV_HOST,
-      port: Number(process.env.CLAMAV_PORT)
+      port: process.env.CLAMAV_PORT
     }
   });
 };

compose.yml

@@ -50,12 +50,6 @@ services:
       - DB_PORT=5432
       - DB_DATABASE=${DB_DATABASE}
       - DB_SCHEMA=${DB_SCHEMA}
-      # Seed
-      - PROJECT_SEEDER_USER_IDENTIFIER=${PROJECT_SEEDER_USER_IDENTIFIER}
-      - NUM_SEED_PROJECTS=${NUM_SEED_PROJECTS}
-      - NUM_SEED_SURVEYS_PER_PROJECT=${NUM_SEED_SURVEYS_PER_PROJECT}
-      - NUM_SEED_OBSERVATIONS_PER_SURVEY=${NUM_SEED_OBSERVATIONS_PER_SURVEY}
-      - NUM_SEED_SUBCOUNTS_PER_OBSERVATION=${NUM_SEED_SUBCOUNTS_PER_OBSERVATION}
       # Keycloak
       - KEYCLOAK_HOST=${KEYCLOAK_HOST}
       - KEYCLOAK_REALM=${KEYCLOAK_REALM}