SIMSBIOHUB-600: Handle Keycloak Auto-Renew Error, Write Logs to File

api/.gitignore

@@ -12,6 +12,9 @@ dist
 # Testing
 coverage
 
+# persistent storage
+data
+
 # SonarQube
 .sonarqube
 

api/.pipeline/config.js

@@ -91,7 +91,14 @@ const phases = {
     tz: config.timezone.api,
     sso: config.sso.dev,
     featureFlags: '',
-    logLevel: 'info',
+    logLevel: (isStaticDeployment && 'info') || 'debug',
+    logLevelFile: (isStaticDeployment && 'debug') || 'debug',
+    logFileDir: 'data/logs',
+    logFileName: 'sims-api-%DATE%.log',
+    logFileDatePattern: 'YYYY-MM-DD-HH',
+    logFileMaxSize: '50m',
+    logFileMaxFiles: (isStaticDeployment && '10') || '2',
+    volumeCapacity: (isStaticDeployment && '500Mi') || '100Mi',
     apiResponseValidationEnabled: true,
     databaseResponseValidationEnabled: true,
     nodeOptions: '--max_old_space_size=3000', // 75% of memoryLimit (bytes)
@@ -126,8 +133,15 @@ const phases = {
     s3KeyPrefix: 'sims',
     tz: config.timezone.api,
     sso: config.sso.test,
-    logLevel: 'info',
     featureFlags: '',
+    logLevel: 'warn',
+    logLevelFile: 'debug',
+    logFileDir: 'data/logs',
+    logFileName: 'sims-api-%DATE%.log',
+    logFileDatePattern: 'YYYY-MM-DD-HH',
+    logFileMaxSize: '50m',
+    logFileMaxFiles: '10',
+    volumeCapacity: '500Mi',
     apiResponseValidationEnabled: true,
     databaseResponseValidationEnabled: true,
     nodeOptions: '--max_old_space_size=3000', // 75% of memoryLimit (bytes)
@@ -163,7 +177,14 @@ const phases = {
     tz: config.timezone.api,
     sso: config.sso.prod,
     featureFlags: 'API_FF_SUBMIT_BIOHUB',
-    logLevel: 'warn',
+    logLevel: 'silent',
+    logLevelFile: 'debug',
+    logFileDir: 'data/logs',
+    logFileName: 'sims-api-%DATE%.log',
+    logFileDatePattern: 'YYYY-MM-DD-HH',
+    logFileMaxSize: '50m',
+    logFileMaxFiles: '10',
+    volumeCapacity: '500Mi',
     apiResponseValidationEnabled: false,
     databaseResponseValidationEnabled: false,
     nodeOptions: '--max_old_space_size=6000', // 75% of memoryLimit (bytes)

api/.pipeline/lib/api.deploy.js

@@ -34,6 +34,8 @@ const apiDeploy = async (settings) => {
         // Node
         NODE_ENV: phases[phase].nodeEnv,
         NODE_OPTIONS: phases[phase].nodeOptions,
+        // Persistent Volume
+        VOLUME_CAPACITY: phases[phase].volumeCapacity,
         // BioHub Platform (aka: Backbone)
         BACKBONE_INTERNAL_API_HOST: phases[phase].backboneInternalApiHost,
         BACKBONE_INTAKE_PATH: phases[phase].backboneIntakePath,
@@ -65,6 +67,13 @@ const apiDeploy = async (settings) => {
         KEYCLOAK_API_ENVIRONMENT: phases[phase].sso.cssApi.cssApiEnvironment,
         // Log Level
         LOG_LEVEL: phases[phase].logLevel,
+        LOG_LEVEL_FILE: phases[phase].logLevelFile,
+        LOG_FILE_DIR: phases[phase].logFileDir,
+        LOG_FILE_NAME: phases[phase].logFileName,
+        LOG_FILE_DATE_PATTERN: phases[phase].logFileDatePattern,
+        LOG_FILE_MAX_SIZE: phases[phase].logFileMaxSize,
+        LOG_FILE_MAX_FILES: phases[phase].logFileMaxFiles,
+        // Api Validation
         API_RESPONSE_VALIDATION_ENABLED: phases[phase].apiResponseValidationEnabled,
         DATABASE_RESPONSE_VALIDATION_ENABLED: phases[phase].databaseResponseValidationEnabled,
         // Feature Flags

api/.pipeline/templates/api.dc.yaml

@@ -32,6 +32,12 @@ parameters:
   - name: API_PORT_DEFAULT_NAME
     description: Api default port name
     value: '6100-tcp'
+  # Volume (for API logs and other persistent data)
+  - description: Volume space available for data, e.g. 512Mi, 2Gi.
+    displayName: Volume Capacity
+    name: VOLUME_CAPACITY
+    required: true
+    value: '500Mi'
   # Clamav
   - name: ENABLE_FILE_VIRUS_SCAN
     value: 'true'
@@ -115,9 +121,29 @@ parameters:
     description: S3 key optional prefix
     required: false
     value: 'sims'
-  # Logging and Validation
+  # Logging
   - name: LOG_LEVEL
-    value: 'warn'
+    value: 'silent'
+    description: Log level for logs written to the console (console transport)
+  - name: LOG_LEVEL_FILE
+    value: 'debug'
+    description: Log level for logs written to log files (file transport)
+  - name: LOG_FILE_DIR
+    value: data
+    description: Directory where log files are stored
+  - name: LOG_FILE_NAME
+    value: sims-api-%DATE%.log
+    description: Name of the log file
+  - name: LOG_FILE_DATE_PATTERN
+    value: YYYY-MM-DD-HH
+    description: Date pattern for log the files
+  - name: LOG_FILE_MAX_SIZE
+    value: 50m
+    description: Maximum size an individual log file can reach before a new file is created
+  - name: LOG_FILE_MAX_FILES
+    value: '10'
+    description: Either the maximum number of log files to keep (10) or the maximum number of days to keep log files (10d)
+  # Api Validation
   - name: API_RESPONSE_VALIDATION_ENABLED
     value: 'false'
   - name: DATABASE_RESPONSE_VALIDATION_ENABLED
@@ -160,8 +186,8 @@ parameters:
   - name: REPLICAS_MAX
     value: '1'
 objects:
-  - apiVersion: image.openshift.io/v1
-    kind: ImageStream
+  - kind: ImageStream
+    apiVersion: image.openshift.io/v1
     metadata:
       annotations:
         description: Nodejs Runtime Image
@@ -175,6 +201,17 @@ objects:
     status:
       dockerImageRepository: null
 
+  - kind: PersistentVolumeClaim
+    apiVersion: v1
+    metadata:
+      name: ${NAME}${SUFFIX}
+    spec:
+      accessModes:
+        - ReadWriteMany
+      resources:
+        requests:
+          storage: '${VOLUME_CAPACITY}'
+
   - kind: DeploymentConfig
     apiVersion: apps.openshift.io/v1
     metadata:
@@ -320,9 +357,22 @@ objects:
                     secretKeyRef:
                       key: object_store_bucket_name
                       name: ${OBJECT_STORE_SECRETS}
-                # Logging and Validation
+                # Logging
                 - name: LOG_LEVEL
                   value: ${LOG_LEVEL}
+                - name: LOG_LEVEL_FILE
+                  value: ${LOG_LEVEL_FILE}
+                - name: LOG_FILE_DIR
+                  value: ${LOG_FILE_DIR}
+                - name: LOG_FILE_NAME
+                  value: ${LOG_FILE_NAME}
+                - name: LOG_FILE_DATE_PATTERN
+                  value: ${LOG_FILE_DATE_PATTERN}
+                - name: LOG_FILE_MAX_SIZE
+                  value: ${LOG_FILE_MAX_SIZE}
+                - name: LOG_FILE_MAX_FILES
+                  value: ${LOG_FILE_MAX_FILES}
+                # Api Validation
                 - name: API_RESPONSE_VALIDATION_ENABLED
                   value: ${API_RESPONSE_VALIDATION_ENABLED}
                 - name: DATABASE_RESPONSE_VALIDATION_ENABLED
@@ -380,16 +430,17 @@ objects:
               terminationMessagePath: /dev/termination-log
               terminationMessagePolicy: File
               volumeMounts:
-                - mountPath: /opt/app-root/app
-                  name: ${NAME}${SUFFIX}
+                - name: ${NAME}${SUFFIX}
+                  mountPath: /opt/app-root/src/data
           dnsPolicy: ClusterFirst
           restartPolicy: Always
           schedulerName: default-scheduler
           securityContext: {}
           terminationGracePeriodSeconds: 30
           volumes:
-            - emptyDir: {}
-              name: ${NAME}${SUFFIX}
+            - name: ${NAME}${SUFFIX}
+              persistentVolumeClaim:
+                claimName: ${NAME}${SUFFIX}
       test: false
       triggers:
         - imageChangeParams:
@@ -421,8 +472,8 @@ objects:
       name: ${NAME}${SUFFIX}
     type: Opaque
 
-  - apiVersion: v1
-    kind: Service
+  - kind: Service
+    apiVersion: v1
     metadata:
       annotations: null
       labels: {}

api/README.md

@@ -84,12 +84,10 @@ A centralized logger has been created (see `api/utils/logger.ts`).
 
 ## Logger configuration
 
-The loggers log level can be configured via an environment variable: `LOG_LEVEL`
+The loggers log level can be configured via environment variables: `LOG_LEVEL` and `LOG_LEVEL_FILE`
 
 Set this variable to one of: `silent`, `error`, `warn`, `info`, `debug`, `silly`
 
-Default value: `info`
-
 ## Instantiating the logger in your class/file
 
 ```

api/package.json

@@ -62,6 +62,7 @@
     "utm": "^1.1.1",
     "uuid": "^8.3.2",
     "winston": "^3.3.3",
+    "winston-daily-rotate-file": "^5.0.0",
     "xlsx": "https://cdn.sheetjs.com/xlsx-0.19.3/xlsx-0.19.3.tgz",
     "zod": "^3.23.0"
   },

api/src/paths/logger.test.ts

@@ -1,34 +1,19 @@
 import { expect } from 'chai';
 import { describe } from 'mocha';
-import { HTTPError } from '../errors/http-error';
 import { getRequestHandlerMocks } from '../__mocks__/db';
 import * as logger from './logger';
 
 describe('logger', () => {
   describe('updateLoggerLevel', () => {
-    it('should throw a 400 error when `level` query param is missing', async () => {
-      const requestHandler = logger.updateLoggerLevel();
-
-      const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
-
-      mockReq.query = {};
-
-      try {
-        await requestHandler(mockReq, mockRes, mockNext);
-
-        expect.fail();
-      } catch (error) {
-        expect((error as HTTPError).status).to.equal(400);
-        expect((error as HTTPError).message).to.equal('Missing required query param `level`');
-      }
-    });
-
     it('should return 200 on success', async () => {
       const requestHandler = logger.updateLoggerLevel();
 
       const { mockReq, mockRes, mockNext } = getRequestHandlerMocks();
 
-      mockReq.query = { level: 'info' };
+      mockReq.query = {
+        logLevel: 'info',
+        logLevelFile: 'debug'
+      };
 
       await requestHandler(mockReq, mockRes, mockNext);