- Added logs throttling and better redis logging.

magieno · May 23, 2024 · 9eefe4c · 9eefe4c
1 parent e798ddb
commit 9eefe4c
Show file tree

Hide file tree

Showing 3 changed files with 48 additions and 6 deletions.
diff --git a/packages/logging/src/loggers/console.logger.ts b/packages/logging/src/loggers/console.logger.ts
@@ -24,6 +24,12 @@ export class ConsoleLogger extends BaseLogger implements LoggerInterface {
    */
   public readableStream?: Readable;
 
+  private currentSecond = -1;
+
+  private numberOfLogsInThisSecond = 0;
+
+  private currentlyThrottlingLogs = false;
+
   /**
    * The ConsoleLogger outputs the logs in the console.
    * @param numberOfStackedLogs The number of logs to keep in the stack and to print once a log with a high enough severity arrives.
@@ -41,6 +47,7 @@ export class ConsoleLogger extends BaseLogger implements LoggerInterface {
    * We often do not need to go to the bottom layer of an object, so we can truncate at a certain depth.
    * @param isActivated Whether or not this particular logger is activated and should output logs.
    * @param outputMode The output mode, that the logger should use.
+   * @param maximumLogsPerSecond The maximum numner of logs per second that can be outputted
    */
   public constructor(@inject("%pristine.logging.numberOfStackedLogs%") numberOfStackedLogs: number,
                      @inject("%pristine.logging.logSeverityLevelConfiguration%") logSeverityLevelConfiguration: number,
@@ -51,6 +58,7 @@ export class ConsoleLogger extends BaseLogger implements LoggerInterface {
                      @inject("%pristine.logging.logCriticalDepthConfiguration%") logCriticalDepthConfiguration: number,
                      @inject("%pristine.logging.consoleLoggerActivated%") isActivated: boolean,
                      @inject("%pristine.logging.consoleLoggerOutputMode%") outputMode: OutputModeEnum,
+                     @inject("%pristine.logging.maximumLogsPerSecond%") private readonly maximumLogsPerSecond: number,
                      ) {
     super(numberOfStackedLogs,
         logSeverityLevelConfiguration,
@@ -90,6 +98,23 @@ export class ConsoleLogger extends BaseLogger implements LoggerInterface {
     }
   }
 
+  private shouldThrottleLogs() {
+    const now = new Date().getSeconds();
+    if (this.currentSecond !== now) {
+      this.currentSecond = now;
+      this.numberOfLogsInThisSecond = 1;
+      this.currentlyThrottlingLogs = false;
+    } else {
+      this.numberOfLogsInThisSecond++;
+      if (this.numberOfLogsInThisSecond > this.maximumLogsPerSecond) {
+        this.currentlyThrottlingLogs = true;
+        console.error(`Throttling the logs as we are outputting too many logs (${this.maximumLogsPerSecond}) per second.`);
+      }
+    }
+
+    return this.currentlyThrottlingLogs;
+  }
+
   /**
    * Outputs the log in the console.
    * @param log The log to be outputted
@@ -98,6 +123,10 @@ export class ConsoleLogger extends BaseLogger implements LoggerInterface {
   protected log(log: LogModel): void {
     const outputLog = this.getFormattedOutputLog(log);
 
+    if(this.shouldThrottleLogs()) {
+      return;
+    }
+
     switch (log.severity) {
       case SeverityEnum.Debug:
         console.debug(outputLog);

diff --git a/packages/logging/src/logging.module.ts b/packages/logging/src/logging.module.ts
@@ -183,6 +183,19 @@ export const LoggingModule: ModuleInterface = {
                 new BooleanResolver(new EnvironmentVariableResolver("PRISTINE_LOGGING_ACTIVATE_DIAGNOSTICS")),
             ]
         },
+        /**
+         * Whether or not the diagnostic should be activated.
+         * When activated, the stack trace and other diagnostic information will be added to the logs.
+         * This is an intensive process and can dramatically reduce the performance of the code.
+         */
+        {
+            parameterName: LoggingModuleKeyname + ".maximumLogsPerSecond",
+            defaultValue: 100,
+            isRequired: false,
+            defaultResolvers: [
+                new NumberResolver(new EnvironmentVariableResolver("PRISTINE_LOGGING_MAXIMUM_LOGS_PER_SECOND")),
+            ]
+        },
     ],
     importModules: [
         CommonModule,

diff --git a/packages/redis/src/clients/redis.client.ts b/packages/redis/src/clients/redis.client.ts
@@ -67,7 +67,7 @@ export class RedisClient implements RedisClientInterface {
                 response = client.SET(redisKey, value);
             }
 
-            this.logHandler.debug("Redis response", {response}, RedisModuleKeyname)
+            this.logHandler.debug("Redis - `set` response", {response, table, key, value, ttl}, RedisModuleKeyname)
         } catch (error) {
             throw new RedisError("Error setting in redis", error as Error, table, key, redisKey);
         }
@@ -86,7 +86,7 @@ export class RedisClient implements RedisClientInterface {
         try {
             const response = await client.RPUSH(redisKey, ...value);
 
-            this.logHandler.debug("Redis response", {response}, RedisModuleKeyname)
+            this.logHandler.debug("Redis - `setList` response", {response, table, key, value, ttl}, RedisModuleKeyname)
         } catch (error) {
             throw new RedisError("Error setting in redis", error as Error, table, key, redisKey);
         }
@@ -104,7 +104,7 @@ export class RedisClient implements RedisClientInterface {
         try {
             const response = await client.GET(redisKey);
 
-            this.logHandler.debug("Redis response", {response}, RedisModuleKeyname)
+            this.logHandler.debug("Redis - `get` response", {response, table, key}, RedisModuleKeyname)
 
             return response;
         } catch (error) {
@@ -126,7 +126,7 @@ export class RedisClient implements RedisClientInterface {
         try {
             const response = await client.LRANGE(redisKey, start, stop);
 
-            this.logHandler.debug("Redis response", {response}, RedisModuleKeyname)
+            this.logHandler.debug("Redis `getList` response", {response, table, key, start, stop}, RedisModuleKeyname)
 
             return response;
         } catch (error) {
@@ -146,7 +146,7 @@ export class RedisClient implements RedisClientInterface {
         try {
             const response = await client.DEL(redisKey);
 
-            this.logHandler.debug("Redis response", {response}, RedisModuleKeyname)
+            this.logHandler.debug("Redis - `remove` response", {response, table, key}, RedisModuleKeyname)
 
         } catch (error) {
             throw new RedisError("Error removing in redis", error as Error, table, key, redisKey);
@@ -162,7 +162,7 @@ export class RedisClient implements RedisClientInterface {
         try {
             const response = await client.FLUSHALL();
 
-            this.logHandler.debug("Redis response", {response}, RedisModuleKeyname)
+            this.logHandler.debug("Redis - `clearAll` response", {response}, RedisModuleKeyname)
 
         } catch (error) {
             throw new RedisError("Error clearing redis", error as Error);