refactor: Add support for monitoring trace counts

yonasBSD · Aug 8, 2024 · 41a3bc8 · 41a3bc8
1 parent 2621dc7
commit 41a3bc8
Show file tree

Hide file tree

Showing 22 changed files with 953 additions and 9,674 deletions.
diff --git a/App/FeatureSet/Workers/Jobs/TelemetryMonitor/MonitorTelemetryMonitor.ts b/App/FeatureSet/Workers/Jobs/TelemetryMonitor/MonitorTelemetryMonitor.ts
@@ -21,6 +21,11 @@ import PositiveNumber from "Common/Types/PositiveNumber";
 import JSONFunctions from "Common/Types/JSONFunctions";
 import DatabaseQueryHelper from "Common/Server/Types/Database/QueryHelper";
 import ObjectID from "Common/Types/ObjectID";
+import TraceMonitorResponse from "Common/Types/Monitor/TraceMonitor/TraceMonitorResponse";
+import MonitorStepTraceMonitor, {
+  MonitorStepTraceMonitorUtil,
+} from "Common/Types/Monitor/MonitorStepTraceMonitor";
+import SpanService from "Common/Server/Services/SpanService";
 
 RunCron(
   "LogMonitor:MonitorLogMonitor",
@@ -98,7 +103,9 @@ RunCron(
 
     logger.debug(telemetryMonitors);
 
-    const monitorResponses: Array<Promise<LogMonitorResponse>> = [];
+    const monitorResponses: Array<
+      Promise<LogMonitorResponse | TraceMonitorResponse>
+    > = [];
 
     for (const monitor of telemetryMonitors) {
       try {
@@ -127,7 +134,7 @@ RunCron(
       }
     }
 
-    const responses: Array<LogMonitorResponse> =
+    const responses: Array<LogMonitorResponse | TraceMonitorResponse> =
       await Promise.all(monitorResponses);
 
     for (const response of responses) {
@@ -140,13 +147,13 @@ type MonitorTelemetryMonitorFunction = (data: {
   monitorStep: MonitorStep;
   monitorType: MonitorType;
   monitorId: ObjectID;
-}) => Promise<LogMonitorResponse>;
+}) => Promise<LogMonitorResponse | TraceMonitorResponse>;
 
 const monitorTelemetryMonitor: MonitorTelemetryMonitorFunction = async (data: {
   monitorStep: MonitorStep;
   monitorType: MonitorType;
   monitorId: ObjectID;
-}): Promise<LogMonitorResponse> => {
+}): Promise<LogMonitorResponse | TraceMonitorResponse> => {
   const { monitorStep, monitorType, monitorId } = data;
 
   if (monitorType === MonitorType.Logs) {
@@ -156,9 +163,51 @@ const monitorTelemetryMonitor: MonitorTelemetryMonitorFunction = async (data: {
     });
   }
 
+  if (monitorType === MonitorType.Traces) {
+    return monitorTrace({
+      monitorStep,
+      monitorId,
+    });
+  }
+
   throw new BadDataException("Monitor type is not supported");
 };
 
+type MonitorTraceFunction = (data: {
+  monitorStep: MonitorStep;
+  monitorId: ObjectID;
+}) => Promise<TraceMonitorResponse>;
+
+const monitorTrace: MonitorTraceFunction = async (data: {
+  monitorStep: MonitorStep;
+  monitorId: ObjectID;
+}): Promise<TraceMonitorResponse> => {
+  // Monitor traces
+  const traceQuery: MonitorStepTraceMonitor | undefined =
+    data.monitorStep.data?.traceMonitor;
+
+  if (!traceQuery) {
+    throw new BadDataException("Trace query is missing");
+  }
+
+  const query: Query<Log> = MonitorStepTraceMonitorUtil.toQuery(traceQuery);
+
+  const countTraces: PositiveNumber = await SpanService.countBy({
+    query: query,
+    limit: LIMIT_PER_PROJECT,
+    skip: 0,
+    props: {
+      isRoot: true,
+    },
+  });
+
+  return {
+    traceCount: countTraces.toNumber(),
+    traceQuery: query,
+    monitorId: data.monitorId,
+  };
+};
+
 type MonitorLogsFunction = (data: {
   monitorStep: MonitorStep;
   monitorId: ObjectID;

diff --git a/Common/Server/Utils/Monitor/Criteria/TraceMonitorCriteria.ts b/Common/Server/Utils/Monitor/Criteria/TraceMonitorCriteria.ts
@@ -0,0 +1,31 @@
+import TraceMonitorResponse from "../../../../Types/Monitor/TraceMonitor/TraceMonitorResponse";
+import DataToProcess from "../DataToProcess";
+import CompareCriteria from "./CompareCriteria";
+import { CheckOn, CriteriaFilter } from "Common/Types/Monitor/CriteriaFilter";
+
+export default class TraceMonitorCriteria {
+  public static async isMonitorInstanceCriteriaFilterMet(input: {
+    dataToProcess: DataToProcess;
+    criteriaFilter: CriteriaFilter;
+  }): Promise<string | null> {
+    // Server Monitoring Checks
+
+    let threshold: number | string | undefined | null =
+      input.criteriaFilter.value;
+
+    if (input.criteriaFilter.checkOn === CheckOn.SpanCount) {
+      threshold = CompareCriteria.convertToNumber(threshold);
+
+      const currentSpanCount: number =
+        (input.dataToProcess as TraceMonitorResponse).spanCount || 0;
+
+      return CompareCriteria.compareCriteriaNumbers({
+        value: currentSpanCount,
+        threshold: threshold as number,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
+    return null;
+  }
+}
diff --git a/Common/Server/Utils/Monitor/DataToProcess.ts b/Common/Server/Utils/Monitor/DataToProcess.ts
@@ -2,11 +2,13 @@ import IncomingMonitorRequest from "Common/Types/Monitor/IncomingMonitor/Incomin
 import ServerMonitorResponse from "Common/Types/Monitor/ServerMonitor/ServerMonitorResponse";
 import ProbeMonitorResponse from "Common/Types/Probe/ProbeMonitorResponse";
 import LogMonitorResponse from "Common/Types/Monitor/LogMonitor/LogMonitorResponse";
+import TraceMonitorResponse from "Common/Types/Monitor/TraceMonitor/TraceMonitorResponse";
 
 type DataToProcess =
   | ProbeMonitorResponse
   | IncomingMonitorRequest
   | ServerMonitorResponse
-  | LogMonitorResponse;
+  | LogMonitorResponse
+  | TraceMonitorResponse;
 
 export default DataToProcess;
diff --git a/Common/Server/Utils/Monitor/MonitorResource.ts b/Common/Server/Utils/Monitor/MonitorResource.ts
@@ -54,6 +54,8 @@ import OneUptimeDate from "Common/Types/Date";
 import LogMonitorCriteria from "./Criteria/LogMonitorCriteria";
 import LogMonitorResponse from "Common/Types/Monitor/LogMonitor/LogMonitorResponse";
 import TelemetryType from "Common/Types/Telemetry/TelemetryType";
+import TraceMonitorResponse from "../../../Types/Monitor/TraceMonitor/TraceMonitorResponse";
+import TraceMonitorCriteria from "./Criteria/TraceMonitorCriteria";
 
 export default class MonitorResourceUtil {
   public static async monitorResource(
@@ -346,6 +348,13 @@ export default class MonitorResourceUtil {
         };
       }
 
+      if (dataToProcess && (dataToProcess as TraceMonitorResponse).spanQuery) {
+        telemetryQuery = {
+          telemetryQuery: (dataToProcess as TraceMonitorResponse).spanQuery,
+          telemetryType: TelemetryType.Trace,
+        };
+      }
+
       await this.criteriaMetCreateIncidentsAndUpdateMonitorStatus({
         monitor: monitor,
         rootCause: response.rootCause,
@@ -1239,6 +1248,19 @@ export default class MonitorResourceUtil {
       }
     }
 
+    if (input.monitor.monitorType === MonitorType.Traces) {
+      // check server monitor
+      const traceMonitorResult: string | null =
+        await TraceMonitorCriteria.isMonitorInstanceCriteriaFilterMet({
+          dataToProcess: input.dataToProcess,
+          criteriaFilter: input.criteriaFilter,
+        });
+
+      if (traceMonitorResult) {
+        return traceMonitorResult;
+      }
+    }
+
     return null;
   }
 }
diff --git a/Common/Types/BaseDatabase/Includes.ts b/Common/Types/BaseDatabase/Includes.ts
@@ -4,7 +4,7 @@ import JSONFunctions from "../JSONFunctions";
 import ObjectID from "../ObjectID";
 import SerializableObject from "../SerializableObject";
 
-export type IncludesType = Array<string> | Array<ObjectID>;
+export type IncludesType = Array<string> | Array<ObjectID> | Array<number>;
 
 export default class Includes extends SerializableObject {
   private _values: IncludesType = [];

diff --git a/Common/Types/Monitor/CriteriaFilter.ts b/Common/Types/Monitor/CriteriaFilter.ts
@@ -37,6 +37,9 @@ export enum CheckOn {
 
   // Log monitors.
   LogCount = "Log Count",
+
+  // Trace monitors.
+  SpanCount = "Span Count",
 }
 
 export interface ServerMonitorOptions {

diff --git a/Common/Types/Monitor/LogMonitor/LogMonitorResponse.ts b/Common/Types/Monitor/LogMonitor/LogMonitorResponse.ts
@@ -1,8 +1,9 @@
-import { JSONObject } from "../../JSON";
+import Log from "../../../Models/AnalyticsModels/Log";
+import Query from "../../BaseDatabase/Query";
 import ObjectID from "../../ObjectID";
 
 export default interface LogMonitorResponse {
   logCount: number;
-  logQuery: JSONObject;
+  logQuery: Query<Log>;
   monitorId: ObjectID;
 }
diff --git a/Common/Types/Monitor/MonitorCriteriaInstance.ts b/Common/Types/Monitor/MonitorCriteriaInstance.ts
@@ -105,6 +105,31 @@ export default class MonitorCriteriaInstance extends DatabaseProperty {
       return monitorCriteriaInstance;
     }
 
+    if (arg.monitorType === MonitorType.Traces) {
+      const monitorCriteriaInstance: MonitorCriteriaInstance =
+        new MonitorCriteriaInstance();
+
+      monitorCriteriaInstance.data = {
+        id: ObjectID.generate().toString(),
+        monitorStatusId: arg.monitorStatusId,
+        filterCondition: FilterCondition.Any,
+        filters: [
+          {
+            checkOn: CheckOn.SpanCount,
+            filterType: FilterType.GreaterThan,
+            value: 0, // if there are some logs then monitor is online.
+          },
+        ],
+        incidents: [],
+        changeMonitorStatus: true,
+        createIncidents: false,
+        name: `Check if ${arg.monitorName} is online`,
+        description: `This criteria checks if the ${arg.monitorName} is online`,
+      };
+
+      return monitorCriteriaInstance;
+    }
+
     if (arg.monitorType === MonitorType.SSLCertificate) {
       const monitorCriteriaInstance: MonitorCriteriaInstance =
         new MonitorCriteriaInstance();
@@ -337,6 +362,35 @@ export default class MonitorCriteriaInstance extends DatabaseProperty {
       };
     }
 
+    if (arg.monitorType === MonitorType.Traces) {
+      monitorCriteriaInstance.data = {
+        id: ObjectID.generate().toString(),
+        monitorStatusId: arg.monitorStatusId,
+        filterCondition: FilterCondition.Any,
+        filters: [
+          {
+            checkOn: CheckOn.SpanCount,
+            filterType: FilterType.EqualTo,
+            value: 0, // if there are no logs then the monitor is offline
+          },
+        ],
+        incidents: [
+          {
+            title: `${arg.monitorName} is offline`,
+            description: `${arg.monitorName} is currently offline.`,
+            incidentSeverityId: arg.incidentSeverityId,
+            autoResolveIncident: true,
+            id: ObjectID.generate().toString(),
+            onCallPolicyIds: [],
+          },
+        ],
+        changeMonitorStatus: true,
+        createIncidents: true,
+        name: `Check if ${arg.monitorName} is offline`,
+        description: `This criteria checks if the ${arg.monitorName} is offline`,
+      };
+    }
+
     if (arg.monitorType === MonitorType.IncomingRequest) {
       monitorCriteriaInstance.data = {
         id: ObjectID.generate().toString(),

diff --git a/Common/Types/Monitor/MonitorStep.ts b/Common/Types/Monitor/MonitorStep.ts
@@ -17,6 +17,9 @@ import MonitorType from "./MonitorType";
 import BrowserType from "./SyntheticMonitors//BrowserType";
 import ScreenSizeType from "./SyntheticMonitors/ScreenSizeType";
 import { FindOperator } from "typeorm";
+import MonitorStepTraceMonitor, {
+  MonitorStepTraceMonitorUtil,
+} from "./MonitorStepTraceMonitor";
 
 export interface MonitorStepType {
   id: string;
@@ -41,6 +44,9 @@ export interface MonitorStepType {
 
   // Log monitor type.
   logMonitor?: MonitorStepLogMonitor | undefined;
+
+  // trace monitor type.
+  traceMonitor?: MonitorStepTraceMonitor | undefined;
 }
 
 export default class MonitorStep extends DatabaseProperty {
@@ -61,6 +67,7 @@ export default class MonitorStep extends DatabaseProperty {
       screenSizeTypes: undefined,
       browserTypes: undefined,
       logMonitor: undefined,
+      traceMonitor: undefined,
     };
   }
 
@@ -85,6 +92,7 @@ export default class MonitorStep extends DatabaseProperty {
       screenSizeTypes: undefined,
       browserTypes: undefined,
       logMonitor: undefined,
+      traceMonitor: undefined,
     };
 
     return monitorStep;
@@ -146,6 +154,11 @@ export default class MonitorStep extends DatabaseProperty {
     return this;
   }
 
+  public setTraceMonitor(traceMonitor: MonitorStepTraceMonitor): MonitorStep {
+    this.data!.traceMonitor = traceMonitor;
+    return this;
+  }
+
   public setCustomCode(customCode: string): MonitorStep {
     this.data!.customCode = customCode;
     return this;
@@ -259,6 +272,12 @@ export default class MonitorStep extends DatabaseProperty {
                 this.data.logMonitor || MonitorStepLogMonitorUtil.getDefault(),
               )
             : undefined,
+          traceMonitor: this.data.traceMonitor
+            ? MonitorStepTraceMonitorUtil.toJSON(
+                this.data.traceMonitor ||
+                  MonitorStepTraceMonitorUtil.getDefault(),
+              )
+            : undefined,
         },
       });
     }
@@ -350,12 +369,19 @@ export default class MonitorStep extends DatabaseProperty {
       logMonitor: json["logMonitor"]
         ? (json["logMonitor"] as JSONObject)
         : undefined,
+      traceMonitor: json["traceMonitor"]
+        ? (json["traceMonitor"] as JSONObject)
+        : undefined,
     }) as any;
 
     if (monitorStep.data && !monitorStep.data?.logMonitor) {
       monitorStep.data.logMonitor = MonitorStepLogMonitorUtil.getDefault();
     }
 
+    if (monitorStep.data && !monitorStep.data?.traceMonitor) {
+      monitorStep.data.traceMonitor = MonitorStepTraceMonitorUtil.getDefault();
+    }
+
     return monitorStep;
   }