From 1a741531b7722d944efdd67af04cccfd4c7f0156 Mon Sep 17 00:00:00 2001 From: "marco.mruz" Date: Fri, 17 May 2024 10:51:05 +0200 Subject: [PATCH] implement connection to the perf monitoring service --- .env.example | 1 + codegen-perf-monitoring.yml | 15 ++ package.json | 4 +- src/__generated__/perf-monitoring.graphql.ts | 159 ++++++++++++++++++ src/config.ts | 8 +- src/context.ts | 3 +- .../performance-monitoring-graphql.ts | 133 +++++++++++++++ src/external-api/performance-monitoring.ts | 28 --- src/schema/api.graphql | 10 +- src/schema/nexus-typegen.ts | 20 +-- src/schema/performance-monitoring.ts | 39 +++-- 11 files changed, 358 insertions(+), 62 deletions(-) create mode 100644 codegen-perf-monitoring.yml create mode 100644 src/__generated__/perf-monitoring.graphql.ts create mode 100644 src/external-api/performance-monitoring-graphql.ts delete mode 100644 src/external-api/performance-monitoring.ts diff --git a/.env.example b/.env.example index 72dfbbe6..4d88bc6e 100644 --- a/.env.example +++ b/.env.example @@ -13,6 +13,7 @@ TOPOLOGY_ENABLED=true # either define all of them or do not define them at all TOPOLOGY_DISCOVERY_API_URL=http://10.19.0.5:8080/api/topology/data TOPOLOGY_DISCOVERY_GRAPHQL_API_URL=http://localhost:5000/api/graphql +PERFORMANCE_MONITORING_GRAPHQL_API_URL=http://localhost:8082/api/graphql KAFKA_BROKER="10.19.0.25:9094" KAFKA_TOPIC=frinx diff --git a/codegen-perf-monitoring.yml b/codegen-perf-monitoring.yml new file mode 100644 index 00000000..a41f249d --- /dev/null +++ b/codegen-perf-monitoring.yml @@ -0,0 +1,15 @@ +schema: + - 'http://localhost:8082' +documents: + - './src/external-api/performance-monitoring-graphql.ts' +generates: + src/__generated__/perf-monitoring.graphql.ts: + plugins: + - 'typescript' + - 'typescript-operations' + config: + enumsAsTypes: true + avoidOptionals: + field: true + object: false + inputValue: false diff --git a/package.json b/package.json index 36515f12..2dbc6d3e 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,9 @@ "test:generate": "cross-env NODE_ENV=testing ts-node --transpile-only -r dotenv/config src/schema/index.ts", "jest": "cross-env NODE_ENV=testing jest --verbose", "test": "run-s test:generate jest", - "codegen": "graphql-codegen --config codegen-topology-discovery.yml" + "codegen": "run-s codegen:discovery codegen:monitoring", + "codegen:discovery": "graphql-codegen --config codegen-topology-discovery.yml", + "codegen:monitoring": "graphql-codegen --config codegen-perf-monitoring.yml" }, "devDependencies": { "@frinx/eslint-config-typescript-base": "15.0.0-build.1", diff --git a/src/__generated__/perf-monitoring.graphql.ts b/src/__generated__/perf-monitoring.graphql.ts new file mode 100644 index 00000000..37b20ae5 --- /dev/null +++ b/src/__generated__/perf-monitoring.graphql.ts @@ -0,0 +1,159 @@ +export type Maybe = T | null; +export type InputMaybe = Maybe; +export type Exact = { [K in keyof T]: T[K] }; +export type MakeOptional = Omit & { [SubKey in K]?: Maybe }; +export type MakeMaybe = Omit & { [SubKey in K]: Maybe }; +/** All built-in and custom scalars, mapped to their actual values */ +export type Scalars = { + ID: string; + String: string; + Boolean: boolean; + Int: number; + Float: number; + Datetime: any; +}; + +/** Bucket width unit types. */ +export type BucketUnit = + | 'centuries' + | 'days' + | 'hours' + | 'microseconds' + | 'milliseconds' + | 'minutes' + | 'months' + | 'seconds' + | 'weeks' + | 'years'; + +/** + * Bucket width input type that wraps unit and value. + * These two values will be join to create 'bucket_width' parameter for TimescaleDB time_bucket function. + */ +export type BucketWidth = { + unit: BucketUnit; + value: Scalars['Float']; +}; + +/** Represents the percentage usage value. Includes also device name. */ +export type Percentage = { + __typename?: 'Percentage'; + device: Scalars['String']; + usage: Maybe; +}; + +/** Represents a percentage value at a specific time. */ +export type PercentageInTime = { + __typename?: 'PercentageInTime'; + /** The timestamp indicating when the percentage value was recorded. */ + time: Scalars['Datetime']; + /** The percentage value recorded at the given time. */ + usage: Scalars['Float']; +}; + +/** Represents a series of percentage values over time. Includes also device name. */ +export type PercentageInTimeSeries = { + __typename?: 'PercentageInTimeSeries'; + device: Scalars['String']; + usages: Maybe>; +}; + +export type Query = { + __typename?: 'Query'; + /** Read CPU usages in time range for single device. */ + cpuUsage: PercentageInTimeSeries; + /** Read CPU usages in time range for multiple devices. */ + cpuUsages: Maybe>; + /** Read the current CPU usage for single device. */ + currentCpuUsage: Percentage; + /** Read the current CPU usage for multiple devices. */ + currentCpuUsages: Maybe>; + /** Read the current memory usage for single device. */ + currentMemoryUsage: Percentage; + /** Read the current memory usage for multiple devices. */ + currentMemoryUsages: Maybe>; + /** Read memory usages in time range for single device. */ + memoryUsage: PercentageInTimeSeries; + /** Read memory usages in time range for multiple devices. */ + memoryUsages: Maybe>; +}; + + +export type QueryCpuUsageArgs = { + bucket_width?: InputMaybe; + device: Scalars['String']; + end_time?: InputMaybe; + start_time?: InputMaybe; +}; + + +export type QueryCpuUsagesArgs = { + bucket_width?: InputMaybe; + devices: Array; + end_time?: InputMaybe; + start_time?: InputMaybe; +}; + + +export type QueryCurrentCpuUsageArgs = { + device: Scalars['String']; +}; + + +export type QueryCurrentCpuUsagesArgs = { + devices: Array; +}; + + +export type QueryCurrentMemoryUsageArgs = { + device: Scalars['String']; +}; + + +export type QueryCurrentMemoryUsagesArgs = { + devices: Array; +}; + + +export type QueryMemoryUsageArgs = { + bucket_width?: InputMaybe; + device: Scalars['String']; + end_time?: InputMaybe; + start_time?: InputMaybe; +}; + + +export type QueryMemoryUsagesArgs = { + bucket_width?: InputMaybe; + devices?: InputMaybe>; + end_time?: InputMaybe; + start_time?: InputMaybe; +}; + +export type CurrentMemoryUsagesQueryVariables = Exact<{ + names: Array | Scalars['String']; +}>; + + +export type CurrentMemoryUsagesQuery = { __typename?: 'Query', currentMemoryUsages: Array<{ __typename?: 'Percentage', device: string, usage: number | null }> | null }; + +export type CurrentCpuUsagesQueryVariables = Exact<{ + names: Array | Scalars['String']; +}>; + + +export type CurrentCpuUsagesQuery = { __typename?: 'Query', currentCpuUsages: Array<{ __typename?: 'Percentage', device: string, usage: number | null }> | null }; + +export type CurrentMemoryUsageQueryVariables = Exact<{ + name: Scalars['String']; +}>; + + +export type CurrentMemoryUsageQuery = { __typename?: 'Query', currentMemoryUsage: { __typename?: 'Percentage', device: string, usage: number | null } }; + +export type CurrentCpuUsageQueryVariables = Exact<{ + name: Scalars['String']; +}>; + + +export type CurrentCpuUsageQuery = { __typename?: 'Query', currentCpuUsage: { __typename?: 'Percentage', device: string, usage: number | null } }; diff --git a/src/config.ts b/src/config.ts index 7757da54..e96df6c8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -33,6 +33,7 @@ type TopologyConfigEnabled = { topologyEnabled: true; topologyDiscoveryURL: string; topologyDiscoveryGraphqlURL: string; + performanceMonitoringGraphqlURL: string; }; type TopologyConfigDisabled = { @@ -46,15 +47,17 @@ function getTopologyConfig(): TopologyConfig { const topologyEnabled = stringToBoolean(envString('TOPOLOGY_ENABLED')); const topologyDiscoveryURL = optionalEnvString('TOPOLOGY_DISCOVERY_API_URL'); const topologyDiscoveryGraphqlURL = optionalEnvString('TOPOLOGY_DISCOVERY_GRAPHQL_API_URL'); + const performanceMonitoringGraphqlURL = optionalEnvString('PERFORMANCE_MONITORING_GRAPHQL_API_URL'); + if (!topologyEnabled) { return { topologyEnabled: false, }; } - if (!topologyDiscoveryURL || !topologyDiscoveryGraphqlURL) { + if (!topologyDiscoveryURL || !topologyDiscoveryGraphqlURL || !performanceMonitoringGraphqlURL) { throw new Error( - 'Not all mandatory topology discovery url (TOPOLOGY_DISCOVERY_API_URL, TOPOLOGY_DISCOVERY_GRAPHQL_API_URL) were found.', + 'Not all mandatory topology discovery url (TOPOLOGY_DISCOVERY_API_URL, TOPOLOGY_DISCOVERY_GRAPHQL_API_URL, PERFORMANCE_MONITORING_GRAPHQL_API_URL) were found.', ); } @@ -62,6 +65,7 @@ function getTopologyConfig(): TopologyConfig { topologyEnabled: true, topologyDiscoveryURL, topologyDiscoveryGraphqlURL, + performanceMonitoringGraphqlURL, }; } diff --git a/src/context.ts b/src/context.ts index bcd4a23a..6f2237e4 100644 --- a/src/context.ts +++ b/src/context.ts @@ -7,7 +7,7 @@ import uniconfigAPI, { UniConfigAPI } from './external-api/uniconfig'; import prismaClient from './prisma-client'; import config from './config'; import kafkaProducers, { KafkaService } from './external-api/kafka'; -import getPerformanceMonitoringAPI, { PerformanceMonitoringAPI } from './external-api/performance-monitoring'; +import getPerformanceMonitoringAPI, { PerformanceMonitoringAPI } from './external-api/performance-monitoring-graphql'; export type Context = { kafka: KafkaService | null; @@ -54,5 +54,6 @@ export default async function createContext(context: ExpressContextFunctionArgum export function createSubscriptionContext() { return { prisma: prismaClient, + performanceMonitoringAPI: getPerformanceMonitoringAPI(), }; } diff --git a/src/external-api/performance-monitoring-graphql.ts b/src/external-api/performance-monitoring-graphql.ts new file mode 100644 index 00000000..5a323c6a --- /dev/null +++ b/src/external-api/performance-monitoring-graphql.ts @@ -0,0 +1,133 @@ +import { gql, GraphQLClient } from 'graphql-request'; +import config from '../config'; +import { + CurrentCpuUsageQuery, + CurrentCpuUsageQueryVariables, + CurrentCpuUsagesQuery, + CurrentCpuUsagesQueryVariables, + CurrentMemoryUsageQuery, + CurrentMemoryUsageQueryVariables, + CurrentMemoryUsagesQuery, + CurrentMemoryUsagesQueryVariables, +} from '../__generated__/perf-monitoring.graphql'; + +export type DeviceLoadUsage = { + cpuUsage: number | null; + memoryUsage: number | null; +}; + +const DEVICE_MEMORY_USAGES = gql` + query CurrentMemoryUsages($names: [String!]!) { + currentMemoryUsages(devices: $names) { + device + usage + } + } +`; + +const DEVICE_CPU_USAGES = gql` + query CurrentCpuUsages($names: [String!]!) { + currentCpuUsages(devices: $names) { + device + usage + } + } +`; + +const DEVICE_MEMORY_USAGE = gql` + query CurrentMemoryUsage($name: String!) { + currentMemoryUsage(device: $name) { + device + usage + } + } +`; + +const DEVICE_CPU_USAGE = gql` + query CurrentCpuUsage($name: String!) { + currentCpuUsage(device: $name) { + device + usage + } + } +`; + +function getPerformanceMonitoringAPI() { + if (config.topologyEnabled === false) { + return undefined; + } + + const client = new GraphQLClient(config.performanceMonitoringGraphqlURL); + + async function getDeviceCpuUsage(deviceName: string): Promise { + const result = await client.request(DEVICE_CPU_USAGE, { + name: deviceName, + }); + + return result; + } + + async function getDeviceMemoryUsage(deviceName: string): Promise { + const result = await client.request( + DEVICE_MEMORY_USAGE, + { name: deviceName }, + ); + + return result; + } + + async function getDeviceCpuUsages(deviceNames: string[]): Promise { + const result = await client.request(DEVICE_CPU_USAGES, { + names: deviceNames, + }); + + return result; + } + + async function getDeviceMemoryUsages(deviceNames: string[]): Promise { + const result = await client.request( + DEVICE_MEMORY_USAGES, + { names: deviceNames }, + ); + + return result; + } + + async function getDeviceLoadUsage(deviceName: string): Promise { + const [cpuUsage, memoryUsage] = await Promise.all([ + getDeviceCpuUsage(deviceName), + getDeviceMemoryUsage(deviceName), + ]); + + return { cpuUsage: cpuUsage.currentCpuUsage.usage, memoryUsage: memoryUsage.currentMemoryUsage.usage }; + } + + async function getDeviceLoadUsages(deviceNames: string[]): Promise<(DeviceLoadUsage & { deviceName: string })[]> { + const [cpuUsages, memoryUsages] = await Promise.all([ + getDeviceCpuUsages(deviceNames), + getDeviceMemoryUsages(deviceNames), + ]); + + const map = new Map(); + + cpuUsages.currentCpuUsages?.forEach((cpuUsage) => { + map.set(cpuUsage.device, { deviceName: cpuUsage.device, cpuUsage: cpuUsage.usage, memoryUsage: null }); + }); + + memoryUsages.currentMemoryUsages?.forEach((memoryUsage) => { + const deviceLoadUsage = map.get(memoryUsage.device); + if (deviceLoadUsage) { + deviceLoadUsage.memoryUsage = memoryUsage.usage; + } else { + map.set(memoryUsage.device, { deviceName: memoryUsage.device, cpuUsage: null, memoryUsage: memoryUsage.usage }); + } + }); + + return Array.from(map.values()); + } + + return { getDeviceLoadUsage, getDeviceLoadUsages }; +} + +export type PerformanceMonitoringAPI = ReturnType; +export default getPerformanceMonitoringAPI; diff --git a/src/external-api/performance-monitoring.ts b/src/external-api/performance-monitoring.ts deleted file mode 100644 index 100ddd43..00000000 --- a/src/external-api/performance-monitoring.ts +++ /dev/null @@ -1,28 +0,0 @@ -export type DeviceLoadUsage = { - cpuUsage: number; - memoryUsage: number; -}; - -async function getDeviceCpuUsage(deviceName: string): Promise { - return Math.random() * 100; -} - -async function getDeviceMemoryUsage(deviceName: string): Promise { - return Math.random() * 100; -} - -function getPerformanceMonitoringAPI() { - async function getDeviceLoadUsage(deviceName: string): Promise { - const [cpuUsage, memoryUsage] = await Promise.all([ - getDeviceCpuUsage(deviceName), - getDeviceMemoryUsage(deviceName), - ]); - - return { cpuUsage, memoryUsage }; - } - - return { getDeviceLoadUsage }; -} - -export type PerformanceMonitoringAPI = ReturnType; -export default getPerformanceMonitoringAPI; diff --git a/src/schema/api.graphql b/src/schema/api.graphql index 512e5768..5ebf2155 100644 --- a/src/schema/api.graphql +++ b/src/schema/api.graphql @@ -262,7 +262,7 @@ type DeviceEdge { } type DeviceListUsage { - devicesUsage: [DevicesUsage]! + devicesUsage: [DevicesUsage!]! } input DeviceOrderByInput { @@ -289,14 +289,14 @@ enum DeviceSource { } type DeviceUsage { - cpuLoad: Float! - memoryLoad: Float! + cpuLoad: Float + memoryLoad: Float } type DevicesUsage { - cpuLoad: Float! + cpuLoad: Float deviceName: String! - memoryLoad: Float! + memoryLoad: Float } type DiffData { diff --git a/src/schema/nexus-typegen.ts b/src/schema/nexus-typegen.ts index 39440375..b24697f0 100644 --- a/src/schema/nexus-typegen.ts +++ b/src/schema/nexus-typegen.ts @@ -338,18 +338,18 @@ export interface NexusGenObjects { }; DeviceListUsage: { // root type - devicesUsage: Array; // [DevicesUsage]! + devicesUsage: NexusGenRootTypes['DevicesUsage'][]; // [DevicesUsage!]! }; DeviceUsage: { // root type - cpuLoad: number; // Float! - memoryLoad: number; // Float! + cpuLoad?: number | null; // Float + memoryLoad?: number | null; // Float }; DevicesUsage: { // root type - cpuLoad: number; // Float! + cpuLoad?: number | null; // Float deviceName: string; // String! - memoryLoad: number; // Float! + memoryLoad?: number | null; // Float }; DiffData: { // root type @@ -883,18 +883,18 @@ export interface NexusGenFieldTypes { }; DeviceListUsage: { // field return type - devicesUsage: Array; // [DevicesUsage]! + devicesUsage: NexusGenRootTypes['DevicesUsage'][]; // [DevicesUsage!]! }; DeviceUsage: { // field return type - cpuLoad: number; // Float! - memoryLoad: number; // Float! + cpuLoad: number | null; // Float + memoryLoad: number | null; // Float }; DevicesUsage: { // field return type - cpuLoad: number; // Float! + cpuLoad: number | null; // Float deviceName: string; // String! - memoryLoad: number; // Float! + memoryLoad: number | null; // Float }; DiffData: { // field return type diff --git a/src/schema/performance-monitoring.ts b/src/schema/performance-monitoring.ts index 870bf6da..15d4b20e 100644 --- a/src/schema/performance-monitoring.ts +++ b/src/schema/performance-monitoring.ts @@ -1,12 +1,12 @@ import { objectType, subscriptionField, nonNull, stringArg, intArg, list } from 'nexus'; -import performanceMonitoringAPI, { DeviceLoadUsage } from '../external-api/performance-monitoring'; +import { DeviceLoadUsage } from '../external-api/performance-monitoring-graphql'; import { asyncGenerator } from '../helpers/async-generator'; export const DeviceUsage = objectType({ name: 'DeviceUsage', definition: (t) => { - t.nonNull.float('cpuLoad'); - t.nonNull.float('memoryLoad'); + t.float('cpuLoad'); + t.float('memoryLoad'); }, }); @@ -16,10 +16,16 @@ export const DeviceUsageSubscription = subscriptionField('deviceUsage', { deviceName: nonNull(stringArg()), refreshEverySec: intArg(), }, - subscribe: async (_, { deviceName, refreshEverySec }) => + subscribe: async (_, { deviceName, refreshEverySec }, { performanceMonitoringAPI }) => asyncGenerator( (refreshEverySec || 10) * 1000, - () => performanceMonitoringAPI().getDeviceLoadUsage(deviceName), + async () => { + if (performanceMonitoringAPI == null) { + return { cpuUsage: null, memoryUsage: null }; + } + + return performanceMonitoringAPI.getDeviceLoadUsage(deviceName); + }, () => true, ), resolve: (data) => ({ @@ -32,15 +38,15 @@ export const DevicesUsage = objectType({ name: 'DevicesUsage', definition: (t) => { t.nonNull.string('deviceName'); - t.nonNull.float('cpuLoad'); - t.nonNull.float('memoryLoad'); + t.float('cpuLoad'); + t.float('memoryLoad'); }, }); export const DeviceListUsage = objectType({ name: 'DeviceListUsage', definition: (t) => { - t.nonNull.list.field('devicesUsage', { + t.nonNull.list.nonNull.field('devicesUsage', { type: DevicesUsage, }); }, @@ -52,17 +58,20 @@ export const DevicesUsageSubscription = subscriptionField('devicesUsage', { deviceNames: nonNull(list(nonNull(stringArg()))), refreshEverySec: intArg(), }, - subscribe: async (_, { deviceNames, refreshEverySec }) => + subscribe: async (_, { deviceNames, refreshEverySec }, { performanceMonitoringAPI }) => asyncGenerator( (refreshEverySec || 10) * 1000, async () => { - const promises = deviceNames.map((deviceName) => performanceMonitoringAPI().getDeviceLoadUsage(deviceName)); - const devicesLoads = await Promise.all(promises); + if (performanceMonitoringAPI == null) { + return []; + } + + const usages = await performanceMonitoringAPI.getDeviceLoadUsages(deviceNames); - return devicesLoads.map((deviceLoad, index) => ({ - deviceName: deviceNames[index], - cpuLoad: deviceLoad.cpuUsage, - memoryLoad: deviceLoad.memoryUsage, + return usages.map(({ cpuUsage, deviceName, memoryUsage }) => ({ + deviceName, + cpuLoad: cpuUsage, + memoryLoad: memoryUsage, })); }, () => true,