Split metrics from device resource

src/balena-init.sql

@@ -188,3 +188,6 @@ ON "application" ("slug" varchar_pattern_ops, "is public", "is host");
 
 CREATE INDEX IF NOT EXISTS "scheduled_job_run_start_timestamp_idx"
 ON "scheduled job run" (DATE_TRUNC('milliseconds', "start timestamp"));
+
+CREATE INDEX IF NOT EXISTS "device_metrics_record_by_device_idx"
+ON "device metrics record" ("is reported by-device");

src/balena-model.ts

@@ -364,14 +364,6 @@ export interface Device {
 	public_address: string | null;
 	ip_address: string | null;
 	mac_address: string | null;
-	memory_usage: number | null;
-	memory_total: number | null;
-	storage_block_device: string | null;
-	storage_usage: number | null;
-	storage_total: number | null;
-	cpu_usage: number | null;
-	cpu_temp: number | null;
-	is_undervolted: boolean;
 	cpu_id: string | null;
 	is_running__release: { __id: number } | [Release?] | null;
 	download_progress: number | null;
@@ -403,6 +395,7 @@ export interface Device {
 	service_install?: ServiceInstall[];
 	installs__image?: ImageInstall[];
 	installs__application__has__service_name?: ServiceInstall[];
+	reports__device_metrics_record?: DeviceMetricsRecord[];
 }
 
 export interface DeviceEnvironmentVariable {
@@ -565,6 +558,21 @@ export interface UserHasPublicKey {
 	title: string;
 }
 
+export interface DeviceMetricsRecord {
+	created_at: DateString;
+	modified_at: DateString;
+	id: number;
+	is_reported_by__device: { __id: number } | [Device];
+	memory_usage: number | null;
+	memory_total: number | null;
+	storage_block_device: string | null;
+	storage_usage: number | null;
+	storage_total: number | null;
+	cpu_usage: number | null;
+	cpu_temp: number | null;
+	is_undervolted: boolean;
+}
+
 export interface DeviceTypeAlias {
 	created_at: DateString;
 	modified_at: DateString;

src/balena.sbvr

@@ -581,6 +581,31 @@ Fact type: device should be managed by release
 	Necessity: each device should be managed by at most one release
 
 
+Term: device metrics record
+
+-- device metrics record
+Fact type: device metrics record is reported by device
+    Synonymous Form: device reports device metrics record
+    Necessity: each device metrics record is reported by exactly one device
+	Necessity: each device reports exactly one device metrics record
+
+Fact type: device metrics record has memory usage
+	Necessity: each device metrics record has at most one memory usage
+Fact type: device metrics record has memory total
+	Necessity: each device metrics record has at most one memory total
+Fact type: device metrics record has storage block device
+	Necessity: each device metrics record has at most one storage block device
+Fact type: device metrics record has storage usage
+	Necessity: each device metrics record has at most one storage usage
+Fact type: device metrics record has storage total
+	Necessity: each device metrics record has at most one storage total
+Fact type: device metrics record has cpu usage
+	Necessity: each device metrics record has at most one cpu usage
+Fact type: device metrics record has cpu temp
+	Necessity: each device metrics record has at most one cpu temp
+Fact type: device metrics record is undervolted
+
+
 -- application config variable
 
 Fact type: application config variable has value

src/features/cascade-delete/hooks.ts

@@ -22,6 +22,7 @@ setupDeleteCascade('device', {
 	device_tag: 'device',
 	image_install: 'device',
 	service_install: 'device',
+	device_metrics_record: 'is_reported_by__device',
 });
 
 setupDeleteCascade('image', {

src/features/device-state/index.ts

@@ -23,9 +23,11 @@ export {
 	serviceInstallFromImage,
 } from './state-get-utils';
 export {
-	metricsPatchFields,
+	validDeviceMetricsRecordPatchFields,
 	v2ValidPatchFields,
+	v2ValidDevicePatchFields,
 	v3ValidPatchFields,
+	v3ValidDevicePatchFields,
 } from './state-patch-utils';
 
 const gracefulGet = resolveOrDenyDevicesWithStatus(304);

src/features/device-state/routes/state-patch-v2.ts

@@ -10,11 +10,12 @@ import { getIP } from '../../../lib/utils';
 import type { ImageInstall, PickDeferred } from '../../../balena-model';
 import {
 	shouldUpdateMetrics,
-	metricsPatchFields,
-	v2ValidPatchFields,
 	upsertImageInstall,
 	deleteOldImageInstalls,
 	truncateShortTextFields,
+	StatePatchDeviceMetricsRecordBody,
+	validDeviceMetricsRecordPatchFields,
+	v2ValidDevicePatchFields,
 } from '../state-patch-utils';
 import type { ResolveDeviceInfoCustomObject } from '../middleware';
 
@@ -26,7 +27,7 @@ type LocalBody = NonNullable<StatePatchV2Body['local']>;
  * These typings should be used as a guide to what should be sent, but cannot be trusted as what actually *is* sent.
  */
 export type StatePatchV2Body = {
-	local?: {
+	local?: StatePatchDeviceMetricsRecordBody & {
 		should_be_running__release?: number;
 		name?: string;
 		/**
@@ -46,15 +47,6 @@ export type StatePatchV2Body = {
 		download_progress?: number | null;
 		api_port?: number;
 		api_secret?: string;
-		memory_usage?: number;
-		memory_total?: number;
-		storage_block_device?: string;
-		storage_usage?: number;
-		storage_total?: number;
-		cpu_temp?: number;
-		cpu_usage?: number;
-		cpu_id?: string;
-		is_undervolted?: boolean;
 		is_on__commit?: string | null;
 		apps?: Array<{
 			services?: {
@@ -139,20 +131,13 @@ export const statePatchV2: RequestHandler = async (req, res) => {
 			const { apps } = local;
 
 			let deviceBody:
-				| Pick<LocalBody, (typeof v2ValidPatchFields)[number]> & {
+				| Pick<LocalBody, (typeof v2ValidDevicePatchFields)[number]> & {
 						is_running__release?: number | null;
-				  } = _.pick(local, v2ValidPatchFields);
-			let metricsBody: Pick<LocalBody, (typeof metricsPatchFields)[number]> =
-				_.pick(local, metricsPatchFields);
-			if (
-				Object.keys(metricsBody).length > 0 &&
-				(await shouldUpdateMetrics(uuid))
-			) {
-				// If we should force a metrics update then merge the two together and clear `metricsBody` so
-				// that we don't try to merge it again later
-				deviceBody = { ...deviceBody, ...metricsBody };
-				metricsBody = {};
-			}
+				  } = _.pick(local, v2ValidDevicePatchFields);
+			const metricsBody: Pick<
+				LocalBody,
+				(typeof validDeviceMetricsRecordPatchFields)[number]
+			> = _.pick(local, validDeviceMetricsRecordPatchFields);
 
 			if (local.name != null) {
 				deviceBody.device_name = local.name;
@@ -191,15 +176,13 @@ export const statePatchV2: RequestHandler = async (req, res) => {
 						}
 					}
 				}
-
-				updateFns.push(async (resinApiTx) => {
-					if (Object.keys(deviceBody).length > 0) {
+				if (Object.keys(deviceBody).length > 0) {
+					updateFns.push(async (resinApiTx) => {
 						// truncate for resilient legacy compatible device state patch so that supervisors don't fail
 						// to update b/c of length violation of 255 (SBVR SHORT TEXT type) for ip and mac address.
 						// sbvr-types does not export SHORT TEXT VARCHAR length 255 to import.
 						deviceBody = truncateShortTextFields(deviceBody);
 						// If we're updating anyway then ensure the metrics data is included
-						deviceBody = { ...deviceBody, ...metricsBody };
 						await resinApiTx.patch({
 							resource: 'device',
 							id: deviceId,
@@ -208,6 +191,32 @@ export const statePatchV2: RequestHandler = async (req, res) => {
 							},
 							body: deviceBody,
 						});
+					});
+				}
+			}
+			if (
+				Object.keys(metricsBody).length > 0 &&
+				(await shouldUpdateMetrics(uuid))
+			) {
+				updateFns.push(async (resinApiTx) => {
+					const latestDeviceMetricsRecord = await resinApiTx.get({
+						resource: 'device_metrics_record',
+						id: { is_reported_by__device: deviceId },
+					});
+					if (latestDeviceMetricsRecord == null) {
+						await resinApiTx.post({
+							resource: 'device_metrics_record',
+							id: { is_reported_by__device: deviceId },
+							body: {
+								...metricsBody,
+							},
+						});
+					} else {
+						await resinApiTx.patch({
+							resource: 'device_metrics_record',
+							id: { is_reported_by__device: deviceId },
+							body: metricsBody,
+						});
 					}
 				});
 			}

src/features/device-state/routes/state-patch-v3.ts

@@ -10,18 +10,21 @@ import { getIP } from '../../../lib/utils';
 import {
 	Application,
 	Device,
+	DeviceMetricsRecord,
 	Image,
 	ImageInstall,
 	PickDeferred,
 	Release,
 } from '../../../balena-model';
 import type { Filter } from 'pinejs-client-core';
-import { metricsPatchFields, v3ValidPatchFields } from '..';
+import { v3ValidDevicePatchFields } from '..';
 import {
 	deleteOldImageInstalls,
 	upsertImageInstall,
 	shouldUpdateMetrics,
 	truncateShortTextFields,
+	StatePatchDeviceMetricsRecordBody,
+	validDeviceMetricsRecordPatchFields,
 } from '../state-patch-utils';
 
 const { BadRequestError, UnauthorizedError, InternalRequestError } = errors;
@@ -31,7 +34,7 @@ const { api } = sbvrUtils;
  * These typings should be used as a guide to what should be sent, but cannot be trusted as what actually *is* sent.
  */
 export type StatePatchV3Body = {
-	[uuid: string]: {
+	[uuid: string]: StatePatchDeviceMetricsRecordBody & {
 		status?: string;
 		os_version?: string;
 		os_variant?: string;
@@ -42,15 +45,7 @@ export type StatePatchV3Body = {
 		mac_address?: string;
 		api_port?: number;
 		api_secret?: string;
-		memory_usage?: number;
-		memory_total?: number;
-		storage_block_device?: string;
-		storage_usage?: number;
-		storage_total?: number;
-		cpu_temp?: number;
-		cpu_usage?: number;
-		cpu_id?: string;
-		is_undervolted?: boolean;
+
 		/**
 		 * Used for setting dependent devices as online
 		 */
@@ -103,11 +98,13 @@ const fetchData = async (
 					belongs_to__application: {
 						$select: 'uuid',
 					},
+					reports__device_metrics_record: {},
 				},
 			},
 		})) as Array<
 			Pick<Device, 'id' | 'uuid'> & {
 				belongs_to__application: Array<Pick<Application, 'uuid'>>;
+				reports__device_metrics_record: DeviceMetricsRecord[];
 			}
 		>;
 		if (devices.length !== uuids.length) {
@@ -269,23 +266,14 @@ export const statePatchV3: RequestHandler = async (req, res) => {
 			let deviceBody:
 				| Pick<
 						StatePatchV3Body[string],
-						(typeof v3ValidPatchFields)[number]
+						(typeof v3ValidDevicePatchFields)[number]
 				  > & {
 						is_running__release?: number | null;
-				  } = _.pick(state, v3ValidPatchFields);
+				  } = _.pick(state, v3ValidDevicePatchFields);
 			let metricsBody: Pick<
 				StatePatchV3Body[string],
-				(typeof metricsPatchFields)[number]
-			> = _.pick(state, metricsPatchFields);
-			if (
-				Object.keys(metricsBody).length > 0 &&
-				(await shouldUpdateMetrics(uuid))
-			) {
-				// If we should force a metrics update then merge the two together and clear `metricsBody` so
-				// that we don't try to merge it again later
-				deviceBody = { ...deviceBody, ...metricsBody };
-				metricsBody = {};
-			}
+				(typeof validDeviceMetricsRecordPatchFields)[number]
+			> = _.pick(state, validDeviceMetricsRecordPatchFields);
 
 			if (deviceBody.cpu_id != null) {
 				if (/[^\x20-\x7E]/.test(deviceBody.cpu_id)) {
@@ -297,12 +285,25 @@ export const statePatchV3: RequestHandler = async (req, res) => {
 				}
 			}
 
-			if (apps != null || Object.keys(deviceBody).length > 0) {
+			if (
+				Object.keys(metricsBody).length > 0 &&
+				!(await shouldUpdateMetrics(uuid))
+			) {
+				// If we don't want to update the metrics then clear them
+				metricsBody = {};
+			}
+
+			if (
+				apps != null ||
+				Object.keys(deviceBody).length > 0 ||
+				Object.keys(metricsBody).length > 0
+			) {
 				// We lazily fetch the necessary data only if we absolutely must to avoid unnecessary work if it turns out we don't need it
 				data ??= await fetchData(req, custom, uuids, appReleasesCriteria);
 				const { images, releasesByAppUuid } = data;
 				const device = data.devicesByUuid[uuid];
-
+				const [latestDeviceMetricsRecord] =
+					device.reports__device_metrics_record;
 				if (apps != null) {
 					const userAppUuid = device.belongs_to__application[0].uuid;
 					if (releasesByAppUuid[userAppUuid] != null) {
@@ -321,7 +322,6 @@ export const statePatchV3: RequestHandler = async (req, res) => {
 					// sbvr-types does not export SHORT TEXT VARCHAR length 255 to import.
 					deviceBody = truncateShortTextFields(deviceBody);
 					// If we're updating anyway then ensure the metrics data is included
-					deviceBody = { ...deviceBody, ...metricsBody };
 					updateFns.push(async (resinApiTx) => {
 						await resinApiTx.patch({
 							resource: 'device',
@@ -334,6 +334,26 @@ export const statePatchV3: RequestHandler = async (req, res) => {
 					});
 				}
 
+				if (Object.keys(metricsBody).length > 0) {
+					updateFns.push(async (resinApiTx) => {
+						if (latestDeviceMetricsRecord == null) {
+							await resinApiTx.post({
+								resource: 'device_metrics_record',
+								body: {
+									...metricsBody,
+									...{ is_reported_by__device: device.id },
+								},
+							});
+						} else {
+							await resinApiTx.patch({
+								resource: 'device_metrics_record',
+								id: { is_reported_by__device: device.id },
+								body: metricsBody,
+							});
+						}
+					});
+				}
+
 				if (apps != null) {
 					const imgInstalls: Array<{
 						imageId: number;