From e2691b2268f747818b8a801aa692e49fc6d6d7dd Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 4 Sep 2024 21:51:47 +0300 Subject: [PATCH] feat: finish container and add system checks --- packages/dashmate/src/createDIContainer.js | 8 + .../doctor/verifySystemRequirementsFactory.js | 94 +++++++++ .../tasks/doctor/analyseSamplesTaskFactory.js | 186 +++++++++++++++++- .../verifySystemRequirementsTaskFactory.js | 71 +------ 4 files changed, 286 insertions(+), 73 deletions(-) create mode 100644 packages/dashmate/src/doctor/verifySystemRequirementsFactory.js diff --git a/packages/dashmate/src/createDIContainer.js b/packages/dashmate/src/createDIContainer.js index 3fae72ac0cb..839b4dfa77e 100644 --- a/packages/dashmate/src/createDIContainer.js +++ b/packages/dashmate/src/createDIContainer.js @@ -116,6 +116,7 @@ import verifySystemRequirementsTaskFactory import analyseSamplesTaskFactory from './listr/tasks/doctor/analyseSamplesTaskFactory.js'; import collectSamplesTaskFactory from './listr/tasks/doctor/collectSamplesTaskFactory.js'; import prescriptionTaskFactory from './listr/tasks/doctor/prescriptionTaskFactory.js'; +import verifySystemRequirementsFactory from './doctor/verifySystemRequirementsFactory.js'; /** * @param {Object} [options] @@ -314,6 +315,13 @@ export default async function createDIContainer(options = {}) { prescriptionTask: asFunction(prescriptionTaskFactory).singleton(), }); + /** + * Doctor + */ + container.register({ + verifySystemRequirements: asFunction(verifySystemRequirementsFactory), + }); + /** * Helper */ diff --git a/packages/dashmate/src/doctor/verifySystemRequirementsFactory.js b/packages/dashmate/src/doctor/verifySystemRequirementsFactory.js new file mode 100644 index 00000000000..64def15869a --- /dev/null +++ b/packages/dashmate/src/doctor/verifySystemRequirementsFactory.js @@ -0,0 +1,94 @@ +/** + * @return {verifySystemRequirements} + */ +function verifySystemRequirementsFactory() { + /** + * @typedef {Function} verifySystemRequirements + * @param {Object} systemInfo + * @param {Object} systemInfo.dockerSystemInfo + * @param {Object} systemInfo.cpu + * @param {Object} systemInfo.memory + * @param {Object} systemInfo.diskSpace + * @param {boolean} isHP + * @param {Object} [overrideRequirements] + * @param {Number} [overrideRequirements.diskSpace] + * @returns {Object} + */ + function verifySystemRequirements( + { + dockerSystemInfo, + cpu, + memory, + diskSpace, + }, + isHP, + overrideRequirements = {}, + ) { + const MINIMUM_CPU_CORES = isHP ? 4 : 2; + const MINIMUM_CPU_FREQUENCY = 2.4; // GHz + const MINIMUM_RAM = isHP ? 8 : 4; // GB + const MINIMUM_DISK_SPACE = overrideRequirements.diskSpace ?? (isHP ? 200 : 100); // GB + + const warnings = {}; + + // CPU cores + const cpuCores = dockerSystemInfo?.NCPU ?? cpu?.cores; + + if (cpuCores) { + if (cpuCores < MINIMUM_CPU_CORES) { + warnings.cpuCores = `${cpuCores} CPU cores detected. At least ${MINIMUM_CPU_CORES} are required`; + } + } else if (process.env.DEBUG) { + // eslint-disable-next-line no-console + console.warn('Can\'t get CPU cores information'); + } + + // Memory + const totalMemory = dockerSystemInfo?.MemTotal ?? memory?.total; + + if (totalMemory) { + const totalMemoryGb = totalMemory / (1024 ** 3); // Convert to GB + + if (totalMemoryGb < MINIMUM_RAM) { + warnings.memory = `${totalMemoryGb.toFixed(2)}GB RAM detected. At least ${MINIMUM_RAM}GB is required`; + } + } else if (process.env.DEBUG) { + // eslint-disable-next-line no-console + console.warn('Can\'t get memory information'); + } + + // CPU speed + if (cpu && cpu.speed !== 0) { + if (cpu.speed < MINIMUM_CPU_FREQUENCY) { + warnings.cpuSpeed = `${cpu.speed.toFixed(1)}GHz CPU frequency detected. At least ${MINIMUM_CPU_FREQUENCY}GHz is required`; + } + } else if (process.env.DEBUG) { + // eslint-disable-next-line no-console + console.warn('Can\'t get CPU frequency'); + } + + // Check swap information + if (memory) { + const swapTotalGb = (memory.swaptotal / (1024 ** 3)); // Convert bytes to GB + + if (swapTotalGb < 2) { + warnings.swap = `Swap space is ${swapTotalGb.toFixed(2)}GB. 2GB is recommended`; + } + } + + // Get disk usage info + if (diskSpace) { + const availableDiskSpace = diskSpace.available / (1024 ** 3); // Convert to GB + + if (availableDiskSpace < MINIMUM_DISK_SPACE) { + warnings.diskSpace = `${availableDiskSpace.toFixed(2)}GB available disk space detected. At least ${MINIMUM_DISK_SPACE}GB is required`; + } + } + + return warnings; + } + + return verifySystemRequirements; +} + +module.exports = verifySystemRequirementsFactory; diff --git a/packages/dashmate/src/listr/tasks/doctor/analyseSamplesTaskFactory.js b/packages/dashmate/src/listr/tasks/doctor/analyseSamplesTaskFactory.js index dd0b445101a..2b7ab8ad32c 100644 --- a/packages/dashmate/src/listr/tasks/doctor/analyseSamplesTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/doctor/analyseSamplesTaskFactory.js @@ -5,9 +5,14 @@ import { Listr } from 'listr2'; * * @param {DockerCompose} dockerCompose * @param {getServiceList} getServiceList + * @param {verifySystemRequirements} verifySystemRequirements * @return {analyseSamplesTask} */ -export default function analyseSamplesTaskFactory(dockerCompose, getServiceList) { +export default function analyseSamplesTaskFactory( + dockerCompose, + getServiceList, + verifySystemRequirements, +) { /** * @typedef {function} analyseSamplesTask * @param config @@ -15,6 +20,111 @@ export default function analyseSamplesTaskFactory(dockerCompose, getServiceList) */ function analyseSamplesTask(config) { return new Listr([ + { + title: 'System resources', + task: async (ctx) => { + const { + cpu, + dockerSystemInfo, + currentLoad, + diskSpace, + fsOpenFiles, + memory, + } = ctx.samples.getSystemInfo(); + + ctx.systemResourceProblems = verifySystemRequirements( + { + dockerSystemInfo, + cpu, + memory, + diskSpace, + }, + config.get('platform.enabled'), + { + diskSpace: 5, + }, + ); + + return new Listr([ + { + title: 'CPU cores', + task: (_ctx, task) => { + if (ctx.systemResourceProblems.cpuCores) { + throw new Error(task.title); + } + }, + }, + { + title: 'CPU speed', + task: (_ctx, task) => { + if (ctx.systemResourceProblems.cpuSpeed) { + throw new Error(task.title); + } + }, + }, + { + title: 'CPU load', + task: (_ctx, task) => { + if (currentLoad.avgLoad > 0.8) { + ctx.systemResourceProblems.avgLoad = `Average system load ${currentLoad.avgLoad.toFixed(2)} is higher than normal. Consider to upgrade CPU.`; + throw new Error(task.title); + } + }, + }, + { + title: 'Total RAM', + task: (_ctx, task) => { + if (ctx.systemResourceProblems.memory) { + throw new Error(task.title); + } + }, + }, + { + title: 'Free RAM', + enabled: Number.isInteger(memory.free), + task: (_ctx, task) => { + if (memory.free) { + const memoryGb = memory.free / (1024 ** 3); + if (memoryGb < 0.5) { + ctx.systemResourceProblems.freeMemory = `Only ${memoryGb.toFixed(1)}GB RAM is available. Consider to upgrade RAM.`; + throw new Error(task.title); + } + } + }, + }, + { + title: 'File descriptors', + enabled: fsOpenFiles?.allocated && fsOpenFiles?.max, + task: (_ctx, task) => { + const available = fsOpenFiles.max - fsOpenFiles.allocated; + if (available < 1000) { + ctx.systemResourceProblems.fsOpenFiles = `${available} available file descriptors left. Consider to increase max limit.`; + throw new Error(task.title); + } + }, + }, + { + title: 'Swap', + task: (_ctx, task) => { + if (ctx.systemResourceProblems.swap) { + throw new Error(task.title); + } + }, + }, + { + title: 'Disk space', + task: (_ctx, task) => { + if (ctx.systemResourceProblems.diskSpace) { + throw new Error(task.title); + } + }, + }, + // TODO: Disk IO + ], { + exitOnError: false, + }); + }, + }, { title: 'Docker is started', task: async (ctx, task) => { @@ -39,8 +149,8 @@ export default function analyseSamplesTaskFactory(dockerCompose, getServiceList) ctx.servicesOOMKilled = []; return new Listr( - services.map((service) => { - return { + services.map((service) => ( + { title: service.title, task: () => { const dockerInspect = ctx.samples.getServiceInfo(service.name, 'dockerInspect'); @@ -50,17 +160,27 @@ export default function analyseSamplesTaskFactory(dockerCompose, getServiceList) service, message: dockerInspect.message, }); - } else if (dockerInspect.State.Restarting) { - // TODO: ctx.servicesFailed - //dockerInspect.State.Started = dockerInspect.State.Started ?? false; + } else if ( + dockerInspect.State.Restarting === true + && dockerInspect.State.ExitCode !== 0 + ) { + ctx.servicesFailed.push({ + service, + message: dockerInspect.State.Error, + code: dockerInspect.State.ExitCode, + }); + } else if (dockerInspect.State.OOMKilled === true) { + ctx.servicesOOMKilled.push({ + service, + }); } else { return; } throw new Error(service.title); }, - }; - }), + } + )), { exitOnError: false, }, @@ -82,6 +202,56 @@ export default function analyseSamplesTaskFactory(dockerCompose, getServiceList) ctx.problems.push(problem); } + + if (ctx.servicesFailed.length > 0) { + let problem; + if (ctx.servicesFailed.length === 1) { + const failedService = ctx.servicesFailed[0]; + + problem = chalk`Service ${failedService.service.title} failed with an error code ${failedService.code}`; + + if (failedService.message?.length > 0) { + problem += `and message: ${failedService.message}`; + } + + problem += '.'; + } else { + problem = chalk`${ctx.servicesFailed.length} services failed:`; + + ctx.servicesFailed.map((failedService) => { + let output = chalk` ${failedService.service.title} failed with an error code ${failedService.code}`; + + if (failedService.message?.length > 0) { + output += `and message: ${failedService.message}`; + } + + output += '.'; + + return output; + }).join('\n'); + } + + problem += chalk`\n\nPlease check corresponding logs or share them with Dash Core Group`; + + ctx.problems.push(problem); + } + + if (ctx.servicesOOMKilled.length > 0) { + let problem; + if (ctx.servicesNotStarted.length === 1) { + problem = chalk`Service ${ctx.servicesNotStarted[0].service.title} is killed due to lack of memory.`; + } else { + problem = chalk`Services ${ctx.servicesNotStarted.map((e) => e.service.title).join(', ')} aren't killed due to lack of memory.`; + } + + problem += chalk`\n\nMake sure you have enough memory to run the node.`; + + ctx.problems.push(problem); + } + + if (ctx.systemResourceProblems.length > 0) { + + } }, }, // TODO: dont have priavate ky to sign diff --git a/packages/dashmate/src/listr/tasks/setup/regular/verifySystemRequirementsTaskFactory.js b/packages/dashmate/src/listr/tasks/setup/regular/verifySystemRequirementsTaskFactory.js index 3e142af4165..37e115bfb18 100644 --- a/packages/dashmate/src/listr/tasks/setup/regular/verifySystemRequirementsTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/setup/regular/verifySystemRequirementsTaskFactory.js @@ -6,12 +6,14 @@ import { Listr } from 'listr2'; * @param {Docker} docker * @param {DockerCompose} dockerCompose * @param {getOperatingSystemInfo} getOperatingSystemInfo + * @param {verifySystemRequirements} verifySystemRequirements * @return {verifySystemRequirementsTask} */ export default function verifySystemRequirementsTaskFactory( docker, dockerCompose, getOperatingSystemInfo, + verifySystemRequirements, ) { /** * @typedef {function} verifySystemRequirementsTask @@ -24,72 +26,11 @@ export default function verifySystemRequirementsTaskFactory( task: async (ctx, task) => { await dockerCompose.throwErrorIfNotInstalled(); - const MINIMUM_CPU_CORES = ctx.isHP ? 4 : 2; - const MINIMUM_CPU_FREQUENCY = 2.4; // GHz - const MINIMUM_RAM = ctx.isHP ? 8 : 4; // GB - const MINIMUM_DISK_SPACE = ctx.isHP ? 200 : 100; // GB + const systemInfo = await getOperatingSystemInfo(); - const warnings = []; - - const { - dockerSystemInfo, cpu, memory, diskSpace, - } = await getOperatingSystemInfo(); - - if (dockerSystemInfo) { - if (Number.isInteger(dockerSystemInfo.NCPU)) { - // Check CPU cores - const cpuCores = dockerSystemInfo.NCPU; - - if (cpuCores < MINIMUM_CPU_CORES) { - warnings.push(`${cpuCores} CPU cores detected. At least ${MINIMUM_CPU_CORES} are required`); - } - } else { - // eslint-disable-next-line no-console - console.warn('Can\'t get NCPU from docker info'); - } - - // Check RAM - if (Number.isInteger(dockerSystemInfo.MemTotal)) { - const memoryGb = dockerSystemInfo.MemTotal / (1024 ** 3); // Convert to GB - - if (memoryGb < MINIMUM_RAM) { - warnings.push(`${memoryGb.toFixed(2)}GB RAM detected. At least ${MINIMUM_RAM}GB is required`); - } - } else { - // eslint-disable-next-line no-console - console.warn('Can\'t get MemTotal from docker info'); - } - } - - // Check CPU frequency - if (cpu) { - if (cpu.speed === 0) { - if (process.env.DEBUG) { - // eslint-disable-next-line no-console - console.warn('Can\'t get CPU frequency'); - } - } else if (cpu.speed < MINIMUM_CPU_FREQUENCY) { - warnings.push(`${cpu.speed.toFixed(1)}GHz CPU frequency detected. At least ${MINIMUM_CPU_FREQUENCY}GHz is required`); - } - } - - // Check swap information - if (memory) { - const swapTotalGb = (memory.swaptotal / (1024 ** 3)); // Convert bytes to GB - - if (swapTotalGb < 2) { - warnings.push(`Swap space is ${swapTotalGb.toFixed(2)}GB. 2GB is recommended`); - } - } - - // Get disk usage info - if (diskSpace) { - const availableDiskSpace = diskSpace.available / (1024 ** 3); // Convert to GB - - if (availableDiskSpace < MINIMUM_DISK_SPACE) { - warnings.push(`${availableDiskSpace.toFixed(2)}GB available disk space detected. At least ${MINIMUM_DISK_SPACE}GB is required`); - } - } + const warnings = Object.values( + verifySystemRequirements(systemInfo, ctx.isHP), + ); let message = ''; if (ctx.isHP) {