From ede18421e2b2169848a2189cacb6162503ae142d Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Wed, 4 Sep 2024 08:53:27 +0100 Subject: [PATCH] feat: use k8s secrets to store node gossip and grpc tls keys and mount them (#499) Signed-off-by: Jeromy Cannon --- .github/workflows/flow-build-application.yaml | 1 - .../workflows/flow-pull-request-checks.yaml | 1 - src/commands/network.mjs | 96 ++- src/commands/node.mjs | 650 ++++++------------ src/core/k8.mjs | 36 +- src/core/key_manager.mjs | 195 +++++- src/core/platform_installer.mjs | 136 ++-- src/core/templates.mjs | 10 + test/e2e/core/account_manager.test.mjs | 2 + test/e2e/core/k8_e2e.test.mjs | 39 +- test/e2e/core/platform_installer_e2e.test.mjs | 63 +- test/test_util.js | 20 +- test/unit/core/platform_installer.test.mjs | 2 +- version.mjs | 2 +- 14 files changed, 649 insertions(+), 604 deletions(-) diff --git a/.github/workflows/flow-build-application.yaml b/.github/workflows/flow-build-application.yaml index 5ec7c6c38..786002730 100644 --- a/.github/workflows/flow-build-application.yaml +++ b/.github/workflows/flow-build-application.yaml @@ -84,7 +84,6 @@ jobs: needs: - env-vars - code-style - - e2e-tests with: custom-job-label: Mirror Node npm-test-script: test-${{ needs.env-vars.outputs.e2e-mirror-node-test-subdir }} diff --git a/.github/workflows/flow-pull-request-checks.yaml b/.github/workflows/flow-pull-request-checks.yaml index 14940cbe9..831fa33fc 100644 --- a/.github/workflows/flow-pull-request-checks.yaml +++ b/.github/workflows/flow-pull-request-checks.yaml @@ -71,7 +71,6 @@ jobs: needs: - env-vars - code-style - - e2e-tests with: custom-job-label: Mirror Node npm-test-script: test-${{ needs.env-vars.outputs.e2e-mirror-node-test-subdir }} diff --git a/src/commands/network.mjs b/src/commands/network.mjs index e1db35598..e1eb458bc 100644 --- a/src/commands/network.mjs +++ b/src/commands/network.mjs @@ -20,17 +20,25 @@ import { Listr } from 'listr2' import { FullstackTestingError, IllegalArgumentError, MissingArgumentError } from '../core/errors.mjs' import { BaseCommand } from './base.mjs' import * as flags from './flags.mjs' -import { constants } from '../core/index.mjs' +import { constants, Templates } from '../core/index.mjs' import * as prompts from './prompts.mjs' import * as helpers from '../core/helpers.mjs' import path from 'path' +import { validatePath } from '../core/helpers.mjs' +import fs from 'fs' export class NetworkCommand extends BaseCommand { constructor (opts) { super(opts) + if (!opts || !opts.k8) throw new Error('An instance of core/K8 is required') + if (!opts || !opts.keyManager) throw new IllegalArgumentError('An instance of core/KeyManager is required', opts.keyManager) + if (!opts || !opts.platformInstaller) throw new IllegalArgumentError('An instance of core/PlatformInstaller is required', opts.platformInstaller) if (!opts || !opts.profileManager) throw new MissingArgumentError('An instance of core/ProfileManager is required', opts.downloader) + this.k8 = opts.k8 + this.keyManager = opts.keyManager + this.platformInstaller = opts.platformInstaller this.profileManager = opts.profileManager } @@ -45,6 +53,7 @@ export class NetworkCommand extends BaseCommand { flags.applicationEnv, flags.applicationProperties, flags.bootstrapProperties, + flags.cacheDir, flags.chainId, flags.chartDirectory, flags.deployHederaExplorer, @@ -54,6 +63,7 @@ export class NetworkCommand extends BaseCommand { flags.fstChartVersion, flags.hederaExplorerTlsHostName, flags.hederaExplorerTlsLoadBalancerIp, + flags.keyFormat, flags.log4j2Xml, flags.namespace, flags.nodeIDs, @@ -143,10 +153,12 @@ export class NetworkCommand extends BaseCommand { flags.applicationEnv, flags.applicationProperties, flags.bootstrapProperties, + flags.cacheDir, flags.chainId, flags.deployHederaExplorer, flags.deployMirrorNode, flags.hederaExplorerTlsLoadBalancerIp, + flags.keyFormat, flags.log4j2Xml, flags.persistentVolumeClaims, flags.profileName, @@ -160,6 +172,7 @@ export class NetworkCommand extends BaseCommand { * @typedef {Object} NetworkDeployConfigClass * -- flags -- * @property {string} applicationEnv + * @property {string} cacheDir * @property {string} chartDirectory * @property {boolean} deployHederaExplorer * @property {boolean} deployMirrorNode @@ -168,6 +181,7 @@ export class NetworkCommand extends BaseCommand { * @property {string} fstChartVersion * @property {string} hederaExplorerTlsHostName * @property {string} hederaExplorerTlsLoadBalancerIp + * @property {string} keyFormat * @property {string} namespace * @property {string} nodeIDs * @property {string} persistentVolumeClaims @@ -176,8 +190,11 @@ export class NetworkCommand extends BaseCommand { * @property {string} releaseTag * @property {string} tlsClusterIssuerType * -- extra args -- - * @property {string[]} nodeIds * @property {string} chartPath + * @property {string} keysDir + * @property {string[]} nodeIds + * @property {string} stagingDir + * @property {string} stagingKeysDir * @property {string} valuesArg * -- methods -- * @property {getUnusedConfigs} getUnusedConfigs @@ -189,7 +206,14 @@ export class NetworkCommand extends BaseCommand { // create a config object for subsequent steps const config = /** @type {NetworkDeployConfigClass} **/ this.getConfig(NetworkCommand.DEPLOY_CONFIGS_NAME, NetworkCommand.DEPLOY_FLAGS_LIST, - ['nodeIds', 'chartPath', 'valuesArg']) + [ + 'chartPath', + 'keysDir', + 'nodeIds', + 'stagingDir', + 'stagingKeysDir', + 'valuesArg' + ]) config.nodeIds = helpers.parseNodeIds(config.nodeIDs) @@ -199,6 +223,28 @@ export class NetworkCommand extends BaseCommand { config.valuesArg = await this.prepareValuesArg(config) + // compute other config parameters + config.keysDir = path.join(validatePath(config.cacheDir), 'keys') + config.stagingDir = Templates.renderStagingDir( + config.cacheDir, + config.releaseTag + ) + config.stagingKeysDir = path.join(validatePath(config.stagingDir), 'keys') + + if (!await this.k8.hasNamespace(config.namespace)) { + await this.k8.createNamespace(config.namespace) + } + + // prepare staging keys directory + if (!fs.existsSync(config.stagingKeysDir)) { + fs.mkdirSync(config.stagingKeysDir, { recursive: true }) + } + + // create cached keys dir if it does not exist yet + if (!fs.existsSync(config.keysDir)) { + fs.mkdirSync(config.keysDir) + } + this.logger.debug('Prepared config', { config, cachedConfig: this.configManager.config @@ -221,6 +267,50 @@ export class NetworkCommand extends BaseCommand { ctx.config = /** @type {NetworkDeployConfigClass} **/ await self.prepareConfig(task, argv) } }, + { + title: 'Prepare staging directory', + task: async (ctx, parentTask) => { + const subTasks = [ + { + title: 'Copy Gossip keys to staging', + task: async (ctx, _) => { + const config = /** @type {NetworkDeployConfigClass} **/ ctx.config + + await this.keyManager.copyGossipKeysToStaging(config.keyFormat, config.keysDir, config.stagingKeysDir, config.nodeIds) + } + }, + { + title: 'Copy gRPC TLS keys to staging', + task: async (ctx, _) => { + const config = /** @type {NetworkDeployConfigClass} **/ ctx.config + for (const nodeId of config.nodeIds) { + const tlsKeyFiles = self.keyManager.prepareTLSKeyFilePaths(nodeId, config.keysDir) + await self.keyManager.copyNodeKeysToStaging(tlsKeyFiles, config.stagingKeysDir) + } + } + } + ] + + return parentTask.newListr(subTasks, { + concurrent: false, + rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION + }) + } + }, + { + title: 'Copy node keys to secrets', + task: async (ctx, parentTask) => { + const config = /** @type {NetworkDeployConfigClass} **/ ctx.config + + const subTasks = self.platformInstaller.copyNodeKeys(config.stagingDir, config.nodeIds, config.keyFormat) + + // set up the sub-tasks + return parentTask.newListr(subTasks, { + concurrent: true, + rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION + }) + } + }, { title: `Install chart '${constants.FULLSTACK_DEPLOYMENT_CHART}'`, task: async (ctx, _) => { diff --git a/src/commands/node.mjs b/src/commands/node.mjs index 3a3dc0b27..eb3badce8 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -24,7 +24,7 @@ import * as helpers from '../core/helpers.mjs' import { getNodeAccountMap, getNodeLogs, - getTmpDir, renameAndCopyFile, + renameAndCopyFile, sleep, validatePath } from '../core/helpers.mjs' @@ -87,10 +87,6 @@ export class NodeCommand extends BaseCommand { flags.appConfig, flags.cacheDir, flags.devMode, - flags.force, - flags.generateGossipKeys, - flags.generateTlsKeys, - flags.keyFormat, flags.localBuildPath, flags.namespace, flags.nodeIDs, @@ -119,10 +115,9 @@ export class NodeCommand extends BaseCommand { static get REFRESH_FLAGS_LIST () { return [ + flags.app, flags.cacheDir, flags.devMode, - flags.force, - flags.keyFormat, flags.localBuildPath, flags.namespace, flags.nodeIDs, @@ -142,7 +137,6 @@ export class NodeCommand extends BaseCommand { flags.chartDirectory, flags.devMode, flags.endpointType, - flags.force, flags.fstChartVersion, flags.generateGossipKeys, flags.generateTlsKeys, @@ -365,163 +359,6 @@ export class NodeCommand extends BaseCommand { }) } - /** - * Return a list of subtasks to generate gossip keys - * - * WARNING: These tasks MUST run in sequence. - * - * @param keyFormat key format (pem | pfx) - * @param nodeIds node ids - * @param keysDir keys directory - * @param curDate current date - * @param allNodeIds includes the nodeIds to get new keys as well as existing nodeIds that will be included in the public.pfx file - * @return a list of subtasks - * @private - */ - _nodeGossipKeysTaskList (keyFormat, nodeIds, keysDir, curDate = new Date(), allNodeIds = null) { - allNodeIds = allNodeIds || nodeIds - if (!Array.isArray(nodeIds) || !nodeIds.every((nodeId) => typeof nodeId === 'string')) { - throw new IllegalArgumentError('nodeIds must be an array of strings') - } - const self = this - const subTasks = [] - - switch (keyFormat) { - case constants.KEY_FORMAT_PFX: { - const tmpDir = getTmpDir() - const keytool = self.keytoolDepManager.getKeytool() - - subTasks.push({ - title: `Check keytool exists (Version: ${self.keytoolDepManager.getKeytoolVersion()})`, - task: async () => self.keytoolDepManager.checkVersion(true) - - }) - - subTasks.push({ - title: 'Backup old files', - task: () => helpers.backupOldPfxKeys(nodeIds, keysDir, curDate) - }) - - for (const nodeId of nodeIds) { - subTasks.push({ - title: `Generate ${Templates.renderGossipPfxPrivateKeyFile(nodeId)} for node: ${chalk.yellow(nodeId)}`, - task: async () => { - const privatePfxFile = await self.keyManager.generatePrivatePfxKeys(keytool, nodeId, keysDir, tmpDir) - const output = await keytool.list(`-storetype pkcs12 -storepass password -keystore ${privatePfxFile}`) - if (!output.includes('Your keystore contains 3 entries')) { - throw new FullstackTestingError(`malformed private pfx file: ${privatePfxFile}`) - } - } - }) - } - - subTasks.push({ - title: `Generate ${constants.PUBLIC_PFX} file`, - task: async () => { - const publicPfxFile = await self.keyManager.updatePublicPfxKey(self.keytoolDepManager.getKeytool(), allNodeIds, keysDir, tmpDir) - const output = await keytool.list(`-storetype pkcs12 -storepass password -keystore ${publicPfxFile}`) - if (!output.includes(`Your keystore contains ${allNodeIds.length * 3} entries`)) { - throw new FullstackTestingError(`malformed public.pfx file: ${publicPfxFile}`) - } - } - }) - - subTasks.push({ - title: 'Clean up temp files', - task: async () => { - if (fs.existsSync(tmpDir)) { - fs.rmSync(tmpDir, { recursive: true }) - } - } - }) - break - } - - case constants.KEY_FORMAT_PEM: { - subTasks.push({ - title: 'Backup old files', - task: () => helpers.backupOldPemKeys(nodeIds, keysDir, curDate) - } - ) - - for (const nodeId of nodeIds) { - subTasks.push({ - title: `Gossip ${keyFormat} key for node: ${chalk.yellow(nodeId)}`, - task: async () => { - const signingKey = await this.keyManager.generateSigningKey(nodeId) - const signingKeyFiles = await this.keyManager.storeSigningKey(nodeId, signingKey, keysDir) - this.logger.debug(`generated Gossip signing keys for node ${nodeId}`, { keyFiles: signingKeyFiles }) - - const agreementKey = await this.keyManager.generateAgreementKey(nodeId, signingKey) - const agreementKeyFiles = await this.keyManager.storeAgreementKey(nodeId, agreementKey, keysDir) - this.logger.debug(`generated Gossip agreement keys for node ${nodeId}`, { keyFiles: agreementKeyFiles }) - } - }) - } - - break - } - - default: - throw new FullstackTestingError(`unsupported key-format: ${keyFormat}`) - } - - return subTasks - } - - /** - * Return a list of subtasks to generate gRPC TLS keys - * - * WARNING: These tasks should run in sequence - * - * @param nodeIds node ids - * @param keysDir keys directory - * @param curDate current date - * @return return a list of subtasks - * @private - */ - _nodeTlsKeyTaskList (nodeIds, keysDir, curDate = new Date()) { - // check if nodeIds is an array of strings - if (!Array.isArray(nodeIds) || !nodeIds.every((nodeId) => typeof nodeId === 'string')) { - throw new FullstackTestingError('nodeIds must be an array of strings') - } - const self = this - const nodeKeyFiles = new Map() - const subTasks = [] - - subTasks.push({ - title: 'Backup old files', - task: () => helpers.backupOldTlsKeys(nodeIds, keysDir, curDate) - } - ) - - for (const nodeId of nodeIds) { - subTasks.push({ - title: `TLS key for node: ${chalk.yellow(nodeId)}`, - task: async () => { - const tlsKey = await self.keyManager.generateGrpcTLSKey(nodeId) - const tlsKeyFiles = await self.keyManager.storeTLSKey(nodeId, tlsKey, keysDir) - nodeKeyFiles.set(nodeId, { - tlsKeyFiles - }) - } - }) - } - - return subTasks - } - - async _copyNodeKeys (nodeKey, destDir) { - for (const keyFile of [nodeKey.privateKeyFile, nodeKey.certificateFile]) { - if (!fs.existsSync(keyFile)) { - throw new FullstackTestingError(`file (${keyFile}) is missing`) - } - - const fileName = path.basename(keyFile) - fs.cpSync(keyFile, path.join(destDir, fileName)) - } - } - async initializeSetup (config, k8) { // compute other config parameters config.keysDir = path.join(validatePath(config.cacheDir), 'keys') @@ -751,7 +588,6 @@ export class NodeCommand extends BaseCommand { prompts.disablePrompts([ flags.appConfig, flags.devMode, - flags.force, flags.localBuildPath ]) @@ -764,21 +600,13 @@ export class NodeCommand extends BaseCommand { * @property {string} appConfig * @property {string} cacheDir * @property {boolean} devMode - * @property {boolean} force - * @property {boolean} generateGossipKeys - * @property {boolean} generateTlsKeys - * @property {string} keyFormat * @property {string} localBuildPath * @property {string} namespace * @property {string} nodeIDs * @property {string} releaseTag * -- extra args -- - * @property {Date} curDate - * @property {string} keysDir * @property {string[]} nodeIds * @property {string[]} podNames - * @property {string} stagingDir - * @property {string} stagingKeysDir * -- methods -- * @property {getUnusedConfigs} getUnusedConfigs */ @@ -790,16 +618,11 @@ export class NodeCommand extends BaseCommand { // create a config object for subsequent steps const config = /** @type {NodeSetupConfigClass} **/ this.getConfig(NodeCommand.SETUP_CONFIGS_NAME, NodeCommand.SETUP_FLAGS_LIST, [ - 'curDate', - 'keysDir', 'nodeIds', - 'podNames', - 'stagingDir', - 'stagingKeysDir' + 'podNames' ]) config.nodeIds = helpers.parseNodeIds(config.nodeIDs) - config.curDate = new Date() await self.initializeSetup(config, self.k8) @@ -813,68 +636,6 @@ export class NodeCommand extends BaseCommand { title: 'Identify network pods', task: (ctx, task) => self.taskCheckNetworkNodePods(ctx, task, ctx.config.nodeIds) }, - { - title: 'Generate Gossip keys', - task: async (ctx, parentTask) => { - const config = /** @type {NodeSetupConfigClass} **/ ctx.config - - const subTasks = self._nodeGossipKeysTaskList(config.keyFormat, config.nodeIds, config.keysDir, config.curDate) - // set up the sub-tasks - return parentTask.newListr(subTasks, { - concurrent: false, - rendererOptions: { - collapseSubtasks: false, - timer: constants.LISTR_DEFAULT_RENDERER_TIMER_OPTION - } - }) - }, - skip: (ctx, _) => !ctx.config.generateGossipKeys - }, - { - title: 'Generate gRPC TLS keys', - task: async (ctx, parentTask) => { - const config = ctx.config - const subTasks = self._nodeTlsKeyTaskList(config.nodeIds, config.keysDir, config.curDate) - // set up the sub-tasks - return parentTask.newListr(subTasks, { - concurrent: false, - rendererOptions: { - collapseSubtasks: false, - timer: constants.LISTR_DEFAULT_RENDERER_TIMER_OPTION - } - }) - }, - skip: (ctx, _) => !ctx.config.generateTlsKeys - }, - { - title: 'Prepare staging directory', - task: async (ctx, parentTask) => { - const subTasks = [ - { - title: 'Copy Gossip keys to staging', - task: async (ctx, _) => { - const config = /** @type {NodeSetupConfigClass} **/ ctx.config - await this.copyGossipKeysToStaging(config.keyFormat, config.keysDir, config.stagingKeysDir, ctx.config.nodeIds) - } - }, - { - title: 'Copy gRPC TLS keys to staging', - task: async (ctx, _) => { - for (const nodeId of ctx.config.nodeIds) { - const config = /** @type {NodeSetupConfigClass} **/ ctx.config - const tlsKeyFiles = self.keyManager.prepareTLSKeyFilePaths(nodeId, config.keysDir) - await self._copyNodeKeys(tlsKeyFiles, config.stagingKeysDir) - } - } - } - ] - - return parentTask.newListr(subTasks, { - concurrent: false, - rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION - }) - } - }, { title: 'Fetch platform software into network nodes', task: @@ -892,12 +653,7 @@ export class NodeCommand extends BaseCommand { subTasks.push({ title: `Node: ${chalk.yellow(nodeId)}`, task: () => - self.platformInstaller.taskInstall( - podName, - ctx.config.stagingDir, - ctx.config.nodeIds, - ctx.config.keyFormat, - ctx.config.force) + self.platformInstaller.taskSetup(podName) }) } @@ -907,15 +663,6 @@ export class NodeCommand extends BaseCommand { rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION }) } - }, - { - title: 'Finalize', - task: (ctx, _) => { - // reset flags so that keys are not regenerated later - self.configManager.setFlag(flags.generateGossipKeys, false) - self.configManager.setFlag(flags.generateTlsKeys, false) - self.configManager.persist() - } } ], { concurrent: false, @@ -1171,7 +918,7 @@ export class NodeCommand extends BaseCommand { */ // create a config object for subsequent steps - const config = /** @type {NodeKeysConfigClass} **/ this.getConfig(NodeCommand.SETUP_CONFIGS_NAME, NodeCommand.SETUP_FLAGS_LIST, + const config = /** @type {NodeKeysConfigClass} **/ this.getConfig(NodeCommand.KEYS_CONFIGS_NAME, NodeCommand.KEYS_FLAGS_LIST, [ 'curDate', 'keysDir', @@ -1193,7 +940,7 @@ export class NodeCommand extends BaseCommand { title: 'Generate gossip keys', task: async (ctx, parentTask) => { const config = ctx.config - const subTasks = self._nodeGossipKeysTaskList(config.keyFormat, config.nodeIds, config.keysDir, config.curDate) + const subTasks = self.keyManager.taskGenerateGossipKeys(self.keytoolDepManager, config.keyFormat, config.nodeIds, config.keysDir, config.curDate) // set up the sub-tasks return parentTask.newListr(subTasks, { concurrent: false, @@ -1209,7 +956,7 @@ export class NodeCommand extends BaseCommand { title: 'Generate gRPC TLS keys', task: async (ctx, parentTask) => { const config = ctx.config - const subTasks = self._nodeTlsKeyTaskList(config.nodeIds, config.keysDir, config.curDate) + const subTasks = self.keyManager.taskGenerateTLSKeys(config.nodeIds, config.keysDir, config.curDate) // set up the sub-tasks return parentTask.newListr(subTasks, { concurrent: true, @@ -1251,8 +998,8 @@ export class NodeCommand extends BaseCommand { self.configManager.update(argv) // disable the prompts that we don't want to prompt the user for prompts.disablePrompts([ + flags.app, flags.devMode, - flags.force, flags.localBuildPath ]) @@ -1261,20 +1008,16 @@ export class NodeCommand extends BaseCommand { /** * @typedef {Object} NodeRefreshConfigClass * -- flags -- + * @property {string} app * @property {string} cacheDir * @property {boolean} devMode - * @property {boolean} force - * @property {string} keyFormat * @property {string} localBuildPath * @property {string} namespace * @property {string} nodeIDs * @property {string} releaseTag * -- extra args -- - * @property {string} keysDir * @property {string[]} nodeIds * @property {Object} podNames - * @property {string} stagingDir - * @property {string} stagingKeysDir * -- methods -- * @property {getUnusedConfigs} getUnusedConfigs */ @@ -1286,11 +1029,8 @@ export class NodeCommand extends BaseCommand { // create a config object for subsequent steps ctx.config = /** @type {NodeRefreshConfigClass} **/ this.getConfig(NodeCommand.REFRESH_CONFIGS_NAME, NodeCommand.REFRESH_FLAGS_LIST, [ - 'keysDir', 'nodeIds', - 'podNames', - 'stagingDir', - 'stagingKeysDir' + 'podNames' ]) ctx.config.nodeIds = helpers.parseNodeIds(ctx.config.nodeIDs) @@ -1308,9 +1048,10 @@ export class NodeCommand extends BaseCommand { title: 'Dump network nodes saved state', task: async (ctx, task) => { + const config = /** @type {NodeRefreshConfigClass} **/ ctx.config const subTasks = [] - for (const nodeId of ctx.config.nodeIds) { - const podName = ctx.config.podNames[nodeId] + for (const nodeId of config.nodeIds) { + const podName = config.podNames[nodeId] subTasks.push({ title: `Node: ${chalk.yellow(nodeId)}`, task: async () => @@ -1338,21 +1079,16 @@ export class NodeCommand extends BaseCommand { { title: 'Setup network nodes', task: async (ctx, parentTask) => { - const config = ctx.config + const config = /** @type {NodeRefreshConfigClass} **/ ctx.config const subTasks = [] - const nodeList = [] - const networkNodeServicesMap = await self.accountManager.getNodeServiceMap(ctx.config.namespace) - for (const networkNodeServices of networkNodeServicesMap.values()) { - nodeList.push(networkNodeServices.nodeName) - } for (const nodeId of config.nodeIds) { const podName = config.podNames[nodeId] subTasks.push({ title: `Node: ${chalk.yellow(nodeId)}`, task: () => - self.platformInstaller.taskInstall(podName, config.stagingDir, nodeList, config.keyFormat, config.force) + self.platformInstaller.taskSetup(podName) }) } @@ -1363,20 +1099,12 @@ export class NodeCommand extends BaseCommand { }) } }, - { - title: 'Finalize', - task: (ctx, _) => { - // reset flags so that keys are not regenerated later - self.configManager.setFlag(flags.generateGossipKeys, false) - self.configManager.setFlag(flags.generateTlsKeys, false) - self.configManager.persist() - } - }, { title: 'Starting nodes', task: (ctx, task) => { + const config = /** @type {NodeRefreshConfigClass} **/ ctx.config const subTasks = [] - self.startNodes(ctx.config.podNames, ctx.config.nodeIds, subTasks) + self.startNodes(config.podNames, config.nodeIds, subTasks) // set up the sub-tasks return task.newListr(subTasks, { @@ -1391,9 +1119,10 @@ export class NodeCommand extends BaseCommand { { title: 'Check nodes are ACTIVE', task: (ctx, task) => { + const config = /** @type {NodeRefreshConfigClass} **/ ctx.config const subTasks = [] for (const nodeId of ctx.config.nodeIds) { - if (ctx.config.app !== '' && ctx.config.app !== constants.HEDERA_APP_NAME) { + if (config.app !== '' && config.app !== constants.HEDERA_APP_NAME) { subTasks.push({ title: `Check node: ${chalk.yellow(nodeId)}`, task: () => self.checkNetworkNodeState(nodeId, 100, 'ACTIVE', 'output/swirlds.log') @@ -1509,7 +1238,6 @@ export class NodeCommand extends BaseCommand { flags.chartDirectory, flags.devMode, flags.endpointType, - flags.force, flags.fstChartVersion, flags.localBuildPath, flags.gossipEndpoints, @@ -1527,7 +1255,6 @@ export class NodeCommand extends BaseCommand { * @property {string} chartDirectory * @property {boolean} devMode * @property {string} endpointType - * @property {boolean} force * @property {string} fstChartVersion * @property {boolean} generateGossipKeys * @property {boolean} generateTlsKeys @@ -1605,6 +1332,9 @@ export class NodeCommand extends BaseCommand { const treasuryAccountPrivateKey = treasuryAccount.privateKey config.treasuryKey = PrivateKey.fromStringED25519(treasuryAccountPrivateKey) + config.serviceMap = await self.accountManager.getNodeServiceMap( + config.namespace) + self.logger.debug('Initialized config', { config }) } }, @@ -1626,6 +1356,8 @@ export class NodeCommand extends BaseCommand { config.existingNodeIds.push(networkNodeServices.nodeName) } + config.allNodeIds = [...config.existingNodeIds, config.nodeId] + return self.taskCheckNetworkNodePods(ctx, task, config.existingNodeIds) } }, @@ -1657,7 +1389,7 @@ export class NodeCommand extends BaseCommand { title: 'Generate Gossip key', task: async (ctx, parentTask) => { const config = /** @type {NodeAddConfigClass} **/ ctx.config - const subTasks = self._nodeGossipKeysTaskList(config.keyFormat, [config.nodeId], config.keysDir, config.curDate, config.allNodeIds) + const subTasks = self.keyManager.taskGenerateGossipKeys(self.keytoolDepManager, config.keyFormat, [config.nodeId], config.keysDir, config.curDate, config.allNodeIds) // set up the sub-tasks return parentTask.newListr(subTasks, { concurrent: false, @@ -1673,7 +1405,7 @@ export class NodeCommand extends BaseCommand { title: 'Generate gRPC TLS key', task: async (ctx, parentTask) => { const config = /** @type {NodeAddConfigClass} **/ ctx.config - const subTasks = self._nodeTlsKeyTaskList([config.nodeId], config.keysDir, config.curDate) + const subTasks = self.keyManager.taskGenerateTLSKeys([config.nodeId], config.keysDir, config.curDate) // set up the sub-tasks return parentTask.newListr(subTasks, { concurrent: false, @@ -1814,8 +1546,8 @@ export class NodeCommand extends BaseCommand { await self.k8.copyFrom(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/config.txt`, config.stagingDir) const signedKeyFiles = (await self.k8.listDir(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, `${constants.HEDERA_HAPI_PATH}/data/upgrade/current`)).filter(file => file.name.startsWith(constants.SIGNING_KEY_PREFIX)) + await self.k8.execContainer(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, ['bash', '-c', `mkdir -p ${constants.HEDERA_HAPI_PATH}/data/keys_backup && cp ${constants.HEDERA_HAPI_PATH}/data/keys/..data/* ${constants.HEDERA_HAPI_PATH}/data/keys_backup/`]) for (const signedKeyFile of signedKeyFiles) { - await self.k8.execContainer(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, ['bash', '-c', `[[ ! -f "${constants.HEDERA_HAPI_PATH}/data/keys/${signedKeyFile.name}" ]] || cp ${constants.HEDERA_HAPI_PATH}/data/keys/${signedKeyFile.name} ${constants.HEDERA_HAPI_PATH}/data/keys/${signedKeyFile.name}.old`]) await self.k8.copyFrom(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/${signedKeyFile.name}`, `${config.keysDir}`) } } @@ -1827,6 +1559,50 @@ export class NodeCommand extends BaseCommand { await this.freezeUpgradeNetworkNodes(config.freezeAdminPrivateKey, ctx.upgradeZipHash, config.nodeClient) } }, + { + title: 'Prepare staging directory', + task: async (ctx, parentTask) => { + const subTasks = [ + { + title: 'Copy Gossip keys to staging', + task: async (ctx, _) => { + const config = /** @type {NodeAddConfigClass} **/ ctx.config + + await this.keyManager.copyGossipKeysToStaging(config.keyFormat, config.keysDir, config.stagingKeysDir, config.allNodeIds) + } + }, + { + title: 'Copy gRPC TLS keys to staging', + task: async (ctx, _) => { + const config = /** @type {NodeAddConfigClass} **/ ctx.config + for (const nodeId of config.allNodeIds) { + const tlsKeyFiles = self.keyManager.prepareTLSKeyFilePaths(nodeId, config.keysDir) + await self.keyManager.copyNodeKeysToStaging(tlsKeyFiles, config.stagingKeysDir) + } + } + } + ] + + return parentTask.newListr(subTasks, { + concurrent: false, + rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION + }) + } + }, + { + title: 'Copy node keys to secrets', + task: async (ctx, parentTask) => { + const config = /** @type {NodeAddConfigClass} **/ ctx.config + + const subTasks = self.platformInstaller.copyNodeKeys(config.stagingDir, config.allNodeIds, config.keyFormat) + + // set up the sub-tasks + return parentTask.newListr(subTasks, { + concurrent: true, + rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION + }) + } + }, { title: 'Check network nodes are frozen', task: (ctx, task) => { @@ -1880,8 +1656,6 @@ export class NodeCommand extends BaseCommand { valuesArg, config.fstChartVersion ) - - config.allNodeIds = [...config.existingNodeIds, config.nodeId] } }, { @@ -1921,36 +1695,6 @@ export class NodeCommand extends BaseCommand { }) } }, - { - title: 'Prepare staging directory', - task: async (ctx, parentTask) => { - const subTasks = [ - { - title: 'Copy Gossip keys to staging', - task: async (ctx, _) => { - const config = /** @type {NodeAddConfigClass} **/ ctx.config - - await this.copyGossipKeysToStaging(config.keyFormat, config.keysDir, config.stagingKeysDir, config.allNodeIds) - } - }, - { - title: 'Copy gRPC TLS keys to staging', - task: async (ctx, _) => { - const config = /** @type {NodeAddConfigClass} **/ ctx.config - for (const nodeId of config.allNodeIds) { - const tlsKeyFiles = self.keyManager.prepareTLSKeyFilePaths(nodeId, config.keysDir) - await self._copyNodeKeys(tlsKeyFiles, config.stagingKeysDir) - } - } - } - ] - - return parentTask.newListr(subTasks, { - concurrent: false, - rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION - }) - } - }, { title: 'Fetch platform software into all network nodes', task: @@ -1960,6 +1704,7 @@ export class NodeCommand extends BaseCommand { config.namespace) config.podNames[config.nodeId] = config.serviceMap.get( config.nodeId).nodePodName + return self.fetchLocalOrReleasedPlatformSoftware(config.allNodeIds, config.podNames, config.releaseTag, task, config.localBuildPath) } }, @@ -2001,7 +1746,7 @@ export class NodeCommand extends BaseCommand { subTasks.push({ title: `Node: ${chalk.yellow(nodeId)}`, task: () => - self.platformInstaller.taskInstall(podName, config.stagingDir, config.allNodeIds, config.keyFormat, config.force) + self.platformInstaller.taskSetup(podName) }) } @@ -2195,33 +1940,6 @@ export class NodeCommand extends BaseCommand { } } - async copyGossipKeysToStaging (keyFormat, keysDir, stagingKeysDir, nodeIds) { - // copy gossip keys to the staging - for (const nodeId of nodeIds) { - switch (keyFormat) { - case constants.KEY_FORMAT_PEM: { - const signingKeyFiles = this.keyManager.prepareNodeKeyFilePaths(nodeId, keysDir, constants.SIGNING_KEY_PREFIX) - await this._copyNodeKeys(signingKeyFiles, stagingKeysDir) - - // generate missing agreement keys - const agreementKeyFiles = this.keyManager.prepareNodeKeyFilePaths(nodeId, keysDir, constants.AGREEMENT_KEY_PREFIX) - await this._copyNodeKeys(agreementKeyFiles, stagingKeysDir) - break - } - - case constants.KEY_FORMAT_PFX: { - const privateKeyFile = Templates.renderGossipPfxPrivateKeyFile(nodeId) - fs.cpSync(path.join(keysDir, privateKeyFile), path.join(stagingKeysDir, privateKeyFile)) - fs.cpSync(path.join(keysDir, constants.PUBLIC_PFX), path.join(stagingKeysDir, constants.PUBLIC_PFX)) - break - } - - default: - throw new FullstackTestingError(`Unsupported key-format ${keyFormat}`) - } - } - } - // Command Definition /** * Return Yargs command definition for 'node' command @@ -2525,6 +2243,8 @@ export class NodeCommand extends BaseCommand { config.existingNodeIds.push(networkNodeServices.nodeName) } + config.allNodeIds = [...config.existingNodeIds] + return self.taskCheckNetworkNodePods(ctx, task, config.existingNodeIds) } }, @@ -2665,13 +2385,6 @@ export class NodeCommand extends BaseCommand { await this.prepareUpgradeNetworkNodes(config.freezeAdminPrivateKey, ctx.upgradeZipHash, config.nodeClient) } }, - { - title: 'Send freeze upgrade transaction', - task: async (ctx, task) => { - const config = /** @type {NodeUpdateConfigClass} **/ ctx.config - await this.freezeUpgradeNetworkNodes(config.freezeAdminPrivateKey, ctx.upgradeZipHash, config.nodeClient) - } - }, { title: 'Download generated files from an existing node', task: async (ctx, task) => { @@ -2682,12 +2395,63 @@ export class NodeCommand extends BaseCommand { await self.k8.copyFrom(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/config.txt`, config.stagingDir) const signedKeyFiles = (await self.k8.listDir(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, `${constants.HEDERA_HAPI_PATH}/data/upgrade/current`)).filter(file => file.name.startsWith(constants.SIGNING_KEY_PREFIX)) + await self.k8.execContainer(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, ['bash', '-c', `mkdir -p ${constants.HEDERA_HAPI_PATH}/data/keys_backup && cp ${constants.HEDERA_HAPI_PATH}/data/keys/..data/* ${constants.HEDERA_HAPI_PATH}/data/keys_backup/`]) for (const signedKeyFile of signedKeyFiles) { - await self.k8.execContainer(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, ['bash', '-c', `[[ ! -f "${constants.HEDERA_HAPI_PATH}/data/keys/${signedKeyFile.name}" ]] || cp ${constants.HEDERA_HAPI_PATH}/data/keys/${signedKeyFile.name} ${constants.HEDERA_HAPI_PATH}/data/keys/${signedKeyFile.name}.old`]) await self.k8.copyFrom(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/${signedKeyFile.name}`, `${config.keysDir}`) } } }, + { + title: 'Send freeze upgrade transaction', + task: async (ctx, task) => { + const config = /** @type {NodeUpdateConfigClass} **/ ctx.config + await this.freezeUpgradeNetworkNodes(config.freezeAdminPrivateKey, ctx.upgradeZipHash, config.nodeClient) + } + }, + { + title: 'Prepare staging directory', + task: async (ctx, parentTask) => { + const subTasks = [ + { + title: 'Copy Gossip keys to staging', + task: async (ctx, _) => { + const config = /** @type {NodeUpdateConfigClass} **/ ctx.config + + await this.keyManager.copyGossipKeysToStaging(config.keyFormat, config.keysDir, config.stagingKeysDir, config.allNodeIds) + } + }, + { + title: 'Copy gRPC TLS keys to staging', + task: async (ctx, _) => { + const config = /** @type {NodeUpdateConfigClass} **/ ctx.config + for (const nodeId of config.allNodeIds) { + const tlsKeyFiles = self.keyManager.prepareTLSKeyFilePaths(nodeId, config.keysDir) + await self.keyManager.copyNodeKeysToStaging(tlsKeyFiles, config.stagingKeysDir) + } + } + } + ] + + return parentTask.newListr(subTasks, { + concurrent: false, + rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION + }) + } + }, + { + title: 'Copy node keys to secrets', + task: async (ctx, parentTask) => { + const config = /** @type {NodeUpdateConfigClass} **/ ctx.config + + const subTasks = self.platformInstaller.copyNodeKeys(config.stagingDir, config.allNodeIds, config.keyFormat) + + // set up the sub-tasks + return parentTask.newListr(subTasks, { + concurrent: true, + rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION + }) + } + }, { title: 'Check network nodes are frozen', task: (ctx, task) => { @@ -2772,7 +2536,7 @@ export class NodeCommand extends BaseCommand { async (ctx, task) => { const subTasks = [] const config = /** @type {NodeUpdateConfigClass} **/ ctx.config - ctx.config.allNodeIds = ctx.config.existingNodeIds + // nodes for (const nodeId of config.allNodeIds) { subTasks.push({ @@ -2794,49 +2558,19 @@ export class NodeCommand extends BaseCommand { }) } }, - { - title: 'Prepare staging directory', - task: async (ctx, parentTask) => { - const subTasks = [ - { - title: 'Copy Gossip keys to staging', - task: async (ctx, _) => { - const config = /** @type {NodeUpdateConfigClass} **/ ctx.config - - await this.copyGossipKeysToStaging(config.keyFormat, config.keysDir, config.stagingKeysDir, config.allNodeIds) - } - }, - { - title: 'Copy gRPC TLS keys to staging', - task: async (ctx, _) => { - const config = /** @type {NodeUpdateConfigClass} **/ ctx.config - for (const nodeId of config.allNodeIds) { - const tlsKeyFiles = self.keyManager.prepareTLSKeyFilePaths(nodeId, config.keysDir) - await self._copyNodeKeys(tlsKeyFiles, config.stagingKeysDir) - } - } - } - ] - - return parentTask.newListr(subTasks, { - concurrent: false, - rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION - }) - } - }, { title: 'Fetch platform software into network nodes', task: async (ctx, task) => { // without this sleep, copy software from local build to container sometimes fail await sleep(15000) - ctx.config.allNodeIds = ctx.config.existingNodeIds + const config = /** @type {NodeUpdateConfigClass} **/ ctx.config return self.fetchLocalOrReleasedPlatformSoftware(config.allNodeIds, config.podNames, config.releaseTag, task, config.localBuildPath) } }, { - title: 'Setup new network node', + title: 'Setup network nodes', task: async (ctx, parentTask) => { const config = /** @type {NodeUpdateConfigClass} **/ ctx.config @@ -2846,7 +2580,7 @@ export class NodeCommand extends BaseCommand { subTasks.push({ title: `Node: ${chalk.yellow(nodeId)}`, task: () => - self.platformInstaller.taskInstall(podName, config.stagingDir, config.allNodeIds, config.keyFormat, config.force) + self.platformInstaller.taskSetup(podName) }) } @@ -2939,7 +2673,7 @@ export class NodeCommand extends BaseCommand { for (const nodeId of config.allNodeIds) { const accountId = accountMap.get(nodeId) config.nodeClient.setOperator(TREASURY_ACCOUNT_ID, config.treasuryKey) - self.logger.info(`Sending 1 tinybar to account: ${accountId}`) + self.logger.info(`Sending 1 HBAR to account: ${accountId}`) await this.accountManager.transferAmount(constants.TREASURY_ACCOUNT_ID, accountId, 1) } } @@ -2999,7 +2733,7 @@ export class NodeCommand extends BaseCommand { * -- flags -- * @property {string} app * @property {string} cacheDir - * @property {string} charDirectory + * @property {string} chartDirectory * @property {boolean} devMode * @property {string} endpointType * @property {string} fstChartVersion @@ -3115,9 +2849,9 @@ export class NodeCommand extends BaseCommand { try { const accountMap = getNodeAccountMap(config.existingNodeIds) - const deleteAccountId = accountMap.get(ctx.config.nodeId) - this.logger.debug(`Deleting node: ${ctx.config.nodeId} with account: ${deleteAccountId}`) - const nodeId = Templates.nodeNumberFromNodeId(ctx.config.nodeId) - 1 + const deleteAccountId = accountMap.get(config.nodeId) + this.logger.debug(`Deleting node: ${config.nodeId} with account: ${deleteAccountId}`) + const nodeId = Templates.nodeNumberFromNodeId(config.nodeId) - 1 const nodeDeleteTx = await new NodeDeleteTransaction() .setNodeId(nodeId) .freezeWith(config.nodeClient) @@ -3139,6 +2873,22 @@ export class NodeCommand extends BaseCommand { await this.prepareUpgradeNetworkNodes(config.freezeAdminPrivateKey, ctx.upgradeZipHash, config.nodeClient) } }, + { + title: 'Download generated files from an existing node', + task: async (ctx, task) => { + const config = /** @type {NodeDeleteConfigClass} **/ ctx.config + const node1FullyQualifiedPodName = Templates.renderNetworkPodName(config.existingNodeIds[0]) + + // copy the config.txt file from the node1 upgrade directory + await self.k8.copyFrom(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/config.txt`, config.stagingDir) + + const signedKeyFiles = (await self.k8.listDir(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, `${constants.HEDERA_HAPI_PATH}/data/upgrade/current`)).filter(file => file.name.startsWith(constants.SIGNING_KEY_PREFIX)) + await self.k8.execContainer(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, ['bash', '-c', `mkdir -p ${constants.HEDERA_HAPI_PATH}/data/keys_backup && cp ${constants.HEDERA_HAPI_PATH}/data/keys/..data/* ${constants.HEDERA_HAPI_PATH}/data/keys_backup/`]) + for (const signedKeyFile of signedKeyFiles) { + await self.k8.copyFrom(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/${signedKeyFile.name}`, `${config.keysDir}`) + } + } + }, { title: 'Send freeze upgrade transaction', task: async (ctx, task) => { @@ -3146,6 +2896,52 @@ export class NodeCommand extends BaseCommand { await this.freezeUpgradeNetworkNodes(config.freezeAdminPrivateKey, ctx.upgradeZipHash, config.nodeClient) } }, + { + title: 'Prepare staging directory', + task: async (ctx, parentTask) => { + const subTasks = [ + { + title: 'Copy Gossip keys to staging', + task: async (ctx, _) => { + const config = /** @type {NodeDeleteConfigClass} **/ ctx.config + + await this.keyManager.copyGossipKeysToStaging(config.keyFormat, config.keysDir, config.stagingKeysDir, config.existingNodeIds) + } + }, + { + title: 'Copy gRPC TLS keys to staging', + task: async (ctx, _) => { + const config = /** @type {NodeDeleteConfigClass} **/ ctx.config + for (const nodeId of config.existingNodeIds) { + const tlsKeyFiles = self.keyManager.prepareTLSKeyFilePaths(nodeId, config.keysDir) + await self.keyManager.copyNodeKeysToStaging(tlsKeyFiles, config.stagingKeysDir) + } + } + } + ] + + return parentTask.newListr(subTasks, { + concurrent: false, + rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION + }) + } + }, + { + title: 'Copy node keys to secrets', + task: async (ctx, parentTask) => { + const config = /** @type {NodeDeleteConfigClass} **/ ctx.config + + // remove nodeId from existingNodeIds + config.allNodeIds = config.existingNodeIds.filter(nodeId => nodeId !== ctx.config.nodeId) + const subTasks = self.platformInstaller.copyNodeKeys(config.stagingDir, config.allNodeIds, config.keyFormat) + + // set up the sub-tasks + return parentTask.newListr(subTasks, { + concurrent: true, + rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION + }) + } + }, { title: 'Check network nodes are frozen', task: (ctx, task) => { @@ -3167,16 +2963,6 @@ export class NodeCommand extends BaseCommand { }) } }, - { - title: 'Download new config.txt', - task: async (ctx, task) => { - const config = /** @type {NodeDeleteConfigClass} **/ ctx.config - const node1FullyQualifiedPodName = Templates.renderNetworkPodName(config.existingNodeIds[0]) - - // copy the config.txt file from the node1 upgrade directory - await self.k8.copyFrom(node1FullyQualifiedPodName, constants.ROOT_CONTAINER, `${constants.HEDERA_HAPI_PATH}/data/upgrade/current/config.txt`, config.stagingDir) - } - }, { title: 'Get node logs and configs', task: async (ctx, task) => { @@ -3227,9 +3013,6 @@ export class NodeCommand extends BaseCommand { const subTasks = [] const config = /** @type {NodeDeleteConfigClass} **/ ctx.config - // remove nodeId from existingNodeIds - ctx.config.allNodeIds = ctx.config.existingNodeIds.filter(nodeId => nodeId !== ctx.config.nodeId) - // nodes for (const nodeId of config.allNodeIds) { subTasks.push({ @@ -3251,36 +3034,6 @@ export class NodeCommand extends BaseCommand { }) } }, - { - title: 'Prepare staging directory', - task: async (ctx, parentTask) => { - const subTasks = [ - { - title: 'Copy Gossip keys to staging', - task: async (ctx, _) => { - const config = /** @type {NodeDeleteConfigClass} **/ ctx.config - - await this.copyGossipKeysToStaging(config.keyFormat, config.keysDir, config.stagingKeysDir, config.allNodeIds) - } - }, - { - title: 'Copy gRPC TLS keys to staging', - task: async (ctx, _) => { - const config = /** @type {NodeDeleteConfigClass} **/ ctx.config - for (const nodeId of config.allNodeIds) { - const tlsKeyFiles = self.keyManager.prepareTLSKeyFilePaths(nodeId, config.keysDir) - await self._copyNodeKeys(tlsKeyFiles, config.stagingKeysDir) - } - } - } - ] - - return parentTask.newListr(subTasks, { - concurrent: false, - rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION - }) - } - }, { title: 'Fetch platform software into all network nodes', task: @@ -3294,20 +3047,17 @@ export class NodeCommand extends BaseCommand { } }, { - title: 'Setup new network node', + title: 'Setup network nodes', task: async (ctx, parentTask) => { const config = /** @type {NodeDeleteConfigClass} **/ ctx.config - // remove nodeId from existingNodeIds - ctx.config.allNodeIds = ctx.config.existingNodeIds.filter(nodeId => nodeId !== ctx.config.nodeId) - const subTasks = [] for (const nodeId of config.allNodeIds) { const podName = config.podNames[nodeId] subTasks.push({ title: `Node: ${chalk.yellow(nodeId)}`, task: () => - self.platformInstaller.taskInstall(podName, config.stagingDir, config.allNodeIds, config.keyFormat, config.force) + self.platformInstaller.taskSetup(podName) }) } diff --git a/src/core/k8.mjs b/src/core/k8.mjs index ea822838e..aa51bd07a 100644 --- a/src/core/k8.mjs +++ b/src/core/k8.mjs @@ -522,10 +522,18 @@ export class K8 { const namespace = this._getNamespace() // get stat for source file in the container - const entries = await this.listDir(podName, containerName, srcPath) + let entries = await this.listDir(podName, containerName, srcPath) if (entries.length !== 1) { throw new FullstackTestingError(`invalid source path: ${srcPath}`) } + // handle symbolic link + if (entries[0].name.indexOf(' -> ') > -1) { + const redirectSrcPath = path.join(path.dirname(srcPath), entries[0].name.substring(entries[0].name.indexOf(' -> ') + 4)) + entries = await this.listDir(podName, containerName, redirectSrcPath) + if (entries.length !== 1) { + throw new FullstackTestingError(`invalid source path: ${redirectSrcPath}`) + } + } const srcFileDesc = entries[0] // cache for later comparison after copy if (!fs.existsSync(destDir)) { @@ -535,8 +543,8 @@ export class K8 { try { const srcFileSize = Number.parseInt(srcFileDesc.size) - const srcFile = path.basename(srcPath) - const srcDir = path.dirname(srcPath) + const srcFile = path.basename(entries[0].name) + const srcDir = path.dirname(entries[0].name) const destPath = path.join(destDir, srcFile) // download the tar file to a temp location @@ -570,17 +578,21 @@ export class K8 { return reject(new FullstackTestingError(`failed to copy because of error (${code}): ${reason}`)) } - // extract the downloaded file - await tar.x({ - file: tmpFile, - cwd: destDir - }) + try { + // extract the downloaded file + await tar.x({ + file: tmpFile, + cwd: destDir + }) - self._deleteTempFile(tmpFile) + self._deleteTempFile(tmpFile) - const stat = fs.statSync(destPath) - if (stat && stat.size === srcFileSize) { - return resolve(true) + const stat = fs.statSync(destPath) + if (stat && stat.size === srcFileSize) { + return resolve(true) + } + } catch (e) { + return reject(new FullstackTestingError(`failed to extract file: ${destPath}`, e)) } return reject(new FullstackTestingError(`failed to download file completely: ${destPath}`)) diff --git a/src/core/key_manager.mjs b/src/core/key_manager.mjs index c3381db5b..940233d31 100644 --- a/src/core/key_manager.mjs +++ b/src/core/key_manager.mjs @@ -18,11 +18,17 @@ import * as x509 from '@peculiar/x509' import crypto from 'crypto' import fs from 'fs' import path from 'path' -import { FullstackTestingError, MissingArgumentError } from './errors.mjs' +import { + FullstackTestingError, + IllegalArgumentError, + MissingArgumentError +} from './errors.mjs' import { getTmpDir } from './helpers.mjs' import { constants, Keytool } from './index.mjs' import { Logger } from './logging.mjs' import { Templates } from './templates.mjs' +import * as helpers from './helpers.mjs' +import chalk from 'chalk' x509.cryptoProvider.set(crypto) @@ -592,6 +598,11 @@ export class KeyManager { this.logger.debug(`Copying generated private pfx file: ${tmpPrivatePfxFile} -> ${privatePfxFile}`) fs.cpSync(tmpPrivatePfxFile, privatePfxFile) + const output = await keytool.list(`-storetype pkcs12 -storepass password -keystore ${privatePfxFile}`) + if (!output.includes('Your keystore contains 3 entries')) { + throw new FullstackTestingError(`malformed private pfx file: ${privatePfxFile}`) + } + return privatePfxFile } @@ -654,6 +665,188 @@ export class KeyManager { this.logger.debug(`Copying generated public.pfx file: ${tmpPublicPfxFile} -> ${publicPfxFile}`) fs.cpSync(tmpPublicPfxFile, publicPfxFile) + const output = await keytool.list(`-storetype pkcs12 -storepass password -keystore ${publicPfxFile}`) + if (!output.includes(`Your keystore contains ${nodeIds.length * 3} entries`)) { + throw new FullstackTestingError(`malformed public.pfx file: ${publicPfxFile}`) + } + return publicPfxFile } + + async copyNodeKeysToStaging (nodeKey, destDir) { + for (const keyFile of [nodeKey.privateKeyFile, nodeKey.certificateFile]) { + if (!fs.existsSync(keyFile)) { + throw new FullstackTestingError(`file (${keyFile}) is missing`) + } + + const fileName = path.basename(keyFile) + fs.cpSync(keyFile, path.join(destDir, fileName)) + } + } + + async copyGossipKeysToStaging (keyFormat, keysDir, stagingKeysDir, nodeIds) { + // copy gossip keys to the staging + for (const nodeId of nodeIds) { + switch (keyFormat) { + case constants.KEY_FORMAT_PEM: { + const signingKeyFiles = this.prepareNodeKeyFilePaths(nodeId, keysDir, constants.SIGNING_KEY_PREFIX) + await this.copyNodeKeysToStaging(signingKeyFiles, stagingKeysDir) + + // generate missing agreement keys + const agreementKeyFiles = this.prepareNodeKeyFilePaths(nodeId, keysDir, constants.AGREEMENT_KEY_PREFIX) + await this.copyNodeKeysToStaging(agreementKeyFiles, stagingKeysDir) + break + } + + case constants.KEY_FORMAT_PFX: { + const privateKeyFile = Templates.renderGossipPfxPrivateKeyFile(nodeId) + fs.cpSync(path.join(keysDir, privateKeyFile), path.join(stagingKeysDir, privateKeyFile)) + fs.cpSync(path.join(keysDir, constants.PUBLIC_PFX), path.join(stagingKeysDir, constants.PUBLIC_PFX)) + break + } + + default: + throw new FullstackTestingError(`Unsupported key-format ${keyFormat}`) + } + } + } + + /** + * Return a list of subtasks to generate gossip keys + * + * WARNING: These tasks MUST run in sequence. + * + * @param keytoolDepManager an instance of core/KeytoolDepManager + * @param keyFormat key format (pem | pfx) + * @param nodeIds node ids + * @param keysDir keys directory + * @param curDate current date + * @param allNodeIds includes the nodeIds to get new keys as well as existing nodeIds that will be included in the public.pfx file + * @return a list of subtasks + * @private + */ + taskGenerateGossipKeys (keytoolDepManager, keyFormat, nodeIds, keysDir, curDate = new Date(), allNodeIds = null) { + allNodeIds = allNodeIds || nodeIds + if (!Array.isArray(nodeIds) || !nodeIds.every((nodeId) => typeof nodeId === 'string')) { + throw new IllegalArgumentError('nodeIds must be an array of strings') + } + const self = this + const subTasks = [] + + switch (keyFormat) { + case constants.KEY_FORMAT_PFX: { + const tmpDir = getTmpDir() + const keytool = keytoolDepManager.getKeytool() + + subTasks.push({ + title: `Check keytool exists (Version: ${keytoolDepManager.getKeytoolVersion()})`, + task: async () => keytoolDepManager.checkVersion(true) + + }) + + subTasks.push({ + title: 'Backup old files', + task: () => helpers.backupOldPfxKeys(nodeIds, keysDir, curDate) + }) + + for (const nodeId of nodeIds) { + subTasks.push({ + title: `Generate ${Templates.renderGossipPfxPrivateKeyFile(nodeId)} for node: ${chalk.yellow(nodeId)}`, + task: async () => { + await self.generatePrivatePfxKeys(keytool, nodeId, keysDir, tmpDir) + } + }) + } + + subTasks.push({ + title: `Generate ${constants.PUBLIC_PFX} file`, + task: async () => { + await self.updatePublicPfxKey(keytool, allNodeIds, keysDir, tmpDir) + } + }) + + subTasks.push({ + title: 'Clean up temp files', + task: async () => { + if (fs.existsSync(tmpDir)) { + fs.rmSync(tmpDir, { recursive: true }) + } + } + }) + break + } + + case constants.KEY_FORMAT_PEM: { + subTasks.push({ + title: 'Backup old files', + task: () => helpers.backupOldPemKeys(nodeIds, keysDir, curDate) + } + ) + + for (const nodeId of nodeIds) { + subTasks.push({ + title: `Gossip ${keyFormat} key for node: ${chalk.yellow(nodeId)}`, + task: async () => { + const signingKey = await self.generateSigningKey(nodeId) + const signingKeyFiles = await self.storeSigningKey(nodeId, signingKey, keysDir) + this.logger.debug(`generated Gossip signing keys for node ${nodeId}`, { keyFiles: signingKeyFiles }) + + const agreementKey = await self.generateAgreementKey(nodeId, signingKey) + const agreementKeyFiles = await self.storeAgreementKey(nodeId, agreementKey, keysDir) + this.logger.debug(`generated Gossip agreement keys for node ${nodeId}`, { keyFiles: agreementKeyFiles }) + } + }) + } + + break + } + + default: + throw new FullstackTestingError(`unsupported key-format: ${keyFormat}`) + } + + return subTasks + } + + /** + * Return a list of subtasks to generate gRPC TLS keys + * + * WARNING: These tasks should run in sequence + * + * @param nodeIds node ids + * @param keysDir keys directory + * @param curDate current date + * @return return a list of subtasks + * @private + */ + taskGenerateTLSKeys (nodeIds, keysDir, curDate = new Date()) { + // check if nodeIds is an array of strings + if (!Array.isArray(nodeIds) || !nodeIds.every((nodeId) => typeof nodeId === 'string')) { + throw new FullstackTestingError('nodeIds must be an array of strings') + } + const self = this + const nodeKeyFiles = new Map() + const subTasks = [] + + subTasks.push({ + title: 'Backup old files', + task: () => helpers.backupOldTlsKeys(nodeIds, keysDir, curDate) + } + ) + + for (const nodeId of nodeIds) { + subTasks.push({ + title: `TLS key for node: ${chalk.yellow(nodeId)}`, + task: async () => { + const tlsKey = await self.generateGrpcTLSKey(nodeId) + const tlsKeyFiles = await self.storeTLSKey(nodeId, tlsKey, keysDir) + nodeKeyFiles.set(nodeId, { + tlsKeyFiles + }) + } + }) + } + + return subTasks + } } diff --git a/src/core/platform_installer.mjs b/src/core/platform_installer.mjs index 2bd4bb3ef..f6f9a00db 100644 --- a/src/core/platform_installer.mjs +++ b/src/core/platform_installer.mjs @@ -15,13 +15,14 @@ * */ import * as fs from 'fs' -import * as os from 'os' import { Listr } from 'listr2' import * as path from 'path' import { FullstackTestingError, IllegalArgumentError, MissingArgumentError } from './errors.mjs' import { constants } from './index.mjs' import { Templates } from './templates.mjs' import { flags } from '../commands/index.mjs' +import * as Base64 from 'js-base64' +import chalk from 'chalk' /** * PlatformInstaller install platform code in the root-container of a network pod @@ -138,16 +139,12 @@ export class PlatformInstaller { } } - async copyGossipKeys (podName, stagingDir, nodeIds, keyFormat = constants.KEY_FORMAT_PEM) { - const self = this - - if (!podName) throw new MissingArgumentError('podName is required') + async copyGossipKeys (nodeId, stagingDir, nodeIds, keyFormat = constants.KEY_FORMAT_PEM) { + if (!nodeId) throw new MissingArgumentError('nodeId is required') if (!stagingDir) throw new MissingArgumentError('stagingDir is required') if (!nodeIds || nodeIds.length <= 0) throw new MissingArgumentError('nodeIds cannot be empty') try { - const keysDir = `${constants.HEDERA_HAPI_PATH}/data/keys` - const nodeId = Templates.extractNodeIdFromPodName(podName) const srcFiles = [] switch (keyFormat) { @@ -170,33 +167,52 @@ export class PlatformInstaller { throw new FullstackTestingError(`Unsupported key file format ${keyFormat}`) } - return await self.copyFiles(podName, srcFiles, keysDir) + const data = {} + for (const srcFile of srcFiles) { + const fileContents = fs.readFileSync(srcFile) + const fileName = path.basename(srcFile) + data[fileName] = Base64.encode(fileContents) + } + + if (!await this.k8.createSecret( + Templates.renderGossipKeySecretName(nodeId), + this._getNamespace(), 'Opaque', data, + Templates.renderGossipKeySecretLabelObject(nodeId), true)) { + throw new FullstackTestingError(`failed to create secret for gossip keys for node '${nodeId}'`) + } } catch (e) { - throw new FullstackTestingError(`failed to copy gossip keys to pod '${podName}': ${e.message}`, e) + this.logger.error(`failed to copy gossip keys to secret '${Templates.renderGossipKeySecretName(nodeId)}': ${e.message}`, e) + throw new FullstackTestingError(`failed to copy gossip keys to secret '${Templates.renderGossipKeySecretName(nodeId)}': ${e.message}`, e) } } - async copyTLSKeys (podName, stagingDir) { - if (!podName) throw new MissingArgumentError('podName is required') + async copyTLSKeys (nodeIds, stagingDir) { + if (!nodeIds) throw new MissingArgumentError('nodeId is required') if (!stagingDir) throw new MissingArgumentError('stagingDir is required') try { - const nodeId = Templates.extractNodeIdFromPodName(podName) - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), `${nodeId}-tls-keys-`)) + const data = {} - // rename files appropriately in the tmp directory - fs.cpSync(path.join(stagingDir, 'keys', Templates.renderTLSPemPrivateKeyFile(nodeId)), - path.join(tmpDir, 'hedera.key')) - fs.cpSync(path.join(stagingDir, 'keys', Templates.renderTLSPemPublicKeyFile(nodeId)), - path.join(tmpDir, 'hedera.crt')) - - const srcFiles = [] - srcFiles.push(path.join(tmpDir, 'hedera.key')) - srcFiles.push(path.join(tmpDir, 'hedera.crt')) + for (const nodeId of nodeIds) { + const srcFiles = [] + srcFiles.push(path.join(stagingDir, 'keys', Templates.renderTLSPemPrivateKeyFile(nodeId))) + srcFiles.push(path.join(stagingDir, 'keys', Templates.renderTLSPemPublicKeyFile(nodeId))) - return this.copyFiles(podName, srcFiles, constants.HEDERA_HAPI_PATH) + for (const srcFile of srcFiles) { + const fileContents = fs.readFileSync(srcFile) + const fileName = path.basename(srcFile) + data[fileName] = Base64.encode(fileContents) + } + } + if (!await this.k8.createSecret( + 'network-node-hapi-app-secrets', + this._getNamespace(), 'Opaque', data, + undefined, true)) { + throw new FullstackTestingError('failed to create secret for TLS keys') + } } catch (e) { - throw new FullstackTestingError(`failed to copy TLS keys to pod '${podName}': ${e.message}`, e) + this.logger.error('failed to copy TLS keys to secret', e) + throw new FullstackTestingError('failed to copy TLS keys to secret', e) } } @@ -239,36 +255,14 @@ export class PlatformInstaller { } /** - * Return a list of task to perform node installation - * - * It assumes the staging directory has the following files and resources: - * ${staging}/keys/s-.key: signing key for a node - * ${staging}/keys/s-.crt: signing cert for a node - * ${staging}/keys/a-.key: agreement key for a node - * ${staging}/keys/a-.crt: agreement cert for a node - * ${staging}/keys/hedera-.key: gRPC TLS key for a node - * ${staging}/keys/hedera-.crt: gRPC TLS cert for a node + * Return a list of task to perform node directory setup * * @param podName name of the pod - * @param stagingDir staging directory path - * @param nodeIds list of node ids - * @param keyFormat key format (pfx or pem) - * @param force force flag * @returns {Listr} */ - taskInstall (podName, stagingDir, nodeIds, keyFormat = constants.KEY_FORMAT_PEM, force = false) { + taskSetup (podName) { const self = this return new Listr([ - { - title: 'Copy Gossip keys', - task: (_, task) => - self.copyGossipKeys(podName, stagingDir, nodeIds, keyFormat) - }, - { - title: 'Copy TLS keys', - task: (_, task) => - self.copyTLSKeys(podName, stagingDir, keyFormat) - }, { title: 'Set file permissions', task: (_, task) => @@ -280,7 +274,51 @@ export class PlatformInstaller { rendererOptions: { collapseSubtasks: false } + }) + } + + /** + * Return a list of task to copy the node keys to the staging directory + * + * It assumes the staging directory has the following files and resources: + *
  • ${staging}/keys/s-public-.pem: private signing key for a node
    • + *
  • ${staging}/keys/s-private-.pem: public signing key for a node
    • + *
  • ${staging}/keys/a-public-.pem: private agreement key for a node
    • + *
  • ${staging}/keys/a-private-.pem: public agreement key for a node
    • + *
  • ${staging}/keys/hedera-.key: gRPC TLS key for a node
    • + *
  • ${staging}/keys/hedera-.crt: gRPC TLS cert for a node
    • + * + * @param stagingDir staging directory path + * @param nodeIds list of node ids + * @param keyFormat key format (pfx or pem) + * @returns {Listr[]} + */ + copyNodeKeys (stagingDir, nodeIds, keyFormat = constants.KEY_FORMAT_PEM) { + const self = this + const subTasks = [] + subTasks.push({ + title: 'Copy TLS keys', + task: (_, task) => + self.copyTLSKeys(nodeIds, stagingDir, keyFormat) + }) + + for (const nodeId of nodeIds) { + subTasks.push({ + title: `Node: ${chalk.yellow(nodeId)}`, + task: () => new Listr([{ + title: 'Copy Gossip keys', + task: (_, task) => + self.copyGossipKeys(nodeId, stagingDir, nodeIds, keyFormat) + } + ], + { + concurrent: false, + rendererOptions: { + collapseSubtasks: false + } + }) + }) } - ) + return subTasks } } diff --git a/src/core/templates.mjs b/src/core/templates.mjs index 114501f0c..079d49e96 100644 --- a/src/core/templates.mjs +++ b/src/core/templates.mjs @@ -185,4 +185,14 @@ export class Templates { } } } + + static renderGossipKeySecretName (nodeId) { + return `network-${nodeId}-keys-secrets` + } + + static renderGossipKeySecretLabelObject (nodeId) { + return { + 'fullstack.hedera.com/node-name': nodeId + } + } } diff --git a/test/e2e/core/account_manager.test.mjs b/test/e2e/core/account_manager.test.mjs index 72a5989b4..04f9b82db 100644 --- a/test/e2e/core/account_manager.test.mjs +++ b/test/e2e/core/account_manager.test.mjs @@ -29,6 +29,8 @@ describe('AccountManager', () => { argv[flags.nodeIDs.name] = 'node0' argv[flags.clusterName.name] = TEST_CLUSTER argv[flags.fstChartVersion.name] = version.FST_CHART_VERSION + argv[flags.generateGossipKeys.name] = true + argv[flags.generateTlsKeys.name] = true // set the env variable SOLO_FST_CHARTS_DIR if developer wants to use local FST charts argv[flags.chartDirectory.name] = process.env.SOLO_FST_CHARTS_DIR ? process.env.SOLO_FST_CHARTS_DIR : undefined const bootstrapResp = bootstrapNetwork(namespace, argv, undefined, undefined, undefined, undefined, undefined, undefined, false) diff --git a/test/e2e/core/k8_e2e.test.mjs b/test/e2e/core/k8_e2e.test.mjs index c85428116..849956579 100644 --- a/test/e2e/core/k8_e2e.test.mjs +++ b/test/e2e/core/k8_e2e.test.mjs @@ -37,7 +37,7 @@ import { V1VolumeResourceRequirements } from '@kubernetes/client-node' -const defaultTimeout = 20000 +const defaultTimeout = 120000 describe('K8', () => { const testLogger = logging.NewLogger('debug', true) @@ -45,21 +45,23 @@ describe('K8', () => { const k8 = new K8(configManager, testLogger) const testNamespace = 'k8-e2e' const argv = [] - const podName = 'test-pod' + const podName = `test-pod-${uuid4()}` const containerName = 'alpine' - const podLabel = 'app=test' - const serviceName = 'test-service' + const podLabelValue = `test-${uuid4()}` + const serviceName = `test-service-${uuid4()}` beforeAll(async () => { try { argv[flags.namespace.name] = testNamespace configManager.update(argv) - await k8.createNamespace(testNamespace) + if (!await k8.hasNamespace(testNamespace)) { + await k8.createNamespace(testNamespace) + } const v1Pod = new V1Pod() const v1Metadata = new V1ObjectMeta() v1Metadata.name = podName v1Metadata.namespace = testNamespace - v1Metadata.labels = { app: 'test' } + v1Metadata.labels = { app: podLabelValue } v1Pod.metadata = v1Metadata const v1Container = new V1Container() v1Container.name = containerName @@ -83,7 +85,7 @@ describe('K8', () => { v1Svc.spec = v1SvcSpec await k8.kubeClient.createNamespacedService(testNamespace, v1Svc) } catch (e) { - console.log(e) + console.log(`${e}, ${e.stack}`) throw e } }, defaultTimeout) @@ -91,7 +93,6 @@ describe('K8', () => { afterAll(async () => { try { await k8.kubeClient.deleteNamespacedPod(podName, testNamespace, undefined, undefined, 1) - await k8.deleteNamespace(testNamespace) argv[flags.namespace.name] = constants.FULLSTACK_SETUP_NAMESPACE configManager.update(argv) } catch (e) { @@ -123,21 +124,21 @@ describe('K8', () => { }, defaultTimeout) it('should be able to run wait for pod', async () => { - const labels = [podLabel] + const labels = [`app=${podLabelValue}`] const pods = await k8.waitForPods([constants.POD_PHASE_RUNNING], labels, 1) expect(pods.length).toStrictEqual(1) }, defaultTimeout) it('should be able to run wait for pod ready', async () => { - const labels = [podLabel] + const labels = [`app=${podLabelValue}`] const pods = await k8.waitForPodReady(labels, 1) expect(pods.length).toStrictEqual(1) }, defaultTimeout) it('should be able to run wait for pod conditions', async () => { - const labels = [podLabel] + const labels = [`app=${podLabelValue}`] const conditions = new Map() .set(constants.POD_CONDITION_INITIALIZED, constants.POD_CONDITION_STATUS_TRUE) @@ -148,7 +149,7 @@ describe('K8', () => { }, defaultTimeout) it('should be able to detect pod IP of a pod', async () => { - const pods = await k8.getPodsByLabel([podLabel]) + const pods = await k8.getPodsByLabel([`app=${podLabelValue}`]) const podName = pods[0].metadata.name await expect(k8.getPodIP(podName)).resolves.not.toBeNull() await expect(k8.getPodIP('INVALID')).rejects.toThrow(FullstackTestingError) @@ -160,13 +161,13 @@ describe('K8', () => { }, defaultTimeout) it('should be able to check if a path is directory inside a container', async () => { - const pods = await k8.getPodsByLabel([podLabel]) + const pods = await k8.getPodsByLabel([`app=${podLabelValue}`]) const podName = pods[0].metadata.name await expect(k8.hasDir(podName, containerName, '/tmp')).resolves.toBeTruthy() }, defaultTimeout) it('should be able to copy a file to and from a container', async () => { - const pods = await k8.waitForPodReady([podLabel], 1, 20) + const pods = await k8.waitForPodReady([`app=${podLabelValue}`], 1, 20) expect(pods.length).toStrictEqual(1) const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'k8-')) const destDir = '/tmp' @@ -216,7 +217,7 @@ describe('K8', () => { }, defaultTimeout) it('should be able to cat a file inside the container', async () => { - const pods = await k8.getPodsByLabel([podLabel]) + const pods = await k8.getPodsByLabel([`app=${podLabelValue}`]) const podName = pods[0].metadata.name const output = await k8.execContainer(podName, containerName, ['cat', '/etc/hostname']) expect(output.indexOf(podName)).toEqual(0) @@ -224,9 +225,9 @@ describe('K8', () => { it('should be able to list persistent volume claims', async () => { let response + const v1Pvc = new V1PersistentVolumeClaim() try { - const v1Pvc = new V1PersistentVolumeClaim() - v1Pvc.name = 'test-pvc' + v1Pvc.name = `test-pvc-${uuid4()}` const v1Spec = new V1PersistentVolumeClaimSpec() v1Spec.accessModes = ['ReadWriteOnce'] const v1ResReq = new V1VolumeResourceRequirements() @@ -234,7 +235,7 @@ describe('K8', () => { v1Spec.resources = v1ResReq v1Pvc.spec = v1Spec const v1Metadata = new V1ObjectMeta() - v1Metadata.name = 'test-pvc' + v1Metadata.name = v1Pvc.name v1Pvc.metadata = v1Metadata response = await k8.kubeClient.createNamespacedPersistentVolumeClaim(testNamespace, v1Pvc) console.log(response) @@ -243,6 +244,8 @@ describe('K8', () => { } catch (e) { console.error(e) throw e + } finally { + await k8.deletePvc(v1Pvc.name, testNamespace) } }, defaultTimeout) }) diff --git a/test/e2e/core/platform_installer_e2e.test.mjs b/test/e2e/core/platform_installer_e2e.test.mjs index a54ffe843..20cec68c1 100644 --- a/test/e2e/core/platform_installer_e2e.test.mjs +++ b/test/e2e/core/platform_installer_e2e.test.mjs @@ -15,17 +15,14 @@ * */ import { afterAll, beforeAll, describe, expect, it } from '@jest/globals' -import { - constants, - Templates -} from '../../../src/core/index.mjs' +import { constants } from '../../../src/core/index.mjs' import * as fs from 'fs' import { bootstrapNetwork, getDefaultArgv, getTestCacheDir, - getTmpDir, TEST_CLUSTER, + TEST_CLUSTER, testLogger } from '../../test_util.js' import { flags } from '../../../src/commands/index.mjs' @@ -42,6 +39,8 @@ describe('PackageInstallerE2E', () => { argv[flags.nodeIDs.name] = 'node0' argv[flags.clusterName.name] = TEST_CLUSTER argv[flags.fstChartVersion.name] = version.FST_CHART_VERSION + argv[flags.generateGossipKeys.name] = true + argv[flags.generateTlsKeys.name] = true // set the env variable SOLO_FST_CHARTS_DIR if developer wants to use local FST charts argv[flags.chartDirectory.name] = process.env.SOLO_FST_CHARTS_DIR ? process.env.SOLO_FST_CHARTS_DIR : undefined const bootstrapResp = bootstrapNetwork(namespace, argv, undefined, undefined, undefined, undefined, undefined, undefined, false) @@ -98,58 +97,4 @@ describe('PackageInstallerE2E', () => { testLogger.showUser(outputs) }, 60000) }) - - describe('copyGossipKeys', () => { - it('should succeed to copy legacy pfx gossip keys for node0', async () => { - const podName = 'network-node0-0' - const nodeId = 'node0' - - // generate pfx keys - const pfxDir = 'test/data/pfx' - await k8.execContainer(podName, constants.ROOT_CONTAINER, ['bash', '-c', `rm -f ${constants.HEDERA_HAPI_PATH}/data/keys/*`]) - const fileList = await installer.copyGossipKeys(podName, pfxDir, ['node0'], constants.KEY_FORMAT_PFX) - - const destDir = `${constants.HEDERA_HAPI_PATH}/data/keys` - expect(fileList.length).toBe(2) - expect(fileList).toContain(`${destDir}/${Templates.renderGossipPfxPrivateKeyFile(nodeId)}`) - expect(fileList).toContain(`${destDir}/public.pfx`) - }, 60000) - - it('should succeed to copy pem gossip keys for node0', async () => { - const podName = 'network-node0-0' - - const pemDir = 'test/data/pem' - await k8.execContainer(podName, constants.ROOT_CONTAINER, ['bash', '-c', `rm -f ${constants.HEDERA_HAPI_PATH}/data/keys/*`]) - const fileList = await installer.copyGossipKeys(podName, pemDir, ['node0'], constants.KEY_FORMAT_PEM) - - const destDir = `${constants.HEDERA_HAPI_PATH}/data/keys` - expect(fileList.length).toBe(4) - expect(fileList).toContain(`${destDir}/${Templates.renderGossipPemPrivateKeyFile(constants.SIGNING_KEY_PREFIX, 'node0')}`) - expect(fileList).toContain(`${destDir}/${Templates.renderGossipPemPrivateKeyFile(constants.AGREEMENT_KEY_PREFIX, 'node0')}`) - - // public keys - expect(fileList).toContain(`${destDir}/${Templates.renderGossipPemPublicKeyFile(constants.SIGNING_KEY_PREFIX, 'node0')}`) - expect(fileList).toContain(`${destDir}/${Templates.renderGossipPemPublicKeyFile(constants.AGREEMENT_KEY_PREFIX, 'node0')}`) - }, 60000) - }) - - describe('copyTLSKeys', () => { - it('should succeed to copy TLS keys for node0', async () => { - const nodeId = 'node0' - const podName = Templates.renderNetworkPodName(nodeId) - const tmpDir = getTmpDir() - - // create mock files - const pemDir = 'test/data/pem' - await k8.execContainer(podName, constants.ROOT_CONTAINER, ['bash', '-c', `rm -f ${constants.HEDERA_HAPI_PATH}/hedera.*`]) - const fileList = await installer.copyTLSKeys(podName, pemDir) - - expect(fileList.length).toBe(2) // [data , hedera.crt, hedera.key] - expect(fileList.length).toBeGreaterThanOrEqual(2) - expect(fileList).toContain(`${constants.HEDERA_HAPI_PATH}/hedera.crt`) - expect(fileList).toContain(`${constants.HEDERA_HAPI_PATH}/hedera.key`) - - fs.rmSync(tmpDir, { recursive: true }) - }, defaultTimeout) - }) }) diff --git a/test/test_util.js b/test/test_util.js index 196591301..b0f4681c4 100644 --- a/test/test_util.js +++ b/test/test_util.js @@ -220,6 +220,14 @@ export function bootstrapNetwork (testName, argv, } }, 120000) + it('generate key files', async () => { + await expect(nodeCmd.keys(argv)).resolves.toBeTruthy() + expect(nodeCmd.getUnusedConfigs(NodeCommand.KEYS_CONFIGS_NAME)).toEqual([ + flags.cacheDir.constName, + flags.devMode.constName + ]) + }, 120000) + it('should succeed with network deploy', async () => { expect.assertions(1) await networkCmd.deploy(argv) @@ -247,16 +255,12 @@ export function bootstrapNetwork (testName, argv, it('should succeed with node setup command', async () => { expect.assertions(2) // cache this, because `solo node setup.finalize()` will reset it to false - const generateGossipKeys = bootstrapResp.opts.configManager.getFlag(flags.generateGossipKeys) try { await expect(nodeCmd.setup(argv)).resolves.toBeTruthy() - const expectedUnusedConfigs = [] - expectedUnusedConfigs.push(flags.appConfig.constName) - expectedUnusedConfigs.push(flags.devMode.constName) - if (!generateGossipKeys) { - expectedUnusedConfigs.push('curDate') - } - expect(nodeCmd.getUnusedConfigs(NodeCommand.SETUP_CONFIGS_NAME)).toEqual(expectedUnusedConfigs) + expect(nodeCmd.getUnusedConfigs(NodeCommand.SETUP_CONFIGS_NAME)).toEqual([ + flags.appConfig.constName, + flags.devMode.constName + ]) } catch (e) { nodeCmd.logger.showUserError(e) expect(e).toBeNull() diff --git a/test/unit/core/platform_installer.test.mjs b/test/unit/core/platform_installer.test.mjs index c027f47e7..c5641a7d2 100644 --- a/test/unit/core/platform_installer.test.mjs +++ b/test/unit/core/platform_installer.test.mjs @@ -109,7 +109,7 @@ describe('PackageInstaller', () => { }) it('should fail for missing stagingDir path', async () => { - await expect(installer.copyGossipKeys('network-node0-0', '')).rejects.toThrow(MissingArgumentError) + await expect(installer.copyGossipKeys('node0', '')).rejects.toThrow(MissingArgumentError) }) }) }) diff --git a/version.mjs b/version.mjs index 4881aedf4..2d6298b7b 100644 --- a/version.mjs +++ b/version.mjs @@ -21,5 +21,5 @@ export const JAVA_VERSION = '21.0.1+12' export const HELM_VERSION = 'v3.14.2' -export const FST_CHART_VERSION = 'v0.29.1' +export const FST_CHART_VERSION = 'v0.30.0' export const HEDERA_PLATFORM_VERSION = 'v0.53.2'