From 38dd17677feec8876f3279072d0e75908ae8567f Mon Sep 17 00:00:00 2001 From: Lenin Mehedy Date: Thu, 16 Nov 2023 23:35:45 +1100 Subject: [PATCH] fix(cli): invoke helm dependency update when local chart-dir is set (#562) Signed-off-by: Lenin Mehedy --- .github/workflows/zxc-compile-code.yaml | 17 ++- fullstack-network-manager/README.md | 13 +- fullstack-network-manager/package-lock.json | 4 +- .../resources/templates/hedera.crt | 33 ++++++ .../src/commands/chart.mjs | 20 +++- .../src/commands/cluster.mjs | 8 +- .../src/commands/init.mjs | 1 + .../src/core/config_manager.mjs | 111 +++++++++--------- .../e2e/core/package_downloader_e2e.test.mjs | 5 +- .../e2e/core/platform_installer_e2e.test.mjs | 10 +- .../unit/core/package_downloader.test.mjs | 7 +- .../unit/core/platform_installer.test.mjs | 10 +- 12 files changed, 152 insertions(+), 87 deletions(-) create mode 100755 fullstack-network-manager/resources/templates/hedera.crt diff --git a/.github/workflows/zxc-compile-code.yaml b/.github/workflows/zxc-compile-code.yaml index e5370e75c..5d9f5908b 100644 --- a/.github/workflows/zxc-compile-code.yaml +++ b/.github/workflows/zxc-compile-code.yaml @@ -143,14 +143,27 @@ jobs: if: ${{ inputs.enable-unit-tests && !cancelled() }} uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1 - - name: NodeJS CLI Tests + - name: NodeJS CLI Unit Tests id: nodejs-test working-directory: fullstack-network-manager if: ${{ inputs.enable-nodejs-tests && !cancelled() && !failure() }} run: | npm i npm test -# npm run test-e2e # tracked by #https://github.com/hashgraph/full-stack-testing/issues/501 + + - name: NodeJS CLI E2E Tests + id: nodejs-test-e2e + working-directory: fullstack-network-manager + if: ${{ inputs.enable-nodejs-tests && !cancelled() && !failure() }} + run: | + export DEV_MODE=true + npm i + npm link + fsnetman init -d ../charts + fsnetman cluster create + fsnetman cluster setup + fsnetman chart install + npm run test-e2e # This step tests the Helm chart direct mode of operation which uses the ubi8-init-java17 image. - name: Helm Chart Test (Direct Install) diff --git a/fullstack-network-manager/README.md b/fullstack-network-manager/README.md index 69a3fe42b..2a0a5759b 100644 --- a/fullstack-network-manager/README.md +++ b/fullstack-network-manager/README.md @@ -42,8 +42,19 @@ Select a command * Note: you need to do it once. If `fsnetman` already exists in your path, you will need to remove it first. * Alternative way would be to run `npm run fsnetman -- ` * Run `npm test` or `npm run test` to run the unit tests -* Run `npm run test-e2e` to run the long-running integration tests * Run `fsnetman` to access the CLI as shown above. * Note that debug logs are stored at `~/.fsnetman/logs/fst.log`. So you may use `tail -f ~/.fsnetman/logs/fst.log | jq ` in a separate terminal to keep an eye on the logs. * Before making a commit run `npm run format` + +## E2E tests + +* In order to run E2E test, we need to set up cluster and install the chart. + +``` + fsnetman init -d ../charts # use the charts directory + fsnetman cluster create + fsnetman cluster setup + fsnetman chart install + npm run test-e2e +``` diff --git a/fullstack-network-manager/package-lock.json b/fullstack-network-manager/package-lock.json index c70be2e17..1c21c98db 100644 --- a/fullstack-network-manager/package-lock.json +++ b/fullstack-network-manager/package-lock.json @@ -1,12 +1,12 @@ { "name": "@hashgraph/fullstack-network-manager", - "version": "0.1.0", + "version": "0.12.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@hashgraph/fullstack-network-manager", - "version": "0.1.0", + "version": "0.12.0", "license": "Apache2.0", "os": [ "darwin", diff --git a/fullstack-network-manager/resources/templates/hedera.crt b/fullstack-network-manager/resources/templates/hedera.crt new file mode 100755 index 000000000..8d075ec78 --- /dev/null +++ b/fullstack-network-manager/resources/templates/hedera.crt @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFpjCCA46gAwIBAgIUdLOB7iQa9RUTU+SBcrJyiCqdJrMwDQYJKoZIhvcNAQEL +BQAwQjESMBAGA1UEAwwJbG9jYWxob3N0MQswCQYDVQQGEwJVUzEQMA4GA1UECwwH +TXkgVW5pdDENMAsGA1UECgwEQUNNRTAgFw0yMjAyMTkwMDA4NDFaGA8yMTIyMDUw +NjAwMDg0MVowQjESMBAGA1UEAwwJbG9jYWxob3N0MQswCQYDVQQGEwJVUzEQMA4G +A1UECwwHTXkgVW5pdDENMAsGA1UECgwEQUNNRTCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAN3w4d7c6Zw9VMg6VIOAFpzbJGcP1uvY76OEINU8pHALz/w8 +WSjzXqpcu2L9jgKy67w4FALooXFTyYobTlFgmm4KxbvyHyZJRJvs/UcyZLw7j7tV +JIsziL3EloaLMfzag45CmN1sYnBl1422FIKGRH/8DgGnIWvWOKlkHm9zxWToYVXb +42tsEriig7GatUUhmIjr3S4L17EqZNtCcN4UTXPelwLDD4lsUUwEYXvumnPrscVo +GcskEZPKBHKDySaQCQ3bd30I5NPNkIfLv1rw79t6JFsJvzdchsHfUlVPx3fwpbwS +iPHNFFJgMtRH3gEd0IFQv5fC2Z8x/trCNL/jFhK+2SdRB8t1uS7OYTIdp8RCVJwS +zHzVi3qLN393tv0Biqfsk+R1tS3ono9gzDkuH0A5MoLKPd0w0bJsT6pgSYeqQ4mC +Bf8dEQDnipC5oG0vbLMPNCuObnLxZo2Y0Yui1YhjX7Om9GKeEdSCiSG2eidGrBY6 +zJanM58st8ix3MYUkeZO74Q4DZbK5Z7690ZdDPsJlCHjVBxw8ir47o3Ln9HBKzKS +lxgfWBI63loAr5ZGUNadZNlsXjxtaDAUf294CtAXdFDH7tLCa3eFOnOAd6spz2i9 +S1tIgXpe5djMgeXk39BWHI5m0TeDICPReGIF9LgvhXLHayvkI+WmOwlWxQFpAgMB +AAGjgZEwgY4wHQYDVR0OBBYEFPUngvCftXZ3TfIiUaA2iqHpwTQNMB8GA1UdIwQY +MBaAFPUngvCftXZ3TfIiUaA2iqHpwTQNMA8GA1UdEwEB/wQFMAMBAf8wCwYDVR0P +BAQDAgSwMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAPBgNVHREECDAG +hwR/AAABMA0GCSqGSIb3DQEBCwUAA4ICAQBTp8DJOcD4w7fZDyz79f6W0b2S4PKF +q6bvHPRtn8XaH/avM9u9VgNdBZjT3x87907xJEf73YCCvBV8/r6Dt+p+Y1qLbLdf +x0vm6djuzN3GPLLQOlC48lO7Qr6x9hrkCnFo1nL6EoqdPd6STEZpNlCCaEHoe5M9 +pQNmrI3Q5pkXtC9+tkV5Y3bw3o69CfHOZfgN368HmHehkMhxki2NknourJ3X54D9 +rYsCfjNng23xt86WSLCBqZPzPHPwDBVNubhbnd2o0H6C1paR7JN7I9jb6taLJcnY +xJ7BCnOHacfZFa1gEhVCyCy5ng8XNbcpaCw2rXKgIOzHUJDDFHkC5HAVKslweA4M +UkZu7/yUA91Mx6AziQ+/XBhPZPn809vWpZ2uDYnCaNFzwUmcWVLP+Id86jrT6T7E +fJCJGLit3CtY/IxoxmCgDoXspqap7DyvrnJ9hNGBjCIWjqqmVhhl55nEb7Op6eqG +CheG8xAxMsj4yc4ydzAQ1O6uv3h5JgrehBRiB0xKgPC4Jj8u5suQbjdcjTHU3/bH +Vcvg9MzR/3Eu53R8SfhIJXTlKQhmNOso4+CSPNR8R8ahmSG55nlNq0KkcICubPl8 +BtqUtJniUjhDJlWB+swbspWSyXTaQiavWnYQ/tTMvtv91o1Q8Cc7xrek/nJ4qPfE +qxD6cDDvXHUriw== +-----END CERTIFICATE----- diff --git a/fullstack-network-manager/src/commands/chart.mjs b/fullstack-network-manager/src/commands/chart.mjs index 59f41788b..7a2a3c99e 100644 --- a/fullstack-network-manager/src/commands/chart.mjs +++ b/fullstack-network-manager/src/commands/chart.mjs @@ -1,3 +1,4 @@ +import chalk from 'chalk' import { BaseCommand } from './base.mjs' import * as flags from './flags.mjs' import { constants } from '../core/index.mjs' @@ -21,11 +22,12 @@ export class ChartCommand extends BaseCommand { return valuesArg } - prepareChartPath (config) { + async prepareChartPath (config) { const chartDir = this.configManager.flagValue(config, flags.chartDirectory) let chartPath = 'full-stack-testing/fullstack-deployment' if (chartDir) { chartPath = `${chartDir}/fullstack-deployment` + await this.helm.dependency('update', chartPath) } return chartPath @@ -37,14 +39,26 @@ export class ChartCommand extends BaseCommand { const config = await this.configManager.setupConfig(argv) const valuesArg = this.prepareValuesArg(argv, config) - const chartPath = this.prepareChartPath(config) + const chartPath = await this.prepareChartPath(config) await this.chartManager.install(namespace, constants.FST_CHART_DEPLOYMENT_NAME, chartPath, config.version, valuesArg) this.logger.showList('charts', await this.chartManager.getInstalledCharts(namespace)) + + this.logger.showUser(chalk.cyan('> waiting for network-node pods to be active (first deployment takes ~10m) ...')) + await this.kubectl.wait('pod', + '--for=jsonpath=\'{.status.phase}\'=Running', + '-l fullstack.hedera.com/type=network-node', + '--timeout=900s' + ) + this.logger.showUser(chalk.green('OK'), 'network-node pods are running') + + return true } catch (e) { this.logger.showUserError(e) } + + return false } async uninstall (argv) { @@ -58,7 +72,7 @@ export class ChartCommand extends BaseCommand { const config = await this.configManager.setupConfig(argv) const valuesArg = this.prepareValuesArg(argv, config) - const chartPath = this.prepareChartPath(config) + const chartPath = await this.prepareChartPath(config) return await this.chartManager.upgrade(namespace, constants.FST_CHART_DEPLOYMENT_NAME, chartPath, valuesArg) } diff --git a/fullstack-network-manager/src/commands/cluster.mjs b/fullstack-network-manager/src/commands/cluster.mjs index a7cb23123..961ea31c4 100644 --- a/fullstack-network-manager/src/commands/cluster.mjs +++ b/fullstack-network-manager/src/commands/cluster.mjs @@ -172,11 +172,8 @@ export class ClusterCommand extends BaseCommand { const config = await this.configManager.setupConfig(argv) const namespace = argv.namespace - // create cluster - await this.create(argv, config) - // install fullstack-cluster-setup chart - const chartPath = this.prepareChartPath(config) + const chartPath = await this.prepareChartPath(config) const valuesArg = this.prepareValuesArg(config, argv.prometheusStack, argv.minio, argv.envoyGateway, argv.certManager, argv.certManagerCrds) this.logger.showUser(chalk.cyan('> setting up cluster:'), chalk.yellow(`${chartPath}`, chalk.yellow(valuesArg))) @@ -311,11 +308,12 @@ export class ClusterCommand extends BaseCommand { return valuesArg } - prepareChartPath (config) { + async prepareChartPath (config) { const chartDir = this.configManager.flagValue(config, flags.chartDirectory) let chartPath = 'full-stack-testing/fullstack-cluster-setup' if (chartDir) { chartPath = `${chartDir}/fullstack-cluster-setup` + await this.helm.dependency('update', chartPath) } return chartPath diff --git a/fullstack-network-manager/src/commands/init.mjs b/fullstack-network-manager/src/commands/init.mjs index 9c4eb9802..ca43d67b2 100644 --- a/fullstack-network-manager/src/commands/init.mjs +++ b/fullstack-network-manager/src/commands/init.mjs @@ -42,6 +42,7 @@ export class InitCommand extends BaseCommand { async init (argv) { try { await this.setupHomeDirectory() + await this.configManager.setupConfig(argv, true) const deps = [ core.constants.HELM, diff --git a/fullstack-network-manager/src/core/config_manager.mjs b/fullstack-network-manager/src/core/config_manager.mjs index e21150ac8..071e9f790 100644 --- a/fullstack-network-manager/src/core/config_manager.mjs +++ b/fullstack-network-manager/src/core/config_manager.mjs @@ -17,9 +17,9 @@ export class ConfigManager { } /** - * load package.json - * @returns {any} - */ + * load package.json + * @returns {any} + */ loadPackageJSON () { try { const raw = fs.readFileSync(`${CUR_FILE_DIR}/../../package.json`) @@ -42,70 +42,69 @@ export class ConfigManager { } /** - * Load and store config - * - * It overwrites previous config values using opts and store in the config file if any value has been changed. - * - * @param opts object containing various config related fields (e.g. argv) - * @param reset if we should reset old values - * @returns {Promise} - */ + * Load and store config + * + * It overwrites previous config values using opts and store in the config file if any value has been changed. + * + * @param opts object containing various config related fields (e.g. argv) + * @param reset if we should reset old values + * @returns {Promise} + */ async setupConfig (opts, reset = false) { const self = this - return new Promise((resolve, reject) => { - try { - let config = {} - let writeConfig = false - const packageJSON = self.loadPackageJSON() - - // if config exist, then load it first - if (!reset && fs.existsSync(constants.FST_CONFIG_FILE)) { - const configJSON = fs.readFileSync(constants.FST_CONFIG_FILE) - config = JSON.parse(configJSON.toString()) - } - - if (!config.flags) { - config.flags = {} - } - - // we always use packageJSON version as the version, so overwrite. - config.version = packageJSON.version + try { + let config = {} + let writeConfig = false + const packageJSON = self.loadPackageJSON() + + // if config exist, then load it first + if (!reset && fs.existsSync(constants.FST_CONFIG_FILE)) { + const configJSON = fs.readFileSync(constants.FST_CONFIG_FILE) + config = JSON.parse(configJSON.toString()) + } - // extract flags from argv - if (opts) { - flags.allFlags.forEach(flag => { - if (opts && opts[flag.name]) { - let val = opts[flag.name] - if (val && flag.name === flags.chartDirectory.name) { - val = paths.resolve(val) - } + if (!config.flags) { + config.flags = {} + } - config.flags[flag.name] = val - writeConfig = true + // we always use packageJSON version as the version, so overwrite. + config.version = packageJSON.version + + // extract flags from argv + if (opts) { + flags.allFlags.forEach(flag => { + if (opts && opts[flag.name]) { + let val = opts[flag.name] + if (val && flag.name === flags.chartDirectory.name) { + console.log(val) + val = paths.resolve(val) } - }) - // store last command that was run - if (opts._) { - config.lastCommand = opts._ + config.flags[flag.name] = val + writeConfig = true } - } - - // store CLI config - if (reset || writeConfig) { - config.updatedAt = new Date().toISOString() + }) - let configJSON = JSON.stringify(config) - fs.writeFileSync(`${constants.FST_CONFIG_FILE}`, configJSON) - configJSON = fs.readFileSync(constants.FST_CONFIG_FILE) - config = JSON.parse(configJSON.toString()) + // store last command that was run + if (opts._) { + config.lastCommand = opts._ } + } + + // store CLI config + if (reset || writeConfig) { + config.updatedAt = new Date().toISOString() - resolve(config) - } catch (e) { - reject(new FullstackTestingError(`failed to load config: ${e.message}`, e)) + let configJSON = JSON.stringify(config) + fs.writeFileSync(`${constants.FST_CONFIG_FILE}`, configJSON) + configJSON = fs.readFileSync(constants.FST_CONFIG_FILE) + config = JSON.parse(configJSON.toString()) } - }) + + return config + } catch (e) { + throw new FullstackTestingError(`failed to load config: ${e.message}`, e) + } } } diff --git a/fullstack-network-manager/test/e2e/core/package_downloader_e2e.test.mjs b/fullstack-network-manager/test/e2e/core/package_downloader_e2e.test.mjs index 1e2975841..ce41e10ed 100644 --- a/fullstack-network-manager/test/e2e/core/package_downloader_e2e.test.mjs +++ b/fullstack-network-manager/test/e2e/core/package_downloader_e2e.test.mjs @@ -20,9 +20,6 @@ describe('PackageDownloaderE2E', () => { testLogger.showUser(destPath) // remove the downloaded files to reduce disk usage - fs.rmSync(`${tmpDir}/v0.42/build-${tag}.zip`) - fs.rmSync(`${tmpDir}/v0.42/build-${tag}.sha384`) - fs.rmdirSync(`${tmpDir}/v0.42`) - fs.rmdirSync(tmpDir) + fs.rmSync(`${tmpDir}`, { recursive: true }) }, 100000) }) diff --git a/fullstack-network-manager/test/e2e/core/platform_installer_e2e.test.mjs b/fullstack-network-manager/test/e2e/core/platform_installer_e2e.test.mjs index e04eb509a..8f047f715 100644 --- a/fullstack-network-manager/test/e2e/core/platform_installer_e2e.test.mjs +++ b/fullstack-network-manager/test/e2e/core/platform_installer_e2e.test.mjs @@ -41,7 +41,7 @@ describe('PackageInstallerE2E', () => { console.error(e) expect(e).toBeNull() } - }) + }, 20000) }) describe('prepareConfigTxt', () => { @@ -69,7 +69,7 @@ describe('PackageInstallerE2E', () => { // verify file content matches expect(fileContents).toBe(configLines.join('\n')) - fs.rmdirSync(tmpDir, { recursive: true }) + fs.rmSync(tmpDir, { recursive: true }) }) }) @@ -88,7 +88,7 @@ describe('PackageInstallerE2E', () => { // verify copy of local-node data is at staging area expect(fs.existsSync(`${tmpDir}/templates`)).toBeTruthy() - fs.rmdirSync(tmpDir, { recursive: true }) + fs.rmSync(tmpDir, { recursive: true }) }) }) @@ -159,7 +159,7 @@ describe('PackageInstallerE2E', () => { expect(fileList).toContain(`${constants.HAPI_PATH}/data/config/api-permission.properties`) expect(fileList).toContain(`${constants.HAPI_PATH}/data/config/application.properties`) expect(fileList).toContain(`${constants.HAPI_PATH}/data/config/bootstrap.properties`) - fs.rmdirSync(tmpDir, { recursive: true }) - }) + fs.rmSync(tmpDir, { recursive: true }) + }, 10000) }) }) diff --git a/fullstack-network-manager/test/unit/core/package_downloader.test.mjs b/fullstack-network-manager/test/unit/core/package_downloader.test.mjs index de5854d1c..dacaac275 100644 --- a/fullstack-network-manager/test/unit/core/package_downloader.test.mjs +++ b/fullstack-network-manager/test/unit/core/package_downloader.test.mjs @@ -78,8 +78,7 @@ describe('PackageDownloader', () => { expect(fs.existsSync(destPath)).toBeTruthy() // remove the file to reduce disk usage - fs.rmSync(destPath) - fs.rmdirSync(tmpDir) + fs.rmSync(tmpDir, { recursive: true }) } catch (e) { expect(e).toBeNull() } @@ -95,7 +94,7 @@ describe('PackageDownloader', () => { try { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'downloader-')) await downloader.fetchPlatform(tag, tmpDir) - fs.rmdirSync(tmpDir, { recursive: true }) + fs.rmSync(tmpDir, { recursive: true }) } catch (e) { expect(e.cause).not.toBeNull() expect(e.cause).toBeInstanceOf(ResourceNotFoundError) @@ -108,7 +107,7 @@ describe('PackageDownloader', () => { try { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'downloader-')) await downloader.fetchPlatform('INVALID', os.tmpdir()) - fs.rmdirSync(tmpDir, { recursive: true }) + fs.rmSync(tmpDir, { recursive: true }) } catch (e) { expect(e.message).toContain('must include major, minor and patch fields') } diff --git a/fullstack-network-manager/test/unit/core/platform_installer.test.mjs b/fullstack-network-manager/test/unit/core/platform_installer.test.mjs index cb5027947..6302f2a2b 100644 --- a/fullstack-network-manager/test/unit/core/platform_installer.test.mjs +++ b/fullstack-network-manager/test/unit/core/platform_installer.test.mjs @@ -30,7 +30,7 @@ describe('PackageInstaller', () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'installer-')) fs.mkdirSync(`${tmpDir}/${core.constants.DATA_LIB_DIR}`, { recursive: true }) await expect(installer.validatePlatformReleaseDir(tmpDir)).rejects.toThrow(IllegalArgumentError) - fs.rmdirSync(tmpDir, { recursive: true }) + fs.rmSync(tmpDir, { recursive: true }) }) it('should fail if directory does not have data/libs directory', async () => { @@ -39,7 +39,7 @@ describe('PackageInstaller', () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'installer-')) fs.mkdirSync(`${tmpDir}/${core.constants.DATA_APPS_DIR}`, { recursive: true }) await expect(installer.validatePlatformReleaseDir(tmpDir)).rejects.toThrow(IllegalArgumentError) - fs.rmdirSync(tmpDir, { recursive: true }) + fs.rmSync(tmpDir, { recursive: true }) }) it('should fail if directory does not have data/app directory is empty', async () => { @@ -50,7 +50,7 @@ describe('PackageInstaller', () => { fs.mkdirSync(`${tmpDir}/${core.constants.DATA_LIB_DIR}`, { recursive: true }) fs.writeFileSync(`${tmpDir}/${core.constants.DATA_LIB_DIR}/test.jar`, '') await expect(installer.validatePlatformReleaseDir()).rejects.toThrow(MissingArgumentError) - fs.rmdirSync(tmpDir, { recursive: true }) + fs.rmSync(tmpDir, { recursive: true }) }) it('should fail if directory does not have data/apps directory is empty', async () => { @@ -59,7 +59,7 @@ describe('PackageInstaller', () => { fs.writeFileSync(`${tmpDir}/${core.constants.DATA_APPS_DIR}/app.jar`, '') fs.mkdirSync(`${tmpDir}/${core.constants.DATA_LIB_DIR}`, { recursive: true }) await expect(installer.validatePlatformReleaseDir()).rejects.toThrow(MissingArgumentError) - fs.rmdirSync(tmpDir, { recursive: true }) + fs.rmSync(tmpDir, { recursive: true }) }) it('should succeed with non-empty data/apps and data/libs directory', async () => { @@ -69,7 +69,7 @@ describe('PackageInstaller', () => { fs.mkdirSync(`${tmpDir}/${core.constants.DATA_LIB_DIR}`, { recursive: true }) fs.writeFileSync(`${tmpDir}/${core.constants.DATA_LIB_DIR}/lib-1.jar`, '') await expect(installer.validatePlatformReleaseDir()).rejects.toThrow(MissingArgumentError) - fs.rmdirSync(tmpDir, { recursive: true }) + fs.rmSync(tmpDir, { recursive: true }) }) })