From 7bc0f85809e289f34e6d0efa0ded570a16f56a1f Mon Sep 17 00:00:00 2001 From: Zuocheng Ding Date: Tue, 12 Apr 2022 13:50:00 -0700 Subject: [PATCH 1/8] [Circuit-Breaker] Add memory circuit breaker configuration (#1347) Add opensearch.memoryCircuitBreaker.enabled, opensearch.memoryCircuitBreaker.maxPercentage flags in opensearch_dashboards.yml Signed-off-by: Zuocheng Ding --- config/opensearch_dashboards.yml | 9 +++++++++ package.json | 2 +- packages/osd-opensearch/package.json | 2 +- src/core/server/opensearch/client/client_config.ts | 11 ++++++++--- src/core/server/opensearch/opensearch_config.test.ts | 4 ++++ src/core/server/opensearch/opensearch_config.ts | 12 ++++++++++++ .../resources/bin/opensearch-dashboards-docker | 4 +++- yarn.lock | 2 +- 8 files changed, 39 insertions(+), 7 deletions(-) diff --git a/config/opensearch_dashboards.yml b/config/opensearch_dashboards.yml index 242032edd025..95bf8c6e6519 100644 --- a/config/opensearch_dashboards.yml +++ b/config/opensearch_dashboards.yml @@ -76,6 +76,15 @@ # must be a positive integer. #opensearch.requestTimeout: 30000 +# Enables the memory circuit breaker that prevents heap out of memory errors for large query responses. +# If enabled, we will provide additional check to prevent potential out of memory error in @openserach-project/opensearch. +# The default threshold is based on the `max-old-space-size` of NodeJS. It is configurable by tuning `opensearch.memoryCircuitBreaker.maxPercentage`. +#opensearch.memoryCircuitBreaker.enabled: false + +# The pecentage of maximum heap allowed for processing response. The default value of the pecentage is `1.0`. The valid input range should be [0, 1] inclusively. +# For reference, the `threshold of memoryCircuitBreaker` = `max-old-space-size of NodeJS` * `opensearch.memoryCircuitBreaker.maxPercentage` +#opensearch.memoryCircuitBreaker.maxPercentage: 1.0 + # List of OpenSearch Dashboards client-side headers to send to OpenSearch. To send *no* client-side # headers, set this value to [] (an empty list). #opensearch.requestHeadersWhitelist: [ authorization ] diff --git a/package.json b/package.json index 7df8e5c87422..28c8a051e74f 100644 --- a/package.json +++ b/package.json @@ -123,7 +123,7 @@ "@hapi/podium": "^4.1.3", "@hapi/vision": "^6.1.0", "@hapi/wreck": "^17.1.0", - "@opensearch-project/opensearch": "^1.0.2", + "@opensearch-project/opensearch": "^1.1.0", "@osd/ace": "1.0.0", "@osd/analytics": "1.0.0", "@osd/apm-config-loader": "1.0.0", diff --git a/packages/osd-opensearch/package.json b/packages/osd-opensearch/package.json index 429b73b6e150..d086033f7288 100644 --- a/packages/osd-opensearch/package.json +++ b/packages/osd-opensearch/package.json @@ -12,7 +12,7 @@ "osd:watch": "node scripts/build --watch" }, "dependencies": { - "@opensearch-project/opensearch": "^1.0.2", + "@opensearch-project/opensearch": "^1.1.0", "@osd/dev-utils": "1.0.0", "abort-controller": "^3.0.0", "chalk": "^4.1.0", diff --git a/src/core/server/opensearch/client/client_config.ts b/src/core/server/opensearch/client/client_config.ts index d8ac605fb964..f53a114977e6 100644 --- a/src/core/server/opensearch/client/client_config.ts +++ b/src/core/server/opensearch/client/client_config.ts @@ -53,6 +53,9 @@ export type OpenSearchClientConfig = Pick< | 'username' | 'password' > & { + memoryCircuitBreaker?: + | OpenSearchConfig['memoryCircuitBreaker'] + | ClientOptions['memoryCircuitBreaker']; pingTimeout?: OpenSearchConfig['pingTimeout'] | ClientOptions['pingTimeout']; requestTimeout?: OpenSearchConfig['requestTimeout'] | ClientOptions['requestTimeout']; ssl?: Partial; @@ -75,13 +78,15 @@ export function parseClientOptions(config: OpenSearchClientConfig, scoped: boole ...config.customHeaders, }, }; - - if (config.pingTimeout != null) { - clientOptions.pingTimeout = getDurationAsMs(config.pingTimeout); + if (config.memoryCircuitBreaker != null) { + clientOptions.memoryCircuitBreaker = config.memoryCircuitBreaker; } if (config.requestTimeout != null) { clientOptions.requestTimeout = getDurationAsMs(config.requestTimeout); } + if (config.pingTimeout != null) { + clientOptions.pingTimeout = getDurationAsMs(config.pingTimeout); + } if (config.sniffInterval != null) { clientOptions.sniffInterval = typeof config.sniffInterval === 'boolean' diff --git a/src/core/server/opensearch/opensearch_config.test.ts b/src/core/server/opensearch/opensearch_config.test.ts index e784f630078c..d5fb77b026bd 100644 --- a/src/core/server/opensearch/opensearch_config.test.ts +++ b/src/core/server/opensearch/opensearch_config.test.ts @@ -78,6 +78,10 @@ test('set correct defaults', () => { ], "ignoreVersionMismatch": false, "logQueries": false, + "memoryCircuitBreaker": Object { + "enabled": false, + "maxPercentage": 1, + }, "optimizedHealthcheckId": undefined, "password": undefined, "pingTimeout": "PT30S", diff --git a/src/core/server/opensearch/opensearch_config.ts b/src/core/server/opensearch/opensearch_config.ts index 2a2d2d4a2f35..d6e80cc5062e 100644 --- a/src/core/server/opensearch/opensearch_config.ts +++ b/src/core/server/opensearch/opensearch_config.ts @@ -77,6 +77,10 @@ export const configSchema = schema.object({ requestHeadersWhitelist: schema.oneOf([schema.string(), schema.arrayOf(schema.string())], { defaultValue: ['authorization'], }), + memoryCircuitBreaker: schema.object({ + enabled: schema.boolean({ defaultValue: false }), + maxPercentage: schema.number({ defaultValue: 1.0 }), + }), customHeaders: schema.recordOf(schema.string(), schema.string(), { defaultValue: {} }), shardTimeout: schema.duration({ defaultValue: '30s' }), requestTimeout: schema.duration({ defaultValue: '30s' }), @@ -244,6 +248,13 @@ export class OpenSearchConfig { */ public readonly shardTimeout: Duration; + /** + * Set of options to configure memory circuit breaker for query response. + * The `maxPercentage` field is to determine the threshold for maximum heap size for memory circuit breaker. By default the value is `1.0`. + * The `enabled` field specifies whether the client should protect large response that can't fit into memory. + */ + + public readonly memoryCircuitBreaker: OpenSearchConfigType['memoryCircuitBreaker']; /** * Specifies whether the client should attempt to detect the rest of the cluster * when it is first instantiated. @@ -300,6 +311,7 @@ export class OpenSearchConfig { this.requestHeadersWhitelist = Array.isArray(rawConfig.requestHeadersWhitelist) ? rawConfig.requestHeadersWhitelist : [rawConfig.requestHeadersWhitelist]; + this.memoryCircuitBreaker = rawConfig.memoryCircuitBreaker; this.pingTimeout = rawConfig.pingTimeout; this.requestTimeout = rawConfig.requestTimeout; this.shardTimeout = rawConfig.shardTimeout; diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker index a9261f4227f1..411cbc7112bc 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker @@ -30,8 +30,10 @@ opensearch_dashboards_vars=( opensearch.hosts opensearch.logQueries opensearch.password - opensearch.pingTimeout opensearch.requestHeadersWhitelist + opensearch.memoryCircuitBreaker.enabled + opensearch.memoryCircuitBreaker.maxPercentage + opensearch.pingTimeout opensearch.requestTimeout opensearch.shardTimeout opensearch.sniffInterval diff --git a/yarn.lock b/yarn.lock index d76406f68099..1c0427a98563 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2365,7 +2365,7 @@ dependencies: "@octokit/openapi-types" "^11.2.0" -"@opensearch-project/opensearch@^1.0.2": +"@opensearch-project/opensearch@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@opensearch-project/opensearch/-/opensearch-1.1.0.tgz#8b3c8b4cbcea01755ba092d2997bf0b4ca7f22f7" integrity sha512-1TDw92JL8rD1b2QGluqBsIBLIiD5SGciIpz4qkrGAe9tcdfQ1ptub5e677rhWl35UULSjr6hP8M6HmISZ/M5HQ== From 1f013aefac1f138a222cc12ce7c617c2b3407847 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Apr 2022 11:56:04 -0500 Subject: [PATCH 2/8] Bump moment from 2.29.1 to 2.29.2 (#1439) Bumps [moment](https://github.com/moment/moment) from 2.29.1 to 2.29.2. - [Release notes](https://github.com/moment/moment/releases) - [Changelog](https://github.com/moment/moment/blob/develop/CHANGELOG.md) - [Commits](https://github.com/moment/moment/compare/2.29.1...2.29.2) --- updated-dependencies: - dependency-name: moment dependency-type: direct:production ... Resolves #1432 Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- packages/osd-config/package.json | 2 +- packages/osd-dev-utils/package.json | 2 +- packages/osd-telemetry-tools/package.json | 2 +- packages/osd-ui-shared-deps/package.json | 2 +- yarn.lock | 8 ++++---- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 28c8a051e74f..64d9d2258104 100644 --- a/package.json +++ b/package.json @@ -174,7 +174,7 @@ "lodash": "^4.17.21", "lru-cache": "^4.1.5", "minimatch": "^3.0.4", - "moment": "^2.24.0", + "moment": "^2.29.2", "moment-timezone": "^0.5.27", "mustache": "^2.3.2", "node-fetch": "^2.6.7", diff --git a/packages/osd-config/package.json b/packages/osd-config/package.json index a9c0d2875c26..83aabd0a3361 100644 --- a/packages/osd-config/package.json +++ b/packages/osd-config/package.json @@ -17,7 +17,7 @@ "js-yaml": "^3.14.0", "load-json-file": "^6.2.0", "lodash": "^4.17.21", - "moment": "^2.24.0", + "moment": "^2.29.2", "rxjs": "^6.5.5", "type-detect": "^4.0.8" }, diff --git a/packages/osd-dev-utils/package.json b/packages/osd-dev-utils/package.json index 3177d8b9fb59..aee715ae5abe 100644 --- a/packages/osd-dev-utils/package.json +++ b/packages/osd-dev-utils/package.json @@ -25,7 +25,7 @@ "globby": "^8.0.1", "load-json-file": "^6.2.0", "markdown-it": "^12.3.2", - "moment": "^2.24.0", + "moment": "^2.29.2", "normalize-path": "^3.0.0", "rxjs": "^6.5.5", "strip-ansi": "^6.0.0", diff --git a/packages/osd-telemetry-tools/package.json b/packages/osd-telemetry-tools/package.json index b697f7012df3..cac379907c9a 100644 --- a/packages/osd-telemetry-tools/package.json +++ b/packages/osd-telemetry-tools/package.json @@ -19,7 +19,7 @@ "@types/normalize-path": "^3.0.0", "normalize-path": "^3.0.0", "@types/lodash": "^4.14.170", - "moment": "^2.24.0", + "moment": "^2.29.2", "typescript": "4.0.2" } } diff --git a/packages/osd-ui-shared-deps/package.json b/packages/osd-ui-shared-deps/package.json index 361b953720b3..49ba5781f053 100644 --- a/packages/osd-ui-shared-deps/package.json +++ b/packages/osd-ui-shared-deps/package.json @@ -23,7 +23,7 @@ "jquery": "^3.5.0", "lodash": "^4.17.21", "mini-css-extract-plugin": "^1.6.0", - "moment": "^2.24.0", + "moment": "^2.29.2", "moment-timezone": "^0.5.27", "react": "^16.14.0", "react-dom": "^16.12.0", diff --git a/yarn.lock b/yarn.lock index 1c0427a98563..b1682040d8ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13569,10 +13569,10 @@ moment-timezone@*, moment-timezone@^0.5.27: dependencies: moment ">= 2.9.0" -"moment@>= 2.9.0", moment@^2.10.6, moment@^2.24.0: - version "2.29.1" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" - integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== +"moment@>= 2.9.0", moment@^2.10.6, moment@^2.29.2: + version "2.29.2" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.2.tgz#00910c60b20843bcba52d37d58c628b47b1f20e4" + integrity sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg== monaco-editor@~0.17.0: version "0.17.1" From e9e290442987acbc153e656e9def811eab4ed47e Mon Sep 17 00:00:00 2001 From: Ashwin P Chandran Date: Wed, 13 Apr 2022 11:50:25 -0700 Subject: [PATCH 3/8] fix(Tooltip): Fixes tooltip when split series charts are used (#1324) On split series charts, the y axis label is incorrect. This change uses the value from the raw table data to correctly assign the value Issues Resolved: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1262 Signed-off-by: Ashwin Pc --- .../tooltip/_pointseries_tooltip_formatter.js | 12 ++++- .../_pointseries_tooltip_formatter.test.js | 54 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.js b/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.js index fbc5b7c6c31c..7e3c466f3d1b 100644 --- a/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.js +++ b/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.js @@ -38,7 +38,7 @@ export function pointSeriesTooltipFormatter() { const details = []; const currentSeries = - data.series && data.series.find((serie) => serie.rawId === datum.seriesId); + data.series && data.series.find((series) => series.rawId === datum.seriesId); const addDetail = (label, value) => details.push({ label, value }); if (datum.extraMetrics) { @@ -53,7 +53,15 @@ export function pointSeriesTooltipFormatter() { if (datum.y !== null && datum.y !== undefined) { const value = datum.yScale ? datum.yScale * datum.y : datum.y; - addDetail(currentSeries.label, currentSeries.yAxisFormatter(value)); + let label = currentSeries.label; + + // For stacked charts the y axis data is only available in the raw table + const tableColumns = datum?.yRaw?.table?.columns; + if (tableColumns && tableColumns.length > 2) { + const yColumn = datum.yRaw.column ? tableColumns[datum.yRaw.column] : {}; + label = yColumn.name || label; + } + addDetail(label, currentSeries.yAxisFormatter(value)); } if (datum.z !== null && datum.z !== undefined) { diff --git a/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.test.js b/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.test.js index 0582615feed5..a1b61a4ffa4b 100644 --- a/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.test.js +++ b/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.test.js @@ -93,4 +93,58 @@ describe('tooltipFormatter', function () { const $rows = $el.find('tr'); expect($rows.length).toBe(3); }); + + it('renders correctly on split series data', function () { + const rawTable = { + columns: [ + { + id: 'col-1', + name: 'first', + }, + { + id: 'col-2', + name: 'second', + }, + { + id: 'col-3', + name: 'third', + }, + ], + rows: [ + { + 'col-1': 'A', + 'col-2': 'C', + 'col-3': 10, + }, + { + 'col-1': 'B', + 'col-2': 'C', + 'col-3': 20, + }, + ], + }; + + const event = _.cloneDeep(baseEvent); + + event.datum.yRaw = { + table: rawTable, + column: 2, + }; + + const $el = $(tooltipFormatter(event)); + const $rows = $el.find('tr'); + expect($rows.length).toBe(3); + + const $row1 = $rows.eq(0).find('td'); + expect(cell($row1, 0)).toBe('inner'); + expect(cell($row1, 1)).toBe('3'); + + const $row2 = $rows.eq(1).find('td'); + expect(cell($row2, 0)).toBe('third'); + expect(cell($row2, 1)).toBe('2'); + + const $row3 = $rows.eq(2).find('td'); + expect(cell($row3, 0)).toBe('top'); + expect(cell($row3, 1)).toBe('1'); + }); }); From 9489da97b7bf6ab53310f86d2daa9327d6055c04 Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Wed, 13 Apr 2022 13:28:38 -0700 Subject: [PATCH 4/8] [2.x] support legacy config (#1428) Allows for migrating from legacy app 6.8 to 7.10 to OSD 2.0, without losing any settings. Caught with BWC tests. We should probably evalulate the strategy when we plan to split from version sync from OpenSearch and if we want to make a configurable matrix here. But for now making this an explicit commit is preferable. Issue resolved: n/a Signed-off-by: Kawika Avilla --- ...h_dashboards_version_compatability.test.ts | 24 +++++++++++++++++++ ...search_dashboards_version_compatability.ts | 12 +++++++++- .../is_config_version_upgradeable.test.ts | 3 ++- .../is_config_version_upgradeable.ts | 13 ++++++++-- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.test.ts b/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.test.ts index a3e5755182b9..44773e22818c 100644 --- a/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.test.ts +++ b/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.test.ts @@ -56,6 +56,18 @@ describe('plugins/opensearch', () => { it('when majors and minors are not equal, but the engine is on legacy version 8.0.0 and OpenSearch Dashboards is on 1.0.0', () => { expect(opensearchVersionCompatibleWithOpenSearchDashboards('8.0.0', '1.0.0')).toBe(false); }); + + it('when majors and minors are not equal, but the engine is on legacy version 6.10.3 and OpenSearch Dashboards is on 2.0.0', () => { + expect(opensearchVersionCompatibleWithOpenSearchDashboards('6.10.3', '2.0.0')).toBe(false); + }); + + it('when majors and minors are not equal, but the engine is on legacy version 7.10.3 and OpenSearch Dashboards is on 2.0.0', () => { + expect(opensearchVersionCompatibleWithOpenSearchDashboards('7.10.3', '2.0.0')).toBe(false); + }); + + it('when majors and minors are not equal, but the engine is on legacy version 8.0.0 and OpenSearch Dashboards is on 2.0.0', () => { + expect(opensearchVersionCompatibleWithOpenSearchDashboards('8.0.0', '2.0.0')).toBe(false); + }); }); describe('returns true', () => { @@ -86,6 +98,18 @@ describe('plugins/opensearch', () => { it('when majors and minors are not equal, but the engine is on legacy version 7.10.2 and OpenSearch Dashboards is on 1.1.0', () => { expect(opensearchVersionCompatibleWithOpenSearchDashboards('7.10.2', '1.1.0')).toBe(true); }); + + it('when majors and minors are not equal, but the engine is on legacy version 7.10.2 and OpenSearch Dashboards is on 2.0.0', () => { + expect(opensearchVersionCompatibleWithOpenSearchDashboards('7.10.2', '2.0.0')).toBe(true); + }); + + it('when majors and minors are not equal, but the engine is on legacy version 7.10.2 and OpenSearch Dashboards is on 2.0.1', () => { + expect(opensearchVersionCompatibleWithOpenSearchDashboards('7.10.2', '2.0.1')).toBe(true); + }); + + it('when majors and minors are not equal, but the engine is on legacy version 7.10.2 and OpenSearch Dashboards is on 2.1.0', () => { + expect(opensearchVersionCompatibleWithOpenSearchDashboards('7.10.2', '2.1.0')).toBe(true); + }); }); }); }); diff --git a/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.ts b/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.ts index ffa4c07e3f50..8ecae4d60250 100644 --- a/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.ts +++ b/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.ts @@ -37,6 +37,16 @@ interface VersionNumbers { patch: number; } +/** + * @private + * + * List of OpenSearch Dashboards major versions that can connect to legacy version + * 7.10.2. + * + * WARNING: OpenSearchDashboards 7.x could cause conflicts. + */ +const osdLegacyCompatibleMajorVersions = [1, 2]; + /** * Checks for the compatibilitiy between OpenSearch and OpenSearchDashboards versions * 1. Major version differences will never work together. @@ -110,6 +120,6 @@ function legacyVersionCompatibleWithOpenSearchDashboards( legacyVersionNumbers.major === 7 && legacyVersionNumbers.minor === 10 && legacyVersionNumbers.patch === 2 && - opensearchDashboardsVersionNumbers.major === 1 + osdLegacyCompatibleMajorVersions.includes(opensearchDashboardsVersionNumbers.major) ); } diff --git a/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.test.ts b/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.test.ts index e2b5ebaea2a2..f1e052979c09 100644 --- a/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.test.ts +++ b/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.test.ts @@ -88,5 +88,6 @@ describe('savedObjects/health_check/isConfigVersionUpgradeable', function () { isUpgradeableTest('1.0.0-rc1', '1.0.0', true); isUpgradeableTest('1.0.1', '1.0.0', false); isUpgradeableTest('7.10.2', '1.1.0', true); - isUpgradeableTest('7.10.2', '2.0.0', false); + isUpgradeableTest('7.10.2', '2.0.0', true); + isUpgradeableTest('7.10.2', '3.0.0', false); }); diff --git a/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.ts b/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.ts index 7967e442a13f..3a111371c578 100644 --- a/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.ts +++ b/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.ts @@ -29,6 +29,13 @@ */ import semver from 'semver'; +/** + * List of OpenSearch Dashboards major versions that can pull the + * config from a legacy version that is higher in semvar. + * + * WARNING: OpenSearchDashboards 7.x could cause conflicts. + */ +const osdValidMajorVersions = [1, 2]; const rcVersionRegex = /^(\d+\.\d+\.\d+)\-rc(\d+)$/i; function extractRcNumber(version: string): [string, number] { @@ -73,10 +80,12 @@ export function isConfigVersionUpgradeable( // If the saved config is from the fork and from 6.8.0 to 7.10.2 then we should be able to upgrade. const savedIsFromPrefork = semver.gte(savedReleaseVersion, '6.8.0') && semver.lte(savedReleaseVersion, '7.10.2'); - const currentVersionIsVersion1 = semver.major(opensearchDashboardsReleaseVersion) === 1; + const currentVersionIsValidOSDVersion = osdValidMajorVersions.includes( + semver.major(opensearchDashboardsReleaseVersion) + ); return ( savedIsLessThanOpenSearchDashboards || (savedIsSameAsOpenSearchDashboards && savedRcIsLessThanOpenSearchDashboards) || - (savedIsFromPrefork && currentVersionIsVersion1) + (savedIsFromPrefork && currentVersionIsValidOSDVersion) ); } From a01e67a0ff01c538938cbad42c048074f37a10ef Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Wed, 13 Apr 2022 18:36:32 -0700 Subject: [PATCH 5/8] Revert "Bump moment from 2.29.1 to 2.29.2 (#1439)" (#1455) This reverts commit 1f013aefac1f138a222cc12ce7c617c2b3407847. Signed-off-by: Kawika Avilla --- package.json | 2 +- packages/osd-config/package.json | 2 +- packages/osd-dev-utils/package.json | 2 +- packages/osd-telemetry-tools/package.json | 2 +- packages/osd-ui-shared-deps/package.json | 2 +- yarn.lock | 8 ++++---- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 64d9d2258104..28c8a051e74f 100644 --- a/package.json +++ b/package.json @@ -174,7 +174,7 @@ "lodash": "^4.17.21", "lru-cache": "^4.1.5", "minimatch": "^3.0.4", - "moment": "^2.29.2", + "moment": "^2.24.0", "moment-timezone": "^0.5.27", "mustache": "^2.3.2", "node-fetch": "^2.6.7", diff --git a/packages/osd-config/package.json b/packages/osd-config/package.json index 83aabd0a3361..a9c0d2875c26 100644 --- a/packages/osd-config/package.json +++ b/packages/osd-config/package.json @@ -17,7 +17,7 @@ "js-yaml": "^3.14.0", "load-json-file": "^6.2.0", "lodash": "^4.17.21", - "moment": "^2.29.2", + "moment": "^2.24.0", "rxjs": "^6.5.5", "type-detect": "^4.0.8" }, diff --git a/packages/osd-dev-utils/package.json b/packages/osd-dev-utils/package.json index aee715ae5abe..3177d8b9fb59 100644 --- a/packages/osd-dev-utils/package.json +++ b/packages/osd-dev-utils/package.json @@ -25,7 +25,7 @@ "globby": "^8.0.1", "load-json-file": "^6.2.0", "markdown-it": "^12.3.2", - "moment": "^2.29.2", + "moment": "^2.24.0", "normalize-path": "^3.0.0", "rxjs": "^6.5.5", "strip-ansi": "^6.0.0", diff --git a/packages/osd-telemetry-tools/package.json b/packages/osd-telemetry-tools/package.json index cac379907c9a..b697f7012df3 100644 --- a/packages/osd-telemetry-tools/package.json +++ b/packages/osd-telemetry-tools/package.json @@ -19,7 +19,7 @@ "@types/normalize-path": "^3.0.0", "normalize-path": "^3.0.0", "@types/lodash": "^4.14.170", - "moment": "^2.29.2", + "moment": "^2.24.0", "typescript": "4.0.2" } } diff --git a/packages/osd-ui-shared-deps/package.json b/packages/osd-ui-shared-deps/package.json index 49ba5781f053..361b953720b3 100644 --- a/packages/osd-ui-shared-deps/package.json +++ b/packages/osd-ui-shared-deps/package.json @@ -23,7 +23,7 @@ "jquery": "^3.5.0", "lodash": "^4.17.21", "mini-css-extract-plugin": "^1.6.0", - "moment": "^2.29.2", + "moment": "^2.24.0", "moment-timezone": "^0.5.27", "react": "^16.14.0", "react-dom": "^16.12.0", diff --git a/yarn.lock b/yarn.lock index b1682040d8ce..1c0427a98563 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13569,10 +13569,10 @@ moment-timezone@*, moment-timezone@^0.5.27: dependencies: moment ">= 2.9.0" -"moment@>= 2.9.0", moment@^2.10.6, moment@^2.29.2: - version "2.29.2" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.2.tgz#00910c60b20843bcba52d37d58c628b47b1f20e4" - integrity sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg== +"moment@>= 2.9.0", moment@^2.10.6, moment@^2.24.0: + version "2.29.1" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3" + integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ== monaco-editor@~0.17.0: version "0.17.1" From e6911a099d9b29d002a66eb01ce2afe64b67e763 Mon Sep 17 00:00:00 2001 From: Tommy Markley Date: Thu, 14 Apr 2022 16:51:57 -0500 Subject: [PATCH 6/8] Cleans up changes from memory circuit breaker PR (#1463) * Cleans up whitespace, reduces delta of the changes. Follow-up from #1347 Signed-off-by: Tommy Markley --- src/core/server/opensearch/client/client_config.ts | 6 +++--- src/core/server/opensearch/opensearch_config.ts | 2 +- .../resources/bin/opensearch-dashboards-docker | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/server/opensearch/client/client_config.ts b/src/core/server/opensearch/client/client_config.ts index f53a114977e6..6746ee5648ed 100644 --- a/src/core/server/opensearch/client/client_config.ts +++ b/src/core/server/opensearch/client/client_config.ts @@ -81,12 +81,12 @@ export function parseClientOptions(config: OpenSearchClientConfig, scoped: boole if (config.memoryCircuitBreaker != null) { clientOptions.memoryCircuitBreaker = config.memoryCircuitBreaker; } - if (config.requestTimeout != null) { - clientOptions.requestTimeout = getDurationAsMs(config.requestTimeout); - } if (config.pingTimeout != null) { clientOptions.pingTimeout = getDurationAsMs(config.pingTimeout); } + if (config.requestTimeout != null) { + clientOptions.requestTimeout = getDurationAsMs(config.requestTimeout); + } if (config.sniffInterval != null) { clientOptions.sniffInterval = typeof config.sniffInterval === 'boolean' diff --git a/src/core/server/opensearch/opensearch_config.ts b/src/core/server/opensearch/opensearch_config.ts index d6e80cc5062e..c93d5360e296 100644 --- a/src/core/server/opensearch/opensearch_config.ts +++ b/src/core/server/opensearch/opensearch_config.ts @@ -253,8 +253,8 @@ export class OpenSearchConfig { * The `maxPercentage` field is to determine the threshold for maximum heap size for memory circuit breaker. By default the value is `1.0`. * The `enabled` field specifies whether the client should protect large response that can't fit into memory. */ - public readonly memoryCircuitBreaker: OpenSearchConfigType['memoryCircuitBreaker']; + /** * Specifies whether the client should attempt to detect the rest of the cluster * when it is first instantiated. diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker index 411cbc7112bc..77d36c2448f0 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker @@ -29,11 +29,11 @@ opensearch_dashboards_vars=( opensearch.customHeaders opensearch.hosts opensearch.logQueries - opensearch.password - opensearch.requestHeadersWhitelist opensearch.memoryCircuitBreaker.enabled opensearch.memoryCircuitBreaker.maxPercentage + opensearch.password opensearch.pingTimeout + opensearch.requestHeadersWhitelist opensearch.requestTimeout opensearch.shardTimeout opensearch.sniffInterval From 1492bfca3a9066eb1d9b258975dded306a7860eb Mon Sep 17 00:00:00 2001 From: Anan <79961084+ananzh@users.noreply.github.com> Date: Thu, 14 Apr 2022 15:45:40 -0700 Subject: [PATCH 7/8] update re2 fix node imcompatible issue (#1453) Issue Resolved: https://github.com/opensearch-project/OpenSearch-Dashboards/issues/1452 Signed-off-by: Anan Zhuang --- src/dev/build/tasks/patch_native_modules_task.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/dev/build/tasks/patch_native_modules_task.ts b/src/dev/build/tasks/patch_native_modules_task.ts index 66533e2dadee..7d1f53869223 100644 --- a/src/dev/build/tasks/patch_native_modules_task.ts +++ b/src/dev/build/tasks/patch_native_modules_task.ts @@ -58,20 +58,20 @@ const packages: Package[] = [ extractMethod: 'gunzip', archives: { 'darwin-x64': { - url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/darwin-x64-64.gz', - sha256: '595c6653d796493ddb288fc0732a0d1df8560099796f55a1dd242357d96bb8d6', + url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/darwin-x64-83.gz', + sha256: 'b45cd8296fd6eb2a091399c20111af43093ba30c99ed9e5d969278f5ff69ba8f', }, 'linux-x64': { - url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/linux-x64-64.gz', - sha256: 'e743587bc96314edf10c3e659c03168bc374a5cd9a6623ee99d989251e331f28', + url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/linux-x64-83.gz', + sha256: '1bbc3f90f0ba105772b37c04e3a718f69544b4df01dda00435c2b8e50b2ad0d9', }, 'linux-arm64': { url: 'https://d1v1sj258etie.cloudfront.net/node-re2/1.15.4/linux-arm64-64.gz', sha256: '19fa97f39d4965276382429bcd932dd696915f711663e7cef9b0a304b3e8e6f7', }, 'win32-x64': { - url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/win32-x64-64.gz', - sha256: 'b33de62cda24fb02dc80a19fb79977d686468ac746e97cd211059d2d4c75d529', + url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/win32-x64-83.gz', + sha256: 'efe939d3cda1d64ee3ee3e60a20613b95166d55632e702c670763ea7e69fca06', }, }, }, From 4e94ea1b48b3cbd0e733f30a5eaf81575de7b50b Mon Sep 17 00:00:00 2001 From: Kawika Avilla Date: Thu, 14 Apr 2022 23:03:03 -0700 Subject: [PATCH 8/8] [Tests] BWC test improvements (#1447) * [Tests] BWC test improvements for CI There were a couple issues related the BWC that was not supported within the current iteration of the BWC script. This fixes the following: * Format of files being saved with color looks unreadable in txt form * Exit code not being cypress exit code which would be the amount of errors. * Issue because texts would fail but would return 0 to test runner because the script worked * CI test runner expects to be able to call the same files twice (security:no-security) * Issue because results for both were saved at the same time while original it was wiped clean * Support from 'Playground' environments * Source to actually call the script * wget not available in CI runner * Track PID of process instead of greping since greping was not reliable. * OpenSearch config setup Signed-off-by: Kawika Avilla * Cleanup Signed-off-by: Kawika Avilla * Fix path of distribution Signed-off-by: Kawika Avilla * set CI and clean up more vars Signed-off-by: Kawika Avilla * Add playground comment Signed-off-by: Kawika Avilla * Readd tests Signed-off-by: Kawika Avilla --- bwctest.sh | 11 +- scripts/bwc/opensearch_dashboards_service.sh | 32 +++ scripts/bwc/opensearch_service.sh | 38 +++ scripts/bwc/utils.sh | 69 ++++++ scripts/bwctest_osd.sh | 237 +++++++------------ 5 files changed, 233 insertions(+), 154 deletions(-) create mode 100755 scripts/bwc/opensearch_dashboards_service.sh create mode 100755 scripts/bwc/opensearch_service.sh create mode 100755 scripts/bwc/utils.sh diff --git a/bwctest.sh b/bwctest.sh index 9f2934d62526..b4d30ef64425 100755 --- a/bwctest.sh +++ b/bwctest.sh @@ -13,8 +13,7 @@ set -e -# TODO: Update to include all known BWC of data -DEFAULT_VERSIONS="osd-1.1.0" +DEFAULT_VERSIONS="osd-1.1.0,odfe-1.13.2,odfe-0.10.0" function usage() { echo "" @@ -75,12 +74,14 @@ done [ -z "$BIND_PORT" ] && BIND_PORT="5601" [ -z "$SECURITY_ENABLED" ] && SECURITY_ENABLED="false" [ -z "$CREDENTIAL" ] && CREDENTIAL="admin:admin" +[ -z "$CI" ] && CI=1 # If no OpenSearch build was passed then this constructs the version if [ -z "$OPENSEARCH" ]; then IFS='/' read -ra SLASH_ARR <<< "$DASHBOARDS" # Expected to be opensearch-x.y.z-platform-arch.tar.gz - TARBALL="${SLASH_ARR[12]}" + # Playground is supported path to enable sandbox testing + [[ "$DASHBOARDS" == *"Playground"* ]] && TARBALL="${SLASH_ARR[14]}" || TARBALL="${SLASH_ARR[13]}" IFS='-' read -ra DASH_ARR <<< "$TARBALL" # Expected to be arch.tar.gz DOTS="${DASH_ARR[4]}" @@ -90,7 +91,7 @@ if [ -z "$OPENSEARCH" ]; then PLATFORM="${DASH_ARR[3]}" ARCH="${DOTS_ARR[0]}" - OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz" + OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/tar/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz" fi -./scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS +source scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS diff --git a/scripts/bwc/opensearch_dashboards_service.sh b/scripts/bwc/opensearch_dashboards_service.sh new file mode 100755 index 000000000000..e3016433b492 --- /dev/null +++ b/scripts/bwc/opensearch_dashboards_service.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +set -e + +function setup_dashboards() { + cd "$DASHBOARDS_DIR" + [ $SECURITY_ENABLED == "false" ] && [ -d "plugins/securityDashboards" ] && ./bin/opensearch-dashboards-plugin remove securityDashboards + [ $SECURITY_ENABLED == "false" ] && rm config/opensearch_dashboards.yml && touch config/opensearch_dashboards.yml + [ $SECURITY_ENABLED == "false" ] && echo "server.host: 0.0.0.0" >> config/opensearch_dashboards.yml + echo "csp.warnLegacyBrowsers: false" >> config/opensearch_dashboards.yml + echo "--max-old-space-size=5120" >> config/node.options +} + +# Starts OpenSearch Dashboards +function run_dashboards() { + echo "[ Attempting to start OpenSearch Dashboards... ]" + cd "$DASHBOARDS_DIR" + spawn_process_and_save_PID "./bin/opensearch-dashboards > ${LOGS_DIR}/opensearch_dashboards.log 2>&1 &" +} + +# Checks the running status of OpenSearch Dashboards +# it calls check_status and passes the OpenSearch Dashboards tmp file path, error msg, url, and arguments +# if success, the while loop in the check_status will end and it prints out "OpenSearch Dashboards is up!" +function check_dashboards_status { + echo "Checking the OpenSearch Dashboards..." + cd "$DIR" + check_status $DASHBOARDS_PATH "$DASHBOARDS_MSG" $DASHBOARDS_URL "" >> /dev/null 2>&1 + echo "OpenSearch Dashboards is up!" +} diff --git a/scripts/bwc/opensearch_service.sh b/scripts/bwc/opensearch_service.sh new file mode 100755 index 000000000000..d0051b5271bc --- /dev/null +++ b/scripts/bwc/opensearch_service.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +set -e + +function setup_opensearch() { + cd "$OPENSEARCH_DIR" + # Required for IM + [ -d "plugins/opensearch-index-management" ] && echo "path.repo: [/tmp]" >> config/opensearch.yml + # Required for Alerting + [ -d "plugins/opensearch-alerting" ] && echo "plugins.destination.host.deny_list: [\"10.0.0.0/8\", \"127.0.0.1\"]" >> config/opensearch.yml + # Required for SQL + [ -d "plugins/opensearch-sql" ] && echo "script.context.field.max_compilations_rate: 1000/1m" >> config/opensearch.yml + # Required for PA + [ -d "plugins/opensearch-performance-analyzer" ] && echo "webservice-bind-host = 0.0.0.0" >> plugins/opensearch-performance-analyzer/pa_config/performance-analyzer.properties + echo "network.host: 0.0.0.0" >> config/opensearch.yml + echo "discovery.type: single-node" >> config/opensearch.yml + [ $SECURITY_ENABLED == "false" ] && [ -d "plugins/opensearch-security" ] && echo "plugins.security.disabled: true" >> config/opensearch.yml +} + +# Starts OpenSearch, if verifying a distribution it will install the certs then start. +function run_opensearch() { + echo "[ Attempting to start OpenSearch... ]" + cd "$OPENSEARCH_DIR" + spawn_process_and_save_PID "./opensearch-tar-install.sh > ${LOGS_DIR}/opensearch.log 2>&1 &" +} + +# Checks the running status of OpenSearch +# it calls check_status and passes the OpenSearch tmp file path, error msg, url, and arguments +# if success, the while loop in the check_status will end and it prints out "OpenSearch is up!" +function check_opensearch_status() { + echo "Checking the status OpenSearch..." + cd "$DIR" + check_status $OPENSEARCH_PATH "$OPENSEARCH_MSG" $OPENSEARCH_URL "$OPENSEARCH_ARGS" >> /dev/null 2>&1 & + echo "OpenSearch is up!" +} diff --git a/scripts/bwc/utils.sh b/scripts/bwc/utils.sh new file mode 100755 index 000000000000..5400a91a9ef4 --- /dev/null +++ b/scripts/bwc/utils.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +set -e + +function open_artifact() { + artifact_dir=$1 + artifact=$2 + cd $artifact_dir + + # check if artifact provided is URL or attempt if passing by absolute path + if curl -I -L $artifact; then + curl -L $artifact | tar -xz --strip-components=1 + else + tar -xf $artifact --strip-components=1 + fi +} + +# remove the running opensearch process +function clean() { + echo "Attempt to Terminate Process with PID: ${PARENT_PID_LIST[*]}" + for pid_kill in "${PARENT_PID_LIST[@]}" + do + echo "Closing PID $pid_kill" + kill $pid_kill || true + done + PARENT_PID_LIST=() +} + +function spawn_process_and_save_PID() { + echo "Spawn '$@'" + eval $@ + curr_pid=$! + echo "PID: $curr_pid" + PARENT_PID_LIST+=( $curr_pid ) +} + +# Print out a textfile line by line +function print_txt() { + while IFS= read -r line; do + echo "text read from $1: $line" + done < $1 +} + +# this function is used to check the running status of OpenSearch or OpenSearch Dashboards +# $1 is the path to the tmp file which saves the running status +# $2 is the error msg to check +# $3 is the url to curl +# $4 contains arguments that need to be passed to the curl command +function check_status() { + while [ ! -f $1 ] || ! grep -q "$2" $1; do + if [ -f $1 ]; then rm $1; fi + curl $3 $4 > $1 || true + done + rm $1 +} + +# this function copies the tested data for the required version to the opensearch data folder +# $1 is the required version +function upload_data() { + rm -rf "$OPENSEARCH_DIR/data" + cd $OPENSEARCH_DIR + cp "$CWD/cypress/test-data/$DASHBOARDS_TYPE/$1.tar.gz" . + tar -xvf "$OPENSEARCH_DIR/$1.tar.gz" >> /dev/null 2>&1 + rm "$1.tar.gz" + echo "Data has been uploaded and ready to test" +} diff --git a/scripts/bwctest_osd.sh b/scripts/bwctest_osd.sh index 9c131e50fc10..a6775e54a858 100755 --- a/scripts/bwctest_osd.sh +++ b/scripts/bwctest_osd.sh @@ -5,6 +5,10 @@ set -e +. scripts/bwc/utils.sh +. scripts/bwc/opensearch_service.sh +. scripts/bwc/opensearch_dashboards_service.sh + # For every release, add sample data and new version below: DEFAULT_VERSIONS=( "odfe-0.10.0" @@ -95,67 +99,56 @@ done [ -z "$BIND_ADDRESS" ] && BIND_ADDRESS="localhost" [ -z "$BIND_PORT" ] && BIND_PORT="5601" -[ -z "$VERSIONS" ] && test_array=("${DEFAULT_VERSIONS[@]}") || IFS=',' read -r -a test_array <<<"$VERSIONS" +[ -z "$VERSIONS" ] && TEST_VERSIONS=("${DEFAULT_VERSIONS[@]}") || IFS=',' read -r -a TEST_VERSIONS <<<"$VERSIONS" [ -z "$SECURITY_ENABLED" ] && SECURITY_ENABLED="false" -[ $SECURITY_ENABLED == "false" ] && dashboards_type="without-security" || dashboards_type="with-security" -[ $SECURITY_ENABLED == "false" ] && releases_array=() || IFS=',' read -r -a releases_array <<<"$RELEASES" +[ $SECURITY_ENABLED == "false" ] && DASHBOARDS_TYPE="without-security" || DASHBOARDS_TYPE="with-security" +[ $SECURITY_ENABLED == "false" ] && RELEASES_ARRAY=() || IFS=',' read -r -a RELEASES_ARRAY <<<"$RELEASES" [ -z "$CREDENTIAL" ] && CREDENTIAL="admin:admin" +TOTAL_TEST_FAILURES=0 +# OpenSearch and OpenSearch Dashboards Process IDs +PARENT_PID_LIST=() # define test path -cwd=$(pwd) -dir="$cwd/bwc_tmp" -test_dir="$dir/test" -opensearch_dir="$dir/opensearch" -dashboards_dir="$dir/opensearch-dashboards" -if [ -d "$dir" ]; then - echo "Temporary directory exists. Removing." - rm -rf "$dir" -fi -mkdir "$dir" -mkdir "$test_dir" -mkdir "$opensearch_dir" -mkdir "$dashboards_dir" - -function open_artifact { - artifact_dir=$1 - artifact=$2 - cd $artifact_dir - - # check if artifact provided is URL or attempt if passing by absolute path - if wget -q --method=HEAD $artifact; then - wget -c $artifact -O - | tar -xz --strip-components=1 - else - tar -xf $artifact --strip-components=1 - fi - -} +CWD=$(pwd) +DIR="$CWD/bwc_tmp" +TEST_DIR="$DIR/test" +LOGS_DIR="$TEST_DIR/cypress/results/local-cluster-logs" +OPENSEARCH_DIR="$DIR/opensearch" +DASHBOARDS_DIR="$DIR/opensearch-dashboards" +[ ! -d "$DIR" ] && mkdir "$DIR" +[ ! -d "$TEST_DIR" ] && mkdir "$TEST_DIR" +[ -d "$OPENSEARCH_DIR" ] && rm -rf "$OPENSEARCH_DIR" +mkdir "$OPENSEARCH_DIR" +[ -d "$DASHBOARDS_DIR" ] && rm -rf "$DASHBOARDS_DIR" +mkdir "$DASHBOARDS_DIR" # un-tar OpenSearch and OpenSearch Dashboards echo "[ unzip OpenSearch and OpenSearch Dashboards ]" -open_artifact $opensearch_dir $OPENSEARCH -open_artifact $dashboards_dir $DASHBOARDS +echo $OPENSEARCH +open_artifact $OPENSEARCH_DIR $OPENSEARCH +open_artifact $DASHBOARDS_DIR $DASHBOARDS # define other paths and tmp files -opensearch_file='opensearch.txt' -dashboards_file='dashboards.txt' -opensearch_path="$dir/$opensearch_file" -dashboards_path="$dir/$dashboards_file" -dashboards_msg="\"state\":\"green\",\"title\":\"Green\",\"nickname\":\"Looking good\",\"icon\":\"success\"" -dashboards_url="http://$BIND_ADDRESS:$BIND_PORT/api/status" +OPENSEARCH_FILE='opensearch.txt' +DASHBOARDS_FILE='dashboards.txt' +OPENSEARCH_PATH="$DIR/$OPENSEARCH_FILE" +DASHBOARDS_PATH="$DIR/$DASHBOARDS_FILE" +DASHBOARDS_MSG="\"state\":\"green\",\"title\":\"Green\",\"nickname\":\"Looking good\",\"icon\":\"success\"" +DASHBOARDS_URL="http://$BIND_ADDRESS:$BIND_PORT/api/status" if [ $SECURITY_ENABLED == "false" ]; then - opensearch_msg="\"status\":\"green\"" - opensearch_url="http://$BIND_ADDRESS:9200/_cluster/health" - opensearch_args="" + OPENSEARCH_MSG="\"status\":\"green\"" + OPENSEARCH_URL="http://$BIND_ADDRESS:9200/_cluster/health" + OPENSEARCH_ARGS="" else - opensearch_msg="\"status\":\"yellow\"" - opensearch_url="https://$BIND_ADDRESS:9200/_cluster/health" - opensearch_args="-u $CREDENTIAL --insecure" + OPENSEARCH_MSG="\"status\":\"yellow\"" + OPENSEARCH_URL="https://$BIND_ADDRESS:9200/_cluster/health" + OPENSEARCH_ARGS="-u $CREDENTIAL --insecure" fi # define test groups to test suites -declare -A test_suites -test_suites=( +declare -A TEST_SUITES +TEST_SUITES=( ["odfe-0.10.0"]=$TEST_GROUP_1 ["odfe-1.0.2"]=$TEST_GROUP_2 ["odfe-1.1.0"]=$TEST_GROUP_2 @@ -171,128 +164,73 @@ test_suites=( ["osd-1.1.0"]=$TEST_GROUP_4 ) -# remove the running opensearch process -function clean { - echo "Closing the running OpenSearch" - process=($(ps -ef | grep "Dopensearch" | awk '{print $2}')) - kill ${process[0]} - echo "Closing any usage on port $BIND_PORT" - process=($(lsof -i -P -n | grep $BIND_PORT | awk '{print $2}')) - kill ${process[0]} -} - -# Print out a textfile line by line -function print_txt { - while IFS= read -r line; do - echo "text read from $1: $line" - done < $1 -} - -# this function is used to check the running status of OpenSearch or OpenSearch Dashboards -# $1 is the path to the tmp file which saves the running status -# $2 is the error msg to check -# $3 is the url to curl -# $4 contains arguments that need to be passed to the curl command -function check_status { - while [ ! -f $1 ] || ! grep -q "$2" $1; do - if [ -f $1 ]; then rm $1; fi - curl $3 $4 > $1 || true - done - rm $1 -} - # this function sets up the cypress env # it first clones the opensearch-dashboards-functional-test library # then it removes the tests into the cypress integration folder # and copies the backwards compatibility tests into the folder -function setup_cypress { +function setup_cypress() { echo "[ Setup the cypress test environment ]" - git clone https://github.com/opensearch-project/opensearch-dashboards-functional-test "$test_dir" - rm -rf "$test_dir/cypress/integration" - cp -r "$cwd/cypress/integration" "$test_dir/cypress" - cd "$test_dir" + git clone --depth=1 https://github.com/opensearch-project/opensearch-dashboards-functional-test "$TEST_DIR" + rm -rf "$TEST_DIR/cypress/integration" + cp -r "$CWD/cypress/integration" "$TEST_DIR/cypress" + [ ! -d "$LOGS_DIR" ] && mkdir -p "$LOGS_DIR" + cd "$TEST_DIR" npm install echo "Cypress is ready!" } -# this function copies the tested data for the required version to the opensearch data folder -# $1 is the required version -function upload_data { - rm -rf "$opensearch_dir/data" - cd $opensearch_dir - cp "$cwd/cypress/test-data/$dashboards_type/$1.tar.gz" . - tar -xvf "$opensearch_dir/$1.tar.gz" >> /dev/null 2>&1 - rm "$1.tar.gz" - echo "Data has been uploaded and ready to test" -} - -# Starts OpenSearch, if verifying a distribution it will install the certs then start. -function run_opensearch { - echo "[ Attempting to start OpenSearch... ]" - cd "$opensearch_dir" - [ $SECURITY_ENABLED == "false" ] && ./bin/opensearch || ./opensearch-tar-install.sh +function run_cypress() { + [ $1 == "core" ] && IS_CORE=true || IS_CORE=false + TEST_ARRAY=("$@") + SPEC_FILES="" + for test in "${TEST_ARRAY[@]}" + do + SPEC_FILES+="$TEST_DIR/cypress/integration/$DASHBOARDS_TYPE/*$test.js," + done + [ ! $IS_CORE ] && echo "Running tests from plugins" + [ $IS_CORE ] && spec="$SPEC_FILES" || "$TEST_DIR/cypress/integration/$DASHBOARDS_TYPE/plugins/*.js" + [ $IS_CORE ] && success_msg="BWC tests for core passed ($spec)" || success_msg="BWC tests for plugin passed ($spec)" + [ $IS_CORE ] && error_msg="BWC tests for core failed ($spec)" || error_msg="BWC tests for plugin failed ($spec)" + [ $CI == 1 ] && cypress_args="--browser chromium" || cypress_args="" + env NO_COLOR=1 npx cypress run $cypress_args --headless --spec $spec || test_failures=$? + [ -z $test_failures ] && test_failures=0 + [ $test_failures == 0 ] && echo $success_msg || echo "$error_msg::TEST_FAILURES: $test_failures" + TOTAL_TEST_FAILURES=$(( $TOTAL_TEST_FAILURES + $test_failures )) } -# Starts OpenSearch Dashboards -function run_dashboards { - echo "[ Attempting to start OpenSearch Dashboards... ]" - cd "$dashboards_dir" - [ $SECURITY_ENABLED == "false" ] && rm config/opensearch_dashboards.yml && touch config/opensearch_dashboards.yml - ./bin/opensearch-dashboards -} - -# Checks the running status of OpenSearch -# it calls check_status and passes the OpenSearch tmp file path, error msg, url, and arguments -# if success, the while loop in the check_status will end and it prints out "OpenSearch is up!" -function check_opensearch_status { - echo "Checking the status OpenSearch..." - cd "$dir" - check_status $opensearch_path "$opensearch_msg" $opensearch_url "$opensearch_args" >> /dev/null 2>&1 - echo "OpenSearch is up!" -} - -# Checks the running status of OpenSearch Dashboards -# it calls check_status and passes the OpenSearch Dashboards tmp file path, error msg, url, and arguments -# if success, the while loop in the check_status will end and it prints out "OpenSearch Dashboards is up!" -function check_dashboards_status { - echo "Checking the OpenSearch Dashboards..." - cd "$dir" - check_status $dashboards_path "$dashboards_msg" $dashboards_url "" >> /dev/null 2>&1 - echo "OpenSearch Dashboards is up!" -} - # Runs the backwards compatibility test using cypress for the required version # $1 is the requested version -function run_bwc { - cd "$test_dir" - [ -z "${test_suites[$1]}" ] && test_suite=$TEST_GROUP_DEFAULT || test_suite="${test_suites[$1]}" +function run_bwc() { + cd "$TEST_DIR" + [ -z "${TEST_SUITES[$1]}" ] && test_suite=$TEST_GROUP_DEFAULT || test_suite="${TEST_SUITES[$1]}" IFS=',' read -r -a tests <<<"$test_suite" - for test in "${tests[@]}" - do - npx cypress run --spec "$test_dir/cypress/integration/$dashboards_type/$test.js" || echo "backwards compatibility tests have issue" - done - # Check if $dashboards_type/plugins has tests in them to execute - if [ "$(ls -A $test_dir/cypress/integration/$dashboards_type/plugins | wc -l)" -gt 1 ]; then - echo "Running tests from plugins" - npx cypress run --spec "$test_dir/cypress/integration/$dashboards_type/plugins/*.js" || echo "backwards compatibility plugins tests have issue" + + # Check if $DASHBOARDS_TYPE/plugins has tests in them to execute + if [ "$(ls -A $TEST_DIR/cypress/integration/$DASHBOARDS_TYPE/plugins | wc -l)" -gt 1 ]; then + run_cypress "plugins" + else + run_cypress "core" "${tests[@]}" fi } # Main function -function execute_tests { +function execute_tests() { + setup_opensearch >> /dev/null 2>&1 & + setup_dashboards >> /dev/null 2>&1 & + # for each required testing version, do the following # first run opensearch and check the status # second run dashboards and check the status # run the backwards compatibility tests - for version in "${test_array[@]}" + for version in "${TEST_VERSIONS[@]}" do # copy and un-tar data into the OpenSearch data folder echo "[ Setting up the OpenSearch environment for $version ]" upload_data $version - run_opensearch >> /dev/null 2>&1 & + run_opensearch check_opensearch_status - run_dashboards >> /dev/null 2>&1 & + run_dashboards check_dashboards_status echo "[ Run the backwards compatibility tests for $version ]" @@ -304,29 +242,30 @@ function execute_tests { } # Executes the main function with different versions of OpenSearch downloaded -function execute_mismatch_tests { - PACKAGE_VERSION=$(cat $dashboards_dir/package.json \ +function execute_mismatch_tests() { + PACKAGE_VERSION=$(cat $DASHBOARDS_DIR/package.json \ | grep version \ | head -1 \ | awk -F: '{ print $2 }' \ | sed 's/[",]//g' \ | tr -d [:space:]) - for release in "${releases_array[@]}" + for release in "${RELEASES_ARRAY[@]}" do echo "Running tests with OpenSearch Dashboards $PACKAGE_VERSION and OpenSearch $release" ( - rm -rf $opensearch_dir && mkdir "$opensearch_dir" + rm -rf $OPENSEARCH_DIR && mkdir "$OPENSEARCH_DIR" # TODO: support multiple platforms and architectures - cd $opensearch_dir && wget -c https://artifacts.opensearch.org/releases/bundle/opensearch/$release/opensearch-$release-linux-x64.tar.gz -O - | tar -xz --strip-components=1 + cd $OPENSEARCH_DIR && curl https://artifacts.opensearch.org/releases/bundle/opensearch/$release/opensearch-$release-linux-x64.tar.gz | tar -xz --strip-components=1 ) execute_tests done } # setup the cypress test env -setup_cypress +[ ! -d "$TEST_DIR/cypress" ] && setup_cypress execute_tests -(( ${#releases_array[@]} )) && execute_mismatch_tests - -rm -rf "$dir" \ No newline at end of file +(( ${#RELEASES_ARRAY[@]} )) && execute_mismatch_tests +echo "Completed BWC tests for $TEST_VERSIONS [$DASHBOARDS_TYPE]" +echo "Total test failures: $TOTAL_TEST_FAILURES" +exit $TOTAL_TEST_FAILURES \ No newline at end of file