diff --git a/bwctest.sh b/bwctest.sh index 9f2934d62526..b4d30ef64425 100755 --- a/bwctest.sh +++ b/bwctest.sh @@ -13,8 +13,7 @@ set -e -# TODO: Update to include all known BWC of data -DEFAULT_VERSIONS="osd-1.1.0" +DEFAULT_VERSIONS="osd-1.1.0,odfe-1.13.2,odfe-0.10.0" function usage() { echo "" @@ -75,12 +74,14 @@ done [ -z "$BIND_PORT" ] && BIND_PORT="5601" [ -z "$SECURITY_ENABLED" ] && SECURITY_ENABLED="false" [ -z "$CREDENTIAL" ] && CREDENTIAL="admin:admin" +[ -z "$CI" ] && CI=1 # If no OpenSearch build was passed then this constructs the version if [ -z "$OPENSEARCH" ]; then IFS='/' read -ra SLASH_ARR <<< "$DASHBOARDS" # Expected to be opensearch-x.y.z-platform-arch.tar.gz - TARBALL="${SLASH_ARR[12]}" + # Playground is supported path to enable sandbox testing + [[ "$DASHBOARDS" == *"Playground"* ]] && TARBALL="${SLASH_ARR[14]}" || TARBALL="${SLASH_ARR[13]}" IFS='-' read -ra DASH_ARR <<< "$TARBALL" # Expected to be arch.tar.gz DOTS="${DASH_ARR[4]}" @@ -90,7 +91,7 @@ if [ -z "$OPENSEARCH" ]; then PLATFORM="${DASH_ARR[3]}" ARCH="${DOTS_ARR[0]}" - OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz" + OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/tar/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz" fi -./scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS +source scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS diff --git a/config/opensearch_dashboards.yml b/config/opensearch_dashboards.yml index 242032edd025..95bf8c6e6519 100644 --- a/config/opensearch_dashboards.yml +++ b/config/opensearch_dashboards.yml @@ -76,6 +76,15 @@ # must be a positive integer. #opensearch.requestTimeout: 30000 +# Enables the memory circuit breaker that prevents heap out of memory errors for large query responses. +# If enabled, we will provide additional check to prevent potential out of memory error in @openserach-project/opensearch. +# The default threshold is based on the `max-old-space-size` of NodeJS. It is configurable by tuning `opensearch.memoryCircuitBreaker.maxPercentage`. +#opensearch.memoryCircuitBreaker.enabled: false + +# The pecentage of maximum heap allowed for processing response. The default value of the pecentage is `1.0`. The valid input range should be [0, 1] inclusively. +# For reference, the `threshold of memoryCircuitBreaker` = `max-old-space-size of NodeJS` * `opensearch.memoryCircuitBreaker.maxPercentage` +#opensearch.memoryCircuitBreaker.maxPercentage: 1.0 + # List of OpenSearch Dashboards client-side headers to send to OpenSearch. To send *no* client-side # headers, set this value to [] (an empty list). #opensearch.requestHeadersWhitelist: [ authorization ] diff --git a/package.json b/package.json index 7df8e5c87422..28c8a051e74f 100644 --- a/package.json +++ b/package.json @@ -123,7 +123,7 @@ "@hapi/podium": "^4.1.3", "@hapi/vision": "^6.1.0", "@hapi/wreck": "^17.1.0", - "@opensearch-project/opensearch": "^1.0.2", + "@opensearch-project/opensearch": "^1.1.0", "@osd/ace": "1.0.0", "@osd/analytics": "1.0.0", "@osd/apm-config-loader": "1.0.0", diff --git a/packages/osd-opensearch/package.json b/packages/osd-opensearch/package.json index 429b73b6e150..d086033f7288 100644 --- a/packages/osd-opensearch/package.json +++ b/packages/osd-opensearch/package.json @@ -12,7 +12,7 @@ "osd:watch": "node scripts/build --watch" }, "dependencies": { - "@opensearch-project/opensearch": "^1.0.2", + "@opensearch-project/opensearch": "^1.1.0", "@osd/dev-utils": "1.0.0", "abort-controller": "^3.0.0", "chalk": "^4.1.0", diff --git a/scripts/bwc/opensearch_dashboards_service.sh b/scripts/bwc/opensearch_dashboards_service.sh new file mode 100755 index 000000000000..e3016433b492 --- /dev/null +++ b/scripts/bwc/opensearch_dashboards_service.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +set -e + +function setup_dashboards() { + cd "$DASHBOARDS_DIR" + [ $SECURITY_ENABLED == "false" ] && [ -d "plugins/securityDashboards" ] && ./bin/opensearch-dashboards-plugin remove securityDashboards + [ $SECURITY_ENABLED == "false" ] && rm config/opensearch_dashboards.yml && touch config/opensearch_dashboards.yml + [ $SECURITY_ENABLED == "false" ] && echo "server.host: 0.0.0.0" >> config/opensearch_dashboards.yml + echo "csp.warnLegacyBrowsers: false" >> config/opensearch_dashboards.yml + echo "--max-old-space-size=5120" >> config/node.options +} + +# Starts OpenSearch Dashboards +function run_dashboards() { + echo "[ Attempting to start OpenSearch Dashboards... ]" + cd "$DASHBOARDS_DIR" + spawn_process_and_save_PID "./bin/opensearch-dashboards > ${LOGS_DIR}/opensearch_dashboards.log 2>&1 &" +} + +# Checks the running status of OpenSearch Dashboards +# it calls check_status and passes the OpenSearch Dashboards tmp file path, error msg, url, and arguments +# if success, the while loop in the check_status will end and it prints out "OpenSearch Dashboards is up!" +function check_dashboards_status { + echo "Checking the OpenSearch Dashboards..." + cd "$DIR" + check_status $DASHBOARDS_PATH "$DASHBOARDS_MSG" $DASHBOARDS_URL "" >> /dev/null 2>&1 + echo "OpenSearch Dashboards is up!" +} diff --git a/scripts/bwc/opensearch_service.sh b/scripts/bwc/opensearch_service.sh new file mode 100755 index 000000000000..d0051b5271bc --- /dev/null +++ b/scripts/bwc/opensearch_service.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +set -e + +function setup_opensearch() { + cd "$OPENSEARCH_DIR" + # Required for IM + [ -d "plugins/opensearch-index-management" ] && echo "path.repo: [/tmp]" >> config/opensearch.yml + # Required for Alerting + [ -d "plugins/opensearch-alerting" ] && echo "plugins.destination.host.deny_list: [\"10.0.0.0/8\", \"127.0.0.1\"]" >> config/opensearch.yml + # Required for SQL + [ -d "plugins/opensearch-sql" ] && echo "script.context.field.max_compilations_rate: 1000/1m" >> config/opensearch.yml + # Required for PA + [ -d "plugins/opensearch-performance-analyzer" ] && echo "webservice-bind-host = 0.0.0.0" >> plugins/opensearch-performance-analyzer/pa_config/performance-analyzer.properties + echo "network.host: 0.0.0.0" >> config/opensearch.yml + echo "discovery.type: single-node" >> config/opensearch.yml + [ $SECURITY_ENABLED == "false" ] && [ -d "plugins/opensearch-security" ] && echo "plugins.security.disabled: true" >> config/opensearch.yml +} + +# Starts OpenSearch, if verifying a distribution it will install the certs then start. +function run_opensearch() { + echo "[ Attempting to start OpenSearch... ]" + cd "$OPENSEARCH_DIR" + spawn_process_and_save_PID "./opensearch-tar-install.sh > ${LOGS_DIR}/opensearch.log 2>&1 &" +} + +# Checks the running status of OpenSearch +# it calls check_status and passes the OpenSearch tmp file path, error msg, url, and arguments +# if success, the while loop in the check_status will end and it prints out "OpenSearch is up!" +function check_opensearch_status() { + echo "Checking the status OpenSearch..." + cd "$DIR" + check_status $OPENSEARCH_PATH "$OPENSEARCH_MSG" $OPENSEARCH_URL "$OPENSEARCH_ARGS" >> /dev/null 2>&1 & + echo "OpenSearch is up!" +} diff --git a/scripts/bwc/utils.sh b/scripts/bwc/utils.sh new file mode 100755 index 000000000000..5400a91a9ef4 --- /dev/null +++ b/scripts/bwc/utils.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +set -e + +function open_artifact() { + artifact_dir=$1 + artifact=$2 + cd $artifact_dir + + # check if artifact provided is URL or attempt if passing by absolute path + if curl -I -L $artifact; then + curl -L $artifact | tar -xz --strip-components=1 + else + tar -xf $artifact --strip-components=1 + fi +} + +# remove the running opensearch process +function clean() { + echo "Attempt to Terminate Process with PID: ${PARENT_PID_LIST[*]}" + for pid_kill in "${PARENT_PID_LIST[@]}" + do + echo "Closing PID $pid_kill" + kill $pid_kill || true + done + PARENT_PID_LIST=() +} + +function spawn_process_and_save_PID() { + echo "Spawn '$@'" + eval $@ + curr_pid=$! + echo "PID: $curr_pid" + PARENT_PID_LIST+=( $curr_pid ) +} + +# Print out a textfile line by line +function print_txt() { + while IFS= read -r line; do + echo "text read from $1: $line" + done < $1 +} + +# this function is used to check the running status of OpenSearch or OpenSearch Dashboards +# $1 is the path to the tmp file which saves the running status +# $2 is the error msg to check +# $3 is the url to curl +# $4 contains arguments that need to be passed to the curl command +function check_status() { + while [ ! -f $1 ] || ! grep -q "$2" $1; do + if [ -f $1 ]; then rm $1; fi + curl $3 $4 > $1 || true + done + rm $1 +} + +# this function copies the tested data for the required version to the opensearch data folder +# $1 is the required version +function upload_data() { + rm -rf "$OPENSEARCH_DIR/data" + cd $OPENSEARCH_DIR + cp "$CWD/cypress/test-data/$DASHBOARDS_TYPE/$1.tar.gz" . + tar -xvf "$OPENSEARCH_DIR/$1.tar.gz" >> /dev/null 2>&1 + rm "$1.tar.gz" + echo "Data has been uploaded and ready to test" +} diff --git a/scripts/bwctest_osd.sh b/scripts/bwctest_osd.sh index 9c131e50fc10..a6775e54a858 100755 --- a/scripts/bwctest_osd.sh +++ b/scripts/bwctest_osd.sh @@ -5,6 +5,10 @@ set -e +. scripts/bwc/utils.sh +. scripts/bwc/opensearch_service.sh +. scripts/bwc/opensearch_dashboards_service.sh + # For every release, add sample data and new version below: DEFAULT_VERSIONS=( "odfe-0.10.0" @@ -95,67 +99,56 @@ done [ -z "$BIND_ADDRESS" ] && BIND_ADDRESS="localhost" [ -z "$BIND_PORT" ] && BIND_PORT="5601" -[ -z "$VERSIONS" ] && test_array=("${DEFAULT_VERSIONS[@]}") || IFS=',' read -r -a test_array <<<"$VERSIONS" +[ -z "$VERSIONS" ] && TEST_VERSIONS=("${DEFAULT_VERSIONS[@]}") || IFS=',' read -r -a TEST_VERSIONS <<<"$VERSIONS" [ -z "$SECURITY_ENABLED" ] && SECURITY_ENABLED="false" -[ $SECURITY_ENABLED == "false" ] && dashboards_type="without-security" || dashboards_type="with-security" -[ $SECURITY_ENABLED == "false" ] && releases_array=() || IFS=',' read -r -a releases_array <<<"$RELEASES" +[ $SECURITY_ENABLED == "false" ] && DASHBOARDS_TYPE="without-security" || DASHBOARDS_TYPE="with-security" +[ $SECURITY_ENABLED == "false" ] && RELEASES_ARRAY=() || IFS=',' read -r -a RELEASES_ARRAY <<<"$RELEASES" [ -z "$CREDENTIAL" ] && CREDENTIAL="admin:admin" +TOTAL_TEST_FAILURES=0 +# OpenSearch and OpenSearch Dashboards Process IDs +PARENT_PID_LIST=() # define test path -cwd=$(pwd) -dir="$cwd/bwc_tmp" -test_dir="$dir/test" -opensearch_dir="$dir/opensearch" -dashboards_dir="$dir/opensearch-dashboards" -if [ -d "$dir" ]; then - echo "Temporary directory exists. Removing." - rm -rf "$dir" -fi -mkdir "$dir" -mkdir "$test_dir" -mkdir "$opensearch_dir" -mkdir "$dashboards_dir" - -function open_artifact { - artifact_dir=$1 - artifact=$2 - cd $artifact_dir - - # check if artifact provided is URL or attempt if passing by absolute path - if wget -q --method=HEAD $artifact; then - wget -c $artifact -O - | tar -xz --strip-components=1 - else - tar -xf $artifact --strip-components=1 - fi - -} +CWD=$(pwd) +DIR="$CWD/bwc_tmp" +TEST_DIR="$DIR/test" +LOGS_DIR="$TEST_DIR/cypress/results/local-cluster-logs" +OPENSEARCH_DIR="$DIR/opensearch" +DASHBOARDS_DIR="$DIR/opensearch-dashboards" +[ ! -d "$DIR" ] && mkdir "$DIR" +[ ! -d "$TEST_DIR" ] && mkdir "$TEST_DIR" +[ -d "$OPENSEARCH_DIR" ] && rm -rf "$OPENSEARCH_DIR" +mkdir "$OPENSEARCH_DIR" +[ -d "$DASHBOARDS_DIR" ] && rm -rf "$DASHBOARDS_DIR" +mkdir "$DASHBOARDS_DIR" # un-tar OpenSearch and OpenSearch Dashboards echo "[ unzip OpenSearch and OpenSearch Dashboards ]" -open_artifact $opensearch_dir $OPENSEARCH -open_artifact $dashboards_dir $DASHBOARDS +echo $OPENSEARCH +open_artifact $OPENSEARCH_DIR $OPENSEARCH +open_artifact $DASHBOARDS_DIR $DASHBOARDS # define other paths and tmp files -opensearch_file='opensearch.txt' -dashboards_file='dashboards.txt' -opensearch_path="$dir/$opensearch_file" -dashboards_path="$dir/$dashboards_file" -dashboards_msg="\"state\":\"green\",\"title\":\"Green\",\"nickname\":\"Looking good\",\"icon\":\"success\"" -dashboards_url="http://$BIND_ADDRESS:$BIND_PORT/api/status" +OPENSEARCH_FILE='opensearch.txt' +DASHBOARDS_FILE='dashboards.txt' +OPENSEARCH_PATH="$DIR/$OPENSEARCH_FILE" +DASHBOARDS_PATH="$DIR/$DASHBOARDS_FILE" +DASHBOARDS_MSG="\"state\":\"green\",\"title\":\"Green\",\"nickname\":\"Looking good\",\"icon\":\"success\"" +DASHBOARDS_URL="http://$BIND_ADDRESS:$BIND_PORT/api/status" if [ $SECURITY_ENABLED == "false" ]; then - opensearch_msg="\"status\":\"green\"" - opensearch_url="http://$BIND_ADDRESS:9200/_cluster/health" - opensearch_args="" + OPENSEARCH_MSG="\"status\":\"green\"" + OPENSEARCH_URL="http://$BIND_ADDRESS:9200/_cluster/health" + OPENSEARCH_ARGS="" else - opensearch_msg="\"status\":\"yellow\"" - opensearch_url="https://$BIND_ADDRESS:9200/_cluster/health" - opensearch_args="-u $CREDENTIAL --insecure" + OPENSEARCH_MSG="\"status\":\"yellow\"" + OPENSEARCH_URL="https://$BIND_ADDRESS:9200/_cluster/health" + OPENSEARCH_ARGS="-u $CREDENTIAL --insecure" fi # define test groups to test suites -declare -A test_suites -test_suites=( +declare -A TEST_SUITES +TEST_SUITES=( ["odfe-0.10.0"]=$TEST_GROUP_1 ["odfe-1.0.2"]=$TEST_GROUP_2 ["odfe-1.1.0"]=$TEST_GROUP_2 @@ -171,128 +164,73 @@ test_suites=( ["osd-1.1.0"]=$TEST_GROUP_4 ) -# remove the running opensearch process -function clean { - echo "Closing the running OpenSearch" - process=($(ps -ef | grep "Dopensearch" | awk '{print $2}')) - kill ${process[0]} - echo "Closing any usage on port $BIND_PORT" - process=($(lsof -i -P -n | grep $BIND_PORT | awk '{print $2}')) - kill ${process[0]} -} - -# Print out a textfile line by line -function print_txt { - while IFS= read -r line; do - echo "text read from $1: $line" - done < $1 -} - -# this function is used to check the running status of OpenSearch or OpenSearch Dashboards -# $1 is the path to the tmp file which saves the running status -# $2 is the error msg to check -# $3 is the url to curl -# $4 contains arguments that need to be passed to the curl command -function check_status { - while [ ! -f $1 ] || ! grep -q "$2" $1; do - if [ -f $1 ]; then rm $1; fi - curl $3 $4 > $1 || true - done - rm $1 -} - # this function sets up the cypress env # it first clones the opensearch-dashboards-functional-test library # then it removes the tests into the cypress integration folder # and copies the backwards compatibility tests into the folder -function setup_cypress { +function setup_cypress() { echo "[ Setup the cypress test environment ]" - git clone https://github.com/opensearch-project/opensearch-dashboards-functional-test "$test_dir" - rm -rf "$test_dir/cypress/integration" - cp -r "$cwd/cypress/integration" "$test_dir/cypress" - cd "$test_dir" + git clone --depth=1 https://github.com/opensearch-project/opensearch-dashboards-functional-test "$TEST_DIR" + rm -rf "$TEST_DIR/cypress/integration" + cp -r "$CWD/cypress/integration" "$TEST_DIR/cypress" + [ ! -d "$LOGS_DIR" ] && mkdir -p "$LOGS_DIR" + cd "$TEST_DIR" npm install echo "Cypress is ready!" } -# this function copies the tested data for the required version to the opensearch data folder -# $1 is the required version -function upload_data { - rm -rf "$opensearch_dir/data" - cd $opensearch_dir - cp "$cwd/cypress/test-data/$dashboards_type/$1.tar.gz" . - tar -xvf "$opensearch_dir/$1.tar.gz" >> /dev/null 2>&1 - rm "$1.tar.gz" - echo "Data has been uploaded and ready to test" -} - -# Starts OpenSearch, if verifying a distribution it will install the certs then start. -function run_opensearch { - echo "[ Attempting to start OpenSearch... ]" - cd "$opensearch_dir" - [ $SECURITY_ENABLED == "false" ] && ./bin/opensearch || ./opensearch-tar-install.sh +function run_cypress() { + [ $1 == "core" ] && IS_CORE=true || IS_CORE=false + TEST_ARRAY=("$@") + SPEC_FILES="" + for test in "${TEST_ARRAY[@]}" + do + SPEC_FILES+="$TEST_DIR/cypress/integration/$DASHBOARDS_TYPE/*$test.js," + done + [ ! $IS_CORE ] && echo "Running tests from plugins" + [ $IS_CORE ] && spec="$SPEC_FILES" || "$TEST_DIR/cypress/integration/$DASHBOARDS_TYPE/plugins/*.js" + [ $IS_CORE ] && success_msg="BWC tests for core passed ($spec)" || success_msg="BWC tests for plugin passed ($spec)" + [ $IS_CORE ] && error_msg="BWC tests for core failed ($spec)" || error_msg="BWC tests for plugin failed ($spec)" + [ $CI == 1 ] && cypress_args="--browser chromium" || cypress_args="" + env NO_COLOR=1 npx cypress run $cypress_args --headless --spec $spec || test_failures=$? + [ -z $test_failures ] && test_failures=0 + [ $test_failures == 0 ] && echo $success_msg || echo "$error_msg::TEST_FAILURES: $test_failures" + TOTAL_TEST_FAILURES=$(( $TOTAL_TEST_FAILURES + $test_failures )) } -# Starts OpenSearch Dashboards -function run_dashboards { - echo "[ Attempting to start OpenSearch Dashboards... ]" - cd "$dashboards_dir" - [ $SECURITY_ENABLED == "false" ] && rm config/opensearch_dashboards.yml && touch config/opensearch_dashboards.yml - ./bin/opensearch-dashboards -} - -# Checks the running status of OpenSearch -# it calls check_status and passes the OpenSearch tmp file path, error msg, url, and arguments -# if success, the while loop in the check_status will end and it prints out "OpenSearch is up!" -function check_opensearch_status { - echo "Checking the status OpenSearch..." - cd "$dir" - check_status $opensearch_path "$opensearch_msg" $opensearch_url "$opensearch_args" >> /dev/null 2>&1 - echo "OpenSearch is up!" -} - -# Checks the running status of OpenSearch Dashboards -# it calls check_status and passes the OpenSearch Dashboards tmp file path, error msg, url, and arguments -# if success, the while loop in the check_status will end and it prints out "OpenSearch Dashboards is up!" -function check_dashboards_status { - echo "Checking the OpenSearch Dashboards..." - cd "$dir" - check_status $dashboards_path "$dashboards_msg" $dashboards_url "" >> /dev/null 2>&1 - echo "OpenSearch Dashboards is up!" -} - # Runs the backwards compatibility test using cypress for the required version # $1 is the requested version -function run_bwc { - cd "$test_dir" - [ -z "${test_suites[$1]}" ] && test_suite=$TEST_GROUP_DEFAULT || test_suite="${test_suites[$1]}" +function run_bwc() { + cd "$TEST_DIR" + [ -z "${TEST_SUITES[$1]}" ] && test_suite=$TEST_GROUP_DEFAULT || test_suite="${TEST_SUITES[$1]}" IFS=',' read -r -a tests <<<"$test_suite" - for test in "${tests[@]}" - do - npx cypress run --spec "$test_dir/cypress/integration/$dashboards_type/$test.js" || echo "backwards compatibility tests have issue" - done - # Check if $dashboards_type/plugins has tests in them to execute - if [ "$(ls -A $test_dir/cypress/integration/$dashboards_type/plugins | wc -l)" -gt 1 ]; then - echo "Running tests from plugins" - npx cypress run --spec "$test_dir/cypress/integration/$dashboards_type/plugins/*.js" || echo "backwards compatibility plugins tests have issue" + + # Check if $DASHBOARDS_TYPE/plugins has tests in them to execute + if [ "$(ls -A $TEST_DIR/cypress/integration/$DASHBOARDS_TYPE/plugins | wc -l)" -gt 1 ]; then + run_cypress "plugins" + else + run_cypress "core" "${tests[@]}" fi } # Main function -function execute_tests { +function execute_tests() { + setup_opensearch >> /dev/null 2>&1 & + setup_dashboards >> /dev/null 2>&1 & + # for each required testing version, do the following # first run opensearch and check the status # second run dashboards and check the status # run the backwards compatibility tests - for version in "${test_array[@]}" + for version in "${TEST_VERSIONS[@]}" do # copy and un-tar data into the OpenSearch data folder echo "[ Setting up the OpenSearch environment for $version ]" upload_data $version - run_opensearch >> /dev/null 2>&1 & + run_opensearch check_opensearch_status - run_dashboards >> /dev/null 2>&1 & + run_dashboards check_dashboards_status echo "[ Run the backwards compatibility tests for $version ]" @@ -304,29 +242,30 @@ function execute_tests { } # Executes the main function with different versions of OpenSearch downloaded -function execute_mismatch_tests { - PACKAGE_VERSION=$(cat $dashboards_dir/package.json \ +function execute_mismatch_tests() { + PACKAGE_VERSION=$(cat $DASHBOARDS_DIR/package.json \ | grep version \ | head -1 \ | awk -F: '{ print $2 }' \ | sed 's/[",]//g' \ | tr -d [:space:]) - for release in "${releases_array[@]}" + for release in "${RELEASES_ARRAY[@]}" do echo "Running tests with OpenSearch Dashboards $PACKAGE_VERSION and OpenSearch $release" ( - rm -rf $opensearch_dir && mkdir "$opensearch_dir" + rm -rf $OPENSEARCH_DIR && mkdir "$OPENSEARCH_DIR" # TODO: support multiple platforms and architectures - cd $opensearch_dir && wget -c https://artifacts.opensearch.org/releases/bundle/opensearch/$release/opensearch-$release-linux-x64.tar.gz -O - | tar -xz --strip-components=1 + cd $OPENSEARCH_DIR && curl https://artifacts.opensearch.org/releases/bundle/opensearch/$release/opensearch-$release-linux-x64.tar.gz | tar -xz --strip-components=1 ) execute_tests done } # setup the cypress test env -setup_cypress +[ ! -d "$TEST_DIR/cypress" ] && setup_cypress execute_tests -(( ${#releases_array[@]} )) && execute_mismatch_tests - -rm -rf "$dir" \ No newline at end of file +(( ${#RELEASES_ARRAY[@]} )) && execute_mismatch_tests +echo "Completed BWC tests for $TEST_VERSIONS [$DASHBOARDS_TYPE]" +echo "Total test failures: $TOTAL_TEST_FAILURES" +exit $TOTAL_TEST_FAILURES \ No newline at end of file diff --git a/src/core/server/opensearch/client/client_config.ts b/src/core/server/opensearch/client/client_config.ts index d8ac605fb964..6746ee5648ed 100644 --- a/src/core/server/opensearch/client/client_config.ts +++ b/src/core/server/opensearch/client/client_config.ts @@ -53,6 +53,9 @@ export type OpenSearchClientConfig = Pick< | 'username' | 'password' > & { + memoryCircuitBreaker?: + | OpenSearchConfig['memoryCircuitBreaker'] + | ClientOptions['memoryCircuitBreaker']; pingTimeout?: OpenSearchConfig['pingTimeout'] | ClientOptions['pingTimeout']; requestTimeout?: OpenSearchConfig['requestTimeout'] | ClientOptions['requestTimeout']; ssl?: Partial; @@ -75,7 +78,9 @@ export function parseClientOptions(config: OpenSearchClientConfig, scoped: boole ...config.customHeaders, }, }; - + if (config.memoryCircuitBreaker != null) { + clientOptions.memoryCircuitBreaker = config.memoryCircuitBreaker; + } if (config.pingTimeout != null) { clientOptions.pingTimeout = getDurationAsMs(config.pingTimeout); } diff --git a/src/core/server/opensearch/opensearch_config.test.ts b/src/core/server/opensearch/opensearch_config.test.ts index e784f630078c..d5fb77b026bd 100644 --- a/src/core/server/opensearch/opensearch_config.test.ts +++ b/src/core/server/opensearch/opensearch_config.test.ts @@ -78,6 +78,10 @@ test('set correct defaults', () => { ], "ignoreVersionMismatch": false, "logQueries": false, + "memoryCircuitBreaker": Object { + "enabled": false, + "maxPercentage": 1, + }, "optimizedHealthcheckId": undefined, "password": undefined, "pingTimeout": "PT30S", diff --git a/src/core/server/opensearch/opensearch_config.ts b/src/core/server/opensearch/opensearch_config.ts index 2a2d2d4a2f35..c93d5360e296 100644 --- a/src/core/server/opensearch/opensearch_config.ts +++ b/src/core/server/opensearch/opensearch_config.ts @@ -77,6 +77,10 @@ export const configSchema = schema.object({ requestHeadersWhitelist: schema.oneOf([schema.string(), schema.arrayOf(schema.string())], { defaultValue: ['authorization'], }), + memoryCircuitBreaker: schema.object({ + enabled: schema.boolean({ defaultValue: false }), + maxPercentage: schema.number({ defaultValue: 1.0 }), + }), customHeaders: schema.recordOf(schema.string(), schema.string(), { defaultValue: {} }), shardTimeout: schema.duration({ defaultValue: '30s' }), requestTimeout: schema.duration({ defaultValue: '30s' }), @@ -244,6 +248,13 @@ export class OpenSearchConfig { */ public readonly shardTimeout: Duration; + /** + * Set of options to configure memory circuit breaker for query response. + * The `maxPercentage` field is to determine the threshold for maximum heap size for memory circuit breaker. By default the value is `1.0`. + * The `enabled` field specifies whether the client should protect large response that can't fit into memory. + */ + public readonly memoryCircuitBreaker: OpenSearchConfigType['memoryCircuitBreaker']; + /** * Specifies whether the client should attempt to detect the rest of the cluster * when it is first instantiated. @@ -300,6 +311,7 @@ export class OpenSearchConfig { this.requestHeadersWhitelist = Array.isArray(rawConfig.requestHeadersWhitelist) ? rawConfig.requestHeadersWhitelist : [rawConfig.requestHeadersWhitelist]; + this.memoryCircuitBreaker = rawConfig.memoryCircuitBreaker; this.pingTimeout = rawConfig.pingTimeout; this.requestTimeout = rawConfig.requestTimeout; this.shardTimeout = rawConfig.shardTimeout; diff --git a/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.test.ts b/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.test.ts index a3e5755182b9..44773e22818c 100644 --- a/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.test.ts +++ b/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.test.ts @@ -56,6 +56,18 @@ describe('plugins/opensearch', () => { it('when majors and minors are not equal, but the engine is on legacy version 8.0.0 and OpenSearch Dashboards is on 1.0.0', () => { expect(opensearchVersionCompatibleWithOpenSearchDashboards('8.0.0', '1.0.0')).toBe(false); }); + + it('when majors and minors are not equal, but the engine is on legacy version 6.10.3 and OpenSearch Dashboards is on 2.0.0', () => { + expect(opensearchVersionCompatibleWithOpenSearchDashboards('6.10.3', '2.0.0')).toBe(false); + }); + + it('when majors and minors are not equal, but the engine is on legacy version 7.10.3 and OpenSearch Dashboards is on 2.0.0', () => { + expect(opensearchVersionCompatibleWithOpenSearchDashboards('7.10.3', '2.0.0')).toBe(false); + }); + + it('when majors and minors are not equal, but the engine is on legacy version 8.0.0 and OpenSearch Dashboards is on 2.0.0', () => { + expect(opensearchVersionCompatibleWithOpenSearchDashboards('8.0.0', '2.0.0')).toBe(false); + }); }); describe('returns true', () => { @@ -86,6 +98,18 @@ describe('plugins/opensearch', () => { it('when majors and minors are not equal, but the engine is on legacy version 7.10.2 and OpenSearch Dashboards is on 1.1.0', () => { expect(opensearchVersionCompatibleWithOpenSearchDashboards('7.10.2', '1.1.0')).toBe(true); }); + + it('when majors and minors are not equal, but the engine is on legacy version 7.10.2 and OpenSearch Dashboards is on 2.0.0', () => { + expect(opensearchVersionCompatibleWithOpenSearchDashboards('7.10.2', '2.0.0')).toBe(true); + }); + + it('when majors and minors are not equal, but the engine is on legacy version 7.10.2 and OpenSearch Dashboards is on 2.0.1', () => { + expect(opensearchVersionCompatibleWithOpenSearchDashboards('7.10.2', '2.0.1')).toBe(true); + }); + + it('when majors and minors are not equal, but the engine is on legacy version 7.10.2 and OpenSearch Dashboards is on 2.1.0', () => { + expect(opensearchVersionCompatibleWithOpenSearchDashboards('7.10.2', '2.1.0')).toBe(true); + }); }); }); }); diff --git a/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.ts b/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.ts index ffa4c07e3f50..8ecae4d60250 100644 --- a/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.ts +++ b/src/core/server/opensearch/version_check/opensearch_opensearch_dashboards_version_compatability.ts @@ -37,6 +37,16 @@ interface VersionNumbers { patch: number; } +/** + * @private + * + * List of OpenSearch Dashboards major versions that can connect to legacy version + * 7.10.2. + * + * WARNING: OpenSearchDashboards 7.x could cause conflicts. + */ +const osdLegacyCompatibleMajorVersions = [1, 2]; + /** * Checks for the compatibilitiy between OpenSearch and OpenSearchDashboards versions * 1. Major version differences will never work together. @@ -110,6 +120,6 @@ function legacyVersionCompatibleWithOpenSearchDashboards( legacyVersionNumbers.major === 7 && legacyVersionNumbers.minor === 10 && legacyVersionNumbers.patch === 2 && - opensearchDashboardsVersionNumbers.major === 1 + osdLegacyCompatibleMajorVersions.includes(opensearchDashboardsVersionNumbers.major) ); } diff --git a/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.test.ts b/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.test.ts index e2b5ebaea2a2..f1e052979c09 100644 --- a/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.test.ts +++ b/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.test.ts @@ -88,5 +88,6 @@ describe('savedObjects/health_check/isConfigVersionUpgradeable', function () { isUpgradeableTest('1.0.0-rc1', '1.0.0', true); isUpgradeableTest('1.0.1', '1.0.0', false); isUpgradeableTest('7.10.2', '1.1.0', true); - isUpgradeableTest('7.10.2', '2.0.0', false); + isUpgradeableTest('7.10.2', '2.0.0', true); + isUpgradeableTest('7.10.2', '3.0.0', false); }); diff --git a/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.ts b/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.ts index 7967e442a13f..3a111371c578 100644 --- a/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.ts +++ b/src/core/server/ui_settings/create_or_upgrade_saved_config/is_config_version_upgradeable.ts @@ -29,6 +29,13 @@ */ import semver from 'semver'; +/** + * List of OpenSearch Dashboards major versions that can pull the + * config from a legacy version that is higher in semvar. + * + * WARNING: OpenSearchDashboards 7.x could cause conflicts. + */ +const osdValidMajorVersions = [1, 2]; const rcVersionRegex = /^(\d+\.\d+\.\d+)\-rc(\d+)$/i; function extractRcNumber(version: string): [string, number] { @@ -73,10 +80,12 @@ export function isConfigVersionUpgradeable( // If the saved config is from the fork and from 6.8.0 to 7.10.2 then we should be able to upgrade. const savedIsFromPrefork = semver.gte(savedReleaseVersion, '6.8.0') && semver.lte(savedReleaseVersion, '7.10.2'); - const currentVersionIsVersion1 = semver.major(opensearchDashboardsReleaseVersion) === 1; + const currentVersionIsValidOSDVersion = osdValidMajorVersions.includes( + semver.major(opensearchDashboardsReleaseVersion) + ); return ( savedIsLessThanOpenSearchDashboards || (savedIsSameAsOpenSearchDashboards && savedRcIsLessThanOpenSearchDashboards) || - (savedIsFromPrefork && currentVersionIsVersion1) + (savedIsFromPrefork && currentVersionIsValidOSDVersion) ); } diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker index a9261f4227f1..77d36c2448f0 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker @@ -29,6 +29,8 @@ opensearch_dashboards_vars=( opensearch.customHeaders opensearch.hosts opensearch.logQueries + opensearch.memoryCircuitBreaker.enabled + opensearch.memoryCircuitBreaker.maxPercentage opensearch.password opensearch.pingTimeout opensearch.requestHeadersWhitelist diff --git a/src/dev/build/tasks/patch_native_modules_task.ts b/src/dev/build/tasks/patch_native_modules_task.ts index 66533e2dadee..7d1f53869223 100644 --- a/src/dev/build/tasks/patch_native_modules_task.ts +++ b/src/dev/build/tasks/patch_native_modules_task.ts @@ -58,20 +58,20 @@ const packages: Package[] = [ extractMethod: 'gunzip', archives: { 'darwin-x64': { - url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/darwin-x64-64.gz', - sha256: '595c6653d796493ddb288fc0732a0d1df8560099796f55a1dd242357d96bb8d6', + url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/darwin-x64-83.gz', + sha256: 'b45cd8296fd6eb2a091399c20111af43093ba30c99ed9e5d969278f5ff69ba8f', }, 'linux-x64': { - url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/linux-x64-64.gz', - sha256: 'e743587bc96314edf10c3e659c03168bc374a5cd9a6623ee99d989251e331f28', + url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/linux-x64-83.gz', + sha256: '1bbc3f90f0ba105772b37c04e3a718f69544b4df01dda00435c2b8e50b2ad0d9', }, 'linux-arm64': { url: 'https://d1v1sj258etie.cloudfront.net/node-re2/1.15.4/linux-arm64-64.gz', sha256: '19fa97f39d4965276382429bcd932dd696915f711663e7cef9b0a304b3e8e6f7', }, 'win32-x64': { - url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/win32-x64-64.gz', - sha256: 'b33de62cda24fb02dc80a19fb79977d686468ac746e97cd211059d2d4c75d529', + url: 'https://github.com/uhop/node-re2/releases/download/1.15.4/win32-x64-83.gz', + sha256: 'efe939d3cda1d64ee3ee3e60a20613b95166d55632e702c670763ea7e69fca06', }, }, }, diff --git a/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.js b/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.js index fbc5b7c6c31c..7e3c466f3d1b 100644 --- a/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.js +++ b/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.js @@ -38,7 +38,7 @@ export function pointSeriesTooltipFormatter() { const details = []; const currentSeries = - data.series && data.series.find((serie) => serie.rawId === datum.seriesId); + data.series && data.series.find((series) => series.rawId === datum.seriesId); const addDetail = (label, value) => details.push({ label, value }); if (datum.extraMetrics) { @@ -53,7 +53,15 @@ export function pointSeriesTooltipFormatter() { if (datum.y !== null && datum.y !== undefined) { const value = datum.yScale ? datum.yScale * datum.y : datum.y; - addDetail(currentSeries.label, currentSeries.yAxisFormatter(value)); + let label = currentSeries.label; + + // For stacked charts the y axis data is only available in the raw table + const tableColumns = datum?.yRaw?.table?.columns; + if (tableColumns && tableColumns.length > 2) { + const yColumn = datum.yRaw.column ? tableColumns[datum.yRaw.column] : {}; + label = yColumn.name || label; + } + addDetail(label, currentSeries.yAxisFormatter(value)); } if (datum.z !== null && datum.z !== undefined) { diff --git a/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.test.js b/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.test.js index 0582615feed5..a1b61a4ffa4b 100644 --- a/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.test.js +++ b/src/plugins/vis_type_vislib/public/vislib/components/tooltip/_pointseries_tooltip_formatter.test.js @@ -93,4 +93,58 @@ describe('tooltipFormatter', function () { const $rows = $el.find('tr'); expect($rows.length).toBe(3); }); + + it('renders correctly on split series data', function () { + const rawTable = { + columns: [ + { + id: 'col-1', + name: 'first', + }, + { + id: 'col-2', + name: 'second', + }, + { + id: 'col-3', + name: 'third', + }, + ], + rows: [ + { + 'col-1': 'A', + 'col-2': 'C', + 'col-3': 10, + }, + { + 'col-1': 'B', + 'col-2': 'C', + 'col-3': 20, + }, + ], + }; + + const event = _.cloneDeep(baseEvent); + + event.datum.yRaw = { + table: rawTable, + column: 2, + }; + + const $el = $(tooltipFormatter(event)); + const $rows = $el.find('tr'); + expect($rows.length).toBe(3); + + const $row1 = $rows.eq(0).find('td'); + expect(cell($row1, 0)).toBe('inner'); + expect(cell($row1, 1)).toBe('3'); + + const $row2 = $rows.eq(1).find('td'); + expect(cell($row2, 0)).toBe('third'); + expect(cell($row2, 1)).toBe('2'); + + const $row3 = $rows.eq(2).find('td'); + expect(cell($row3, 0)).toBe('top'); + expect(cell($row3, 1)).toBe('1'); + }); }); diff --git a/yarn.lock b/yarn.lock index d76406f68099..1c0427a98563 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2365,7 +2365,7 @@ dependencies: "@octokit/openapi-types" "^11.2.0" -"@opensearch-project/opensearch@^1.0.2": +"@opensearch-project/opensearch@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@opensearch-project/opensearch/-/opensearch-1.1.0.tgz#8b3c8b4cbcea01755ba092d2997bf0b4ca7f22f7" integrity sha512-1TDw92JL8rD1b2QGluqBsIBLIiD5SGciIpz4qkrGAe9tcdfQ1ptub5e677rhWl35UULSjr6hP8M6HmISZ/M5HQ==