Skip to content

Commit

Permalink
Merge branch 'opensearch-project:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
kavilla authored Apr 15, 2022
2 parents bd287e0 + 4e94ea1 commit 2d85f21
Show file tree
Hide file tree
Showing 20 changed files with 387 additions and 170 deletions.
11 changes: 6 additions & 5 deletions bwctest.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,7 @@

set -e

# TODO: Update to include all known BWC of data
DEFAULT_VERSIONS="osd-1.1.0"
DEFAULT_VERSIONS="osd-1.1.0,odfe-1.13.2,odfe-0.10.0"

function usage() {
echo ""
Expand Down Expand Up @@ -75,12 +74,14 @@ done
[ -z "$BIND_PORT" ] && BIND_PORT="5601"
[ -z "$SECURITY_ENABLED" ] && SECURITY_ENABLED="false"
[ -z "$CREDENTIAL" ] && CREDENTIAL="admin:admin"
[ -z "$CI" ] && CI=1

# If no OpenSearch build was passed then this constructs the version
if [ -z "$OPENSEARCH" ]; then
IFS='/' read -ra SLASH_ARR <<< "$DASHBOARDS"
# Expected to be opensearch-x.y.z-platform-arch.tar.gz
TARBALL="${SLASH_ARR[12]}"
# Playground is supported path to enable sandbox testing
[[ "$DASHBOARDS" == *"Playground"* ]] && TARBALL="${SLASH_ARR[14]}" || TARBALL="${SLASH_ARR[13]}"
IFS='-' read -ra DASH_ARR <<< "$TARBALL"
# Expected to be arch.tar.gz
DOTS="${DASH_ARR[4]}"
Expand All @@ -90,7 +91,7 @@ if [ -z "$OPENSEARCH" ]; then
PLATFORM="${DASH_ARR[3]}"
ARCH="${DOTS_ARR[0]}"

OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz"
OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/tar/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz"
fi

./scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS
source scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS
9 changes: 9 additions & 0 deletions config/opensearch_dashboards.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,15 @@
# must be a positive integer.
#opensearch.requestTimeout: 30000

# Enables the memory circuit breaker that prevents heap out of memory errors for large query responses.
# If enabled, we will provide additional check to prevent potential out of memory error in @openserach-project/opensearch.
# The default threshold is based on the `max-old-space-size` of NodeJS. It is configurable by tuning `opensearch.memoryCircuitBreaker.maxPercentage`.
#opensearch.memoryCircuitBreaker.enabled: false

# The pecentage of maximum heap allowed for processing response. The default value of the pecentage is `1.0`. The valid input range should be [0, 1] inclusively.
# For reference, the `threshold of memoryCircuitBreaker` = `max-old-space-size of NodeJS` * `opensearch.memoryCircuitBreaker.maxPercentage`
#opensearch.memoryCircuitBreaker.maxPercentage: 1.0

# List of OpenSearch Dashboards client-side headers to send to OpenSearch. To send *no* client-side
# headers, set this value to [] (an empty list).
#opensearch.requestHeadersWhitelist: [ authorization ]
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@
"@hapi/podium": "^4.1.3",
"@hapi/vision": "^6.1.0",
"@hapi/wreck": "^17.1.0",
"@opensearch-project/opensearch": "^1.0.2",
"@opensearch-project/opensearch": "^1.1.0",
"@osd/ace": "1.0.0",
"@osd/analytics": "1.0.0",
"@osd/apm-config-loader": "1.0.0",
Expand Down
2 changes: 1 addition & 1 deletion packages/osd-opensearch/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"osd:watch": "node scripts/build --watch"
},
"dependencies": {
"@opensearch-project/opensearch": "^1.0.2",
"@opensearch-project/opensearch": "^1.1.0",
"@osd/dev-utils": "1.0.0",
"abort-controller": "^3.0.0",
"chalk": "^4.1.0",
Expand Down
32 changes: 32 additions & 0 deletions scripts/bwc/opensearch_dashboards_service.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/bin/bash

# Copyright OpenSearch Contributors
# SPDX-License-Identifier: Apache-2.0

set -e

function setup_dashboards() {
cd "$DASHBOARDS_DIR"
[ $SECURITY_ENABLED == "false" ] && [ -d "plugins/securityDashboards" ] && ./bin/opensearch-dashboards-plugin remove securityDashboards
[ $SECURITY_ENABLED == "false" ] && rm config/opensearch_dashboards.yml && touch config/opensearch_dashboards.yml
[ $SECURITY_ENABLED == "false" ] && echo "server.host: 0.0.0.0" >> config/opensearch_dashboards.yml
echo "csp.warnLegacyBrowsers: false" >> config/opensearch_dashboards.yml
echo "--max-old-space-size=5120" >> config/node.options
}

# Starts OpenSearch Dashboards
function run_dashboards() {
echo "[ Attempting to start OpenSearch Dashboards... ]"
cd "$DASHBOARDS_DIR"
spawn_process_and_save_PID "./bin/opensearch-dashboards > ${LOGS_DIR}/opensearch_dashboards.log 2>&1 &"
}

# Checks the running status of OpenSearch Dashboards
# it calls check_status and passes the OpenSearch Dashboards tmp file path, error msg, url, and arguments
# if success, the while loop in the check_status will end and it prints out "OpenSearch Dashboards is up!"
function check_dashboards_status {
echo "Checking the OpenSearch Dashboards..."
cd "$DIR"
check_status $DASHBOARDS_PATH "$DASHBOARDS_MSG" $DASHBOARDS_URL "" >> /dev/null 2>&1
echo "OpenSearch Dashboards is up!"
}
38 changes: 38 additions & 0 deletions scripts/bwc/opensearch_service.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#!/bin/bash

# Copyright OpenSearch Contributors
# SPDX-License-Identifier: Apache-2.0

set -e

function setup_opensearch() {
cd "$OPENSEARCH_DIR"
# Required for IM
[ -d "plugins/opensearch-index-management" ] && echo "path.repo: [/tmp]" >> config/opensearch.yml
# Required for Alerting
[ -d "plugins/opensearch-alerting" ] && echo "plugins.destination.host.deny_list: [\"10.0.0.0/8\", \"127.0.0.1\"]" >> config/opensearch.yml
# Required for SQL
[ -d "plugins/opensearch-sql" ] && echo "script.context.field.max_compilations_rate: 1000/1m" >> config/opensearch.yml
# Required for PA
[ -d "plugins/opensearch-performance-analyzer" ] && echo "webservice-bind-host = 0.0.0.0" >> plugins/opensearch-performance-analyzer/pa_config/performance-analyzer.properties
echo "network.host: 0.0.0.0" >> config/opensearch.yml
echo "discovery.type: single-node" >> config/opensearch.yml
[ $SECURITY_ENABLED == "false" ] && [ -d "plugins/opensearch-security" ] && echo "plugins.security.disabled: true" >> config/opensearch.yml
}

# Starts OpenSearch, if verifying a distribution it will install the certs then start.
function run_opensearch() {
echo "[ Attempting to start OpenSearch... ]"
cd "$OPENSEARCH_DIR"
spawn_process_and_save_PID "./opensearch-tar-install.sh > ${LOGS_DIR}/opensearch.log 2>&1 &"
}

# Checks the running status of OpenSearch
# it calls check_status and passes the OpenSearch tmp file path, error msg, url, and arguments
# if success, the while loop in the check_status will end and it prints out "OpenSearch is up!"
function check_opensearch_status() {
echo "Checking the status OpenSearch..."
cd "$DIR"
check_status $OPENSEARCH_PATH "$OPENSEARCH_MSG" $OPENSEARCH_URL "$OPENSEARCH_ARGS" >> /dev/null 2>&1 &
echo "OpenSearch is up!"
}
69 changes: 69 additions & 0 deletions scripts/bwc/utils.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
#!/bin/bash

# Copyright OpenSearch Contributors
# SPDX-License-Identifier: Apache-2.0

set -e

function open_artifact() {
artifact_dir=$1
artifact=$2
cd $artifact_dir

# check if artifact provided is URL or attempt if passing by absolute path
if curl -I -L $artifact; then
curl -L $artifact | tar -xz --strip-components=1
else
tar -xf $artifact --strip-components=1
fi
}

# remove the running opensearch process
function clean() {
echo "Attempt to Terminate Process with PID: ${PARENT_PID_LIST[*]}"
for pid_kill in "${PARENT_PID_LIST[@]}"
do
echo "Closing PID $pid_kill"
kill $pid_kill || true
done
PARENT_PID_LIST=()
}

function spawn_process_and_save_PID() {
echo "Spawn '$@'"
eval $@
curr_pid=$!
echo "PID: $curr_pid"
PARENT_PID_LIST+=( $curr_pid )
}

# Print out a textfile line by line
function print_txt() {
while IFS= read -r line; do
echo "text read from $1: $line"
done < $1
}

# this function is used to check the running status of OpenSearch or OpenSearch Dashboards
# $1 is the path to the tmp file which saves the running status
# $2 is the error msg to check
# $3 is the url to curl
# $4 contains arguments that need to be passed to the curl command
function check_status() {
while [ ! -f $1 ] || ! grep -q "$2" $1; do
if [ -f $1 ]; then rm $1; fi
curl $3 $4 > $1 || true
done
rm $1
}

# this function copies the tested data for the required version to the opensearch data folder
# $1 is the required version
function upload_data() {
rm -rf "$OPENSEARCH_DIR/data"
cd $OPENSEARCH_DIR
cp "$CWD/cypress/test-data/$DASHBOARDS_TYPE/$1.tar.gz" .
tar -xvf "$OPENSEARCH_DIR/$1.tar.gz" >> /dev/null 2>&1
rm "$1.tar.gz"
echo "Data has been uploaded and ready to test"
}
Loading

0 comments on commit 2d85f21

Please sign in to comment.