diff --git a/.github/workflows/link-checker.yml b/.github/workflows/link-checker.yml index a4b7c5ca3d..f1f7fa30e8 100644 --- a/.github/workflows/link-checker.yml +++ b/.github/workflows/link-checker.yml @@ -12,9 +12,9 @@ jobs: - uses: actions/checkout@v2 - name: lychee Link Checker id: lychee - uses: lycheeverse/lychee-action@v1.5.4 + uses: lycheeverse/lychee-action@v1 with: - args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes --exclude-path tests/ + args: --verbose --max-retries 5 --retry-wait-time 10 --accept=200,403,429 --exclude-path tests/ ./ fail: true env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.lychee.excludes b/.lycheeignore similarity index 54% rename from .lychee.excludes rename to .lycheeignore index e28f9ba462..c23ae6a826 100644 --- a/.lychee.excludes +++ b/.lycheeignore @@ -1,3 +1,10 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + http://staging-artifacts.cloudfront.net/* https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/* http://public.ecr.aws/data-prepper-container-repository @@ -5,3 +12,5 @@ https://www.dummy.com/* # Too many redirects in playground.opensearch.org due to not having user agent information # https://github.com/lycheeverse/lychee/issues/715 https://playground.opensearch.org/* +file://* +https://build.ci.opensearch.org/generic-webhook-trigger/invoke* diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 0ee83ce580..536803bbc3 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -99,7 +99,7 @@ Alternatively, run a command inside the virtualenv with pipenv run. ### Run Tests -This project uses [pytest](https://docs.pytest.org/en/6.x/) to ensure Python code quality. See [tests](tests). +This project uses [pytest](https://docs.pytest.org/en/) to ensure Python code quality. See [tests](tests). ``` $ pipenv run pytest diff --git a/README.md b/README.md index 4e03bf3920..4d568b4dc6 100644 --- a/README.md +++ b/README.md @@ -291,7 +291,7 @@ See [developer guide](DEVELOPER_GUIDE.md) and [how to contribute to this project If you find a bug, or have a feature request, please don't hesitate to open an issue in this repository. -For more information, see [project website](https://opensearch.org/) and [documentation](https://docs-beta.opensearch.org/). If you need help and are unsure where to open an issue, try [forums](https://discuss.opendistrocommunity.dev/). +For more information, see [project website](https://opensearch.org/) and [documentation](https://opensearch.org/docs/). If you need help and are unsure where to open an issue, try [forums](https://discuss.opendistrocommunity.dev/). ## Code of Conduct diff --git a/jenkins/opensearch/benchmark-test.jenkinsfile b/jenkins/opensearch/benchmark-test.jenkinsfile index 6e88944d53..fa6cd85235 100644 --- a/jenkins/opensearch/benchmark-test.jenkinsfile +++ b/jenkins/opensearch/benchmark-test.jenkinsfile @@ -118,7 +118,6 @@ pipeline { H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-0-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-0-slice-1-shard-0-replica-2-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":2};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-0-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-0-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};CAPTURE_NODE_STAT=true;TEST_PROCEDURE=aggs H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true @@ -131,7 +130,6 @@ pipeline { H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-4-slice-1-shard-0-replica-2-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":2};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-4-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true - H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};CAPTURE_NODE_STAT=true;TEST_PROCEDURE=aggs H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true @@ -140,7 +138,10 @@ pipeline { H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-4-clients-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-2-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-2-shards;CAPTURE_NODE_STAT=true H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-5-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-5-shards;DATA_NODE_STORAGE=300;CAPTURE_NODE_STAT=true - H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};CAPTURE_NODE_STAT=true;TEST_PROCEDURE=aggs + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-1-shard-0-replica-2-clients-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":2};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-1-shard-0-replica-4-clients-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-2-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-2-shards;CAPTURE_NODE_STAT=true + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-5-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-5-shards;DATA_NODE_STORAGE=300;CAPTURE_NODE_STAT=true ''' ) } diff --git a/jenkins/release-commit-lock/release-commit-lock.jenkinsfile b/jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile similarity index 85% rename from jenkins/release-commit-lock/release-commit-lock.jenkinsfile rename to jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile index 61acec0fbd..7df3f614f1 100644 --- a/jenkins/release-commit-lock/release-commit-lock.jenkinsfile +++ b/jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile @@ -41,13 +41,13 @@ pipeline { trim: true ) choice( - choices: ['Match Build Manifest', 'Update to Recent Commits'], + choices: ['MATCH_BUILD_MANIFEST', 'UPDATE_TO_RECENT_COMMITS'], name: 'MANIFEST_LOCK_ACTION', - description: 'The manifest lock action to choose.
Match Build Manifest: Will update the manifest with commit ID from release candidate build manifest.
Update to Recent Commits: Will update the manifst with component repo release branch head commit', + description: 'The manifest lock action to choose.
MATCH_BUILD_MANIFEST: Will update the manifest with commit ID from release candidate build manifest.
UPDATE_TO_RECENT_COMMITS: Will update the manifest with component repo release branch head commit.', ) string( name: 'COMPONENTS', - description: 'The selected components from the manifesr to updatet the commit ID', + description: 'The selected components from the manifest to update the commit ID', trim: true ) } @@ -63,15 +63,15 @@ pipeline { } } } - stage('Match Build Manifest') { + stage('MATCH_BUILD_MANIFEST') { when { - expression { params.MANIFEST_LOCK_ACTION == 'Match Build Manifest' } + expression { params.MANIFEST_LOCK_ACTION == 'MATCH_BUILD_MANIFEST' } } steps { script { def updateManifest = { String productName, String releaseCandidate -> def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-${productName}/${params.RELEASE_VERSION}/${releaseCandidate}/linux/x64/tar/dist/${productName}/manifest.yml" - def buildManifestContent = sh(script: "curl -s ${buildManifestUrl}", returnStdout: true).trim() + def buildManifestContent = sh(script: "curl -sSL ${buildManifestUrl}", returnStdout: true).trim() def buildManifest = readYaml text: buildManifestContent def existingManifest = readYaml file: "manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml" def selectedComponents = params.COMPONENTS.isEmpty() ? existingManifest.components : existingManifest.components.findAll { component -> @@ -90,16 +90,16 @@ pipeline { sh """ yq eval -i '.' manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml sed -i '1s/^/---\\n/' manifests/${params.RELEASE_VERSION}/${productName}-${params.RELEASE_VERSION}.yml - """ + """ } updateManifest("opensearch", params.OPENSEARCH_RELEASE_CANDIDATE) updateManifest("opensearch-dashboards", params.OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE) } } } - stage('Update to Recent Commits') { + stage('UPDATE_TO_RECENT_COMMITS') { when { - expression { params.MANIFEST_LOCK_ACTION == 'Update to Recent Commits' } + expression { params.MANIFEST_LOCK_ACTION == 'UPDATE_TO_RECENT_COMMITS' } } steps { script { @@ -135,17 +135,18 @@ pipeline { withCredentials([usernamePassword(credentialsId: 'jenkins-github-bot-token', passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER')]) { try { sh """ - git remote set-url origin https://prudhvigodithi:${GITHUB_TOKEN}@github.com/prudhvigodithi/opensearch-build - git config user.email "pgodithi@amazon.com" - git config user.name "prudhvigodithi" + git remote set-url origin "https://opensearch-ci:${GITHUB_TOKEN}@github.com/opensearch-project/opensearch-build" + git config user.email "opensearch-infra@amazon.com" + git config user.name "opensearch-ci" git checkout -b manifest-lock - """ + """ def status = sh(returnStdout: true, script: 'git status --porcelain') if (status) { sh """ - git add . && git commit -sm "Manifest Commit Lock" + git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add + git commit -sm "Manifest Commit Lock for Release ${params.RELEASE_VERSION}" git push origin manifest-lock --force - gh pr create --title '[${params.RELEASE_VERSION}] Manifest Commit Lock' --body 'Manifest Commit Lock for Release ${params.RELEASE_VERSION} ' -H manifest-lock -B main + gh pr create --title '[${params.RELEASE_VERSION}] Manifest Commit Lock with action ${params.MANIFEST_LOCK_ACTION}' --body 'Manifest Commit Lock for Release ${params.RELEASE_VERSION} ' -H manifest-lock -B main """ } else { println 'Nothing to commit!' diff --git a/manifests/2.13.0/opensearch-2.13.0.yml b/manifests/2.13.0/opensearch-2.13.0.yml index f7b2cf9781..b5be4b9f94 100644 --- a/manifests/2.13.0/opensearch-2.13.0.yml +++ b/manifests/2.13.0/opensearch-2.13.0.yml @@ -10,7 +10,7 @@ ci: components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git - ref: 7ec678d1b7c87d6e779fdef94e33623e1f1e2647 + ref: '2.x' - name: common-utils repository: https://github.com/opensearch-project/common-utils.git ref: 937bc4ecdf5ff07a1087b259c330c9b361db5c32 diff --git a/release-notes/opensearch-release-notes-1.2.2.md b/release-notes/opensearch-release-notes-1.2.2.md index 1aa8d9cb94..11d167bd7f 100644 --- a/release-notes/opensearch-release-notes-1.2.2.md +++ b/release-notes/opensearch-release-notes-1.2.2.md @@ -2,7 +2,7 @@ ## Release Highlights -This patch releases updates the version of Log4j used in OpenSearch to Log4j 2.16.0 as recommended by the advisory in [CVE-2021-45046](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-45046). +This patch releases updates the version of Log4j used in OpenSearch to Log4j 2.16.0 as recommended by the advisory in [CVE-2021-45046](https://www.cve.org/CVERecord?id=CVE-2021-45046). ### OpenSearch * Increment version to 1.2.2 and backport log4j upgrade to 2.16. ([#1728](https://github.com/opensearch-project/OpenSearch/pull/1728)) diff --git a/release-notes/opensearch-release-notes-1.2.3.md b/release-notes/opensearch-release-notes-1.2.3.md index be832507a3..e6f41ad7a0 100644 --- a/release-notes/opensearch-release-notes-1.2.3.md +++ b/release-notes/opensearch-release-notes-1.2.3.md @@ -2,7 +2,7 @@ ## Release Highlights -This patch releases updates the version of Log4j used in OpenSearch to Log4j 2.17.0 as recommended by the advisory in [CVE-2021-45105](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-45105). +This patch releases updates the version of Log4j used in OpenSearch to Log4j 2.17.0 as recommended by the advisory in [CVE-2021-45105](https://www.cve.org/CVERecord?id=CVE-2021-45105). ### OpenSearch diff --git a/release-notes/opensearch-release-notes-2.1.0.md b/release-notes/opensearch-release-notes-2.1.0.md index a94b3324a5..c1239c9994 100644 --- a/release-notes/opensearch-release-notes-2.1.0.md +++ b/release-notes/opensearch-release-notes-2.1.0.md @@ -157,7 +157,7 @@ ES * Bump to 2.1.0 compatibility ([#282](https://github.com/opensearch-project/anomaly-detection-dashboards-plugin/pull/282)) ### Opensearch Alerting -* Bumped version to 2.1.0, and gradle to 7.4.2. ([#475](https://github.com/opensearch-project/alerting/pull/475])) +* Bumped version to 2.1.0, and gradle to 7.4.2. ([#475](https://github.com/opensearch-project/alerting/pull/475)) ### Opensearch Alerting Dashboards Plugin diff --git a/src/report_workflow/README.md b/src/report_workflow/README.md index 289b53cacd..52ee2a9f39 100644 --- a/src/report_workflow/README.md +++ b/src/report_workflow/README.md @@ -55,8 +55,8 @@ The test-report manifest includes the following attributes: - Find the bundle manifest under the DistributionManifest key. It contains information about the bundle artifacts used for testing. ### _**How Do I See the CI Logs of This Test Run?**_ - Access our Jenkins job: - - For `OpenSearch`: `https://build.ci.opensearch.org/job/<__TestType__>/<__TestID__>/`. e.g. https://build.ci.opensearch.org/job/integ-test/1234/ - - For `OpenSearch Dashboards`: `https://build.ci.opensearch.org/job/<__TestType__>-opensearch-dashboards/<__TestID__>/`. e.g. https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/1234/ + - For `OpenSearch`: `https://build.ci.opensearch.org/job/<__TestType__>/<__TestID__>/`. e.g. `https://build.ci.opensearch.org/job/integ-test/<__TestID__>/` + - For `OpenSearch Dashboards`: `https://build.ci.opensearch.org/job/<__TestType__>-opensearch-dashboards/<__TestID__>/`. e.g. `https://build.ci.opensearch.org/job/integ-test-opensearch-dashboards/<__TestID__>/` ### _**Where Can I See Detailed Testing Logs or Service Logs (e.g., Local Cluster Logs, Testing Videos)?**_ - You can find details in the component YAML file under `components/configs/yml` in this test-report manifest. The component YAML file includes local cluster logs, stdout & stderr files, specific testing logs, and Cypress videos. ### _**Why Are Some Component Testing Results Missing or Unavailable?**_ diff --git a/tests/jenkins/TestReleaseManifestCommitLock.groovy b/tests/jenkins/TestReleaseManifestCommitLock.groovy new file mode 100644 index 0000000000..628f69d33f --- /dev/null +++ b/tests/jenkins/TestReleaseManifestCommitLock.groovy @@ -0,0 +1,118 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test +import org.yaml.snakeyaml.Yaml +import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library +import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.MatcherAssert.assertThat +import static org.junit.jupiter.api.Assertions.assertThrows +import com.lesfurets.jenkins.unit.* +import groovy.json.JsonOutput + +class TestReleaseManifestCommitLock extends BuildPipelineTest { + + @Override + @Before + void setUp() { + + helper.registerSharedLibrary( + library().name('jenkins') + .defaultVersion('6.4.0') + .allowOverride(true) + .implicit(true) + .targetPath('vars') + .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git')) + .build() + ) + + super.setUp() + addParam('RELEASE_VERSION', '2.0.0') + addParam('OPENSEARCH_RELEASE_CANDIDATE', '3813') + addParam('OPENSEARCH_DASHBOARDS_RELEASE_CANDIDATE', '3050') + addParam('COMPONENTS', 'OpenSearch') + + helper.registerAllowedMethod("withCredentials", [Map]) + def buildManifest = "tests/jenkins/data/opensearch-2.0.0.yml" + helper.registerAllowedMethod('readYaml', [Map.class], { args -> + return new Yaml().load((buildManifest as File).text) + }) + helper.registerAllowedMethod("writeYaml", [Map.class], {c -> }) + } + + @Test + public void testManifestCommitLock_matchBuildManifest() { + addParam('MANIFEST_LOCK_ACTION', 'MATCH_BUILD_MANIFEST') + super.testPipeline('jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile', + 'tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_matchBuildManifest') + def callStack = helper.getCallStack() + assertCallStack().contains('stage(Parameters Check, groovy.lang.Closure)') + assertCallStack().contains('stage(MATCH_BUILD_MANIFEST, groovy.lang.Closure)') + } + + @Test + public void testManifestCommitLock_updateToRecentCommits() { + addParam('MANIFEST_LOCK_ACTION', 'UPDATE_TO_RECENT_COMMITS') + super.testPipeline('jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile', + 'tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToRecentCommits') + def callStack = helper.getCallStack() + assertCallStack().contains('stage(Parameters Check, groovy.lang.Closure)') + assertCallStack().contains('stage(UPDATE_TO_RECENT_COMMITS, groovy.lang.Closure)') + } + + @Test + public void testManifestCommitLock_createPullRequest() { + super.testPipeline('jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile', + 'tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_createPullRequest') + assertThat(getShellCommands('git'), hasItem("\n git remote set-url origin \"https://opensearch-ci:GITHUB_TOKEN@github.com/opensearch-project/opensearch-build\"\n git config user.email \"opensearch-infra@amazon.com\"\n git config user.name \"opensearch-ci\"\n git checkout -b manifest-lock\n ")) + assertThat(getShellCommands('git'), hasItem("\n git status --porcelain | grep '^ M' | cut -d \" \" -f3 | xargs git add\n git commit -sm \"Manifest Commit Lock for Release 2.0.0\"\n git push origin manifest-lock --force\n gh pr create --title '[2.0.0] Manifest Commit Lock with action MATCH_BUILD_MANIFEST' --body 'Manifest Commit Lock for Release 2.0.0 ' -H manifest-lock -B main\n ")) + } + + @Test + public void testUpdateToRecentCommit() { + addParam('MANIFEST_LOCK_ACTION', 'UPDATE_TO_RECENT_COMMITS') + helper.addShMock("""git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.0""") { script -> + return [stdout: "", exitValue: 0] + } + super.testPipeline('jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile', + 'tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit') + assertCallStack().contains('release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml})') + assertCallStack().contains('release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml})') + assertCallStack().contains('release-manifest-commit-lock.sh({script=git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.0 | cut -f 1, returnStdout=true})') + assertCallStack().contains("release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml") + assertCallStack().contains("{name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}") + assertCallStack().contains("release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml})") + } + + @Test + public void testMatchBuildManifest() { + addParam('MANIFEST_LOCK_ACTION', 'MATCH_BUILD_MANIFEST') + def buildManifest = "tests/jenkins/data/opensearch-2.0.0-build.yml" + helper.registerAllowedMethod('readYaml', [Map.class], { args -> + return new Yaml().load((buildManifest as File).text) + }) + super.testPipeline('jenkins/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile', + 'tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testMatchBuildManifest') + assertThat(getShellCommands('curl'), hasItem("{script=curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/dist/opensearch/manifest.yml, returnStdout=true}")) + assertCallStack().contains("release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.1, build={name=OpenSearch, version=2.0.0, platform=linux, architecture=x64, distribution=tar, location=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/dist/opensearch/opensearch-2.0.0-linux-x64.tar.gz, id=3813}") + assertCallStack().contains("release-manifest-commit-lock.writeYaml(") + + } + + def getShellCommands(searchtext) { + def shCommands = helper.callStack.findAll { call -> + call.methodName == 'sh' + }.collect { call -> + callArgsToString(call) + }.findAll { gitCommand -> + gitCommand.contains(searchtext) + } + return shCommands + } +} diff --git a/tests/jenkins/data/opensearch-2.0.0-build.yml b/tests/jenkins/data/opensearch-2.0.0-build.yml new file mode 100644 index 0000000000..6c501a976e --- /dev/null +++ b/tests/jenkins/data/opensearch-2.0.0-build.yml @@ -0,0 +1,30 @@ +--- +schema-version: '1.1' +build: + name: OpenSearch + version: 2.0.0 + platform: linux + architecture: x64 + distribution: tar + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/dist/opensearch/opensearch-2.0.0-linux-x64.tar.gz + id: '3813' +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: '2.0' + commit_id: bae3b4e4178c20ac24fece8e82099abe3b2630d0 + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/builds/opensearch/dist/opensearch-min-2.0.0-linux-x64.tar.gz + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: '2.0' + commit_id: e59ea173af31fd468ce443fc4022649cad306e36 + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: '2.0' + commit_id: b5b21097894ecec7a78da622ee96763908b32898 + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/builds/opensearch/plugins/opensearch-job-scheduler-2.0.0.0.zip + - name: ml-commons + repository: https://github.com/opensearch-project/ml-commons.git + ref: '2.0' + commit_id: 5c6e4bd4d996cf2d0a9726e1537ef98822d1795f + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/builds/opensearch/plugins/opensearch-ml-2.0.0.0.zip diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt index 7268944866..6fbce67dbf 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt @@ -96,7 +96,6 @@ H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-0-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-0-slice-1-shard-0-replica-2-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":2};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-0-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-0-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};CAPTURE_NODE_STAT=true;TEST_PROCEDURE=aggs H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true @@ -109,7 +108,6 @@ H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-4-slice-1-shard-0-replica-2-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":2};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-4-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true - H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};CAPTURE_NODE_STAT=true;TEST_PROCEDURE=aggs H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true @@ -118,7 +116,10 @@ H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-4-clients-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-2-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-2-shards;CAPTURE_NODE_STAT=true H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-5-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-5-shards;DATA_NODE_STORAGE=300;CAPTURE_NODE_STAT=true - H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};CAPTURE_NODE_STAT=true;TEST_PROCEDURE=aggs + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-1-shard-0-replica-2-clients-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":2};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-1-shard-0-replica-4-clients-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-2-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-2-shards;CAPTURE_NODE_STAT=true + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-5-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-5-shards;DATA_NODE_STORAGE=300;CAPTURE_NODE_STAT=true ) benchmark-test.stage(validate-and-set-parameters, groovy.lang.Closure) benchmark-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Benchmark-Test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt index 33d2aa1c6c..5fbf399a34 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt @@ -96,7 +96,6 @@ H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-0-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-0-slice-1-shard-0-replica-2-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":2};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-0-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-0-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};CAPTURE_NODE_STAT=true;TEST_PROCEDURE=aggs H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true @@ -109,7 +108,6 @@ H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-4-slice-1-shard-0-replica-2-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":2};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-4-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true - H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};CAPTURE_NODE_STAT=true;TEST_PROCEDURE=aggs H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true @@ -118,7 +116,10 @@ H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-4-clients-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-2-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-2-shards;CAPTURE_NODE_STAT=true H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-5-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-5-shards;DATA_NODE_STORAGE=300;CAPTURE_NODE_STAT=true - H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};CAPTURE_NODE_STAT=true;TEST_PROCEDURE=aggs + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-1-shard-0-replica-2-clients-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":2};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-1-shard-0-replica-4-clients-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-2-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-2-shards;CAPTURE_NODE_STAT=true + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-5-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-5-shards;DATA_NODE_STORAGE=300;CAPTURE_NODE_STAT=true ) benchmark-test.stage(validate-and-set-parameters, groovy.lang.Closure) benchmark-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Benchmark-Test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt index a9fa5dd08c..45039255da 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt @@ -96,7 +96,6 @@ H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-0-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-0-slice-1-shard-0-replica-2-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":2};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-0-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true - H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-0-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};CAPTURE_NODE_STAT=true;TEST_PROCEDURE=aggs H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true @@ -109,7 +108,6 @@ H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-4-slice-1-shard-0-replica-2-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":2};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-concurrent-search-4-slice-1-shard-0-replica-4-clients-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true - H 15 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-concurrent-search-4-slice-1-shard-0-replica-snapshot;ADDITIONAL_CONFIG=search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:4;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};CAPTURE_NODE_STAT=true;TEST_PROCEDURE=aggs H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"big5_1_shard"};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true @@ -118,7 +116,10 @@ H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-4-clients-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-2-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-2-shards;CAPTURE_NODE_STAT=true H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-5-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-5-shards;DATA_NODE_STORAGE=300;CAPTURE_NODE_STAT=true - H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.2xlarge,major-version:2x,cluster-config:x64-r5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"};CAPTURE_NODE_STAT=true;TEST_PROCEDURE=aggs + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-1-shard-0-replica-2-clients-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":2};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-1-shard-0-replica-4-clients-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard","search_clients":4};TEST_PROCEDURE=restore-from-snapshot;CAPTURE_NODE_STAT=true + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-2-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-2-shards;CAPTURE_NODE_STAT=true + H 16 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.13.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=r5.8xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.8xlarge,major-version:2x,cluster-config:x64-r5.8xlarge-5-shard-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_1_shard"};TEST_PROCEDURE=restore-from-snapshot-5-shards;DATA_NODE_STORAGE=300;CAPTURE_NODE_STAT=true ) benchmark-test.stage(validate-and-set-parameters, groovy.lang.Closure) benchmark-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Benchmark-Test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile.txt new file mode 100644 index 0000000000..29a2a97233 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/release-manifest-commit-lock.jenkinsfile.txt @@ -0,0 +1,46 @@ + release-manifest-commit-lock.run() + release-manifest-commit-lock.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + release-manifest-commit-lock.library({identifier=jenkins@6.4.0, retriever=null}) + release-manifest-commit-lock.pipeline(groovy.lang.Closure) + release-manifest-commit-lock.timeout({time=2, unit=HOURS}) + release-manifest-commit-lock.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) + release-manifest-commit-lock.stage(Parameters Check, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.echo(Skipping stage Match Build Manifest) + release-manifest-commit-lock.stage(Update to Recent Commits, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-2.0.0.yml}) + release-manifest-commit-lock.sh({script=git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.0 | cut -f 1, returnStdout=true}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=bbb +ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-2.0.0.yml + ) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml}) + release-manifest-commit-lock.sh({script=git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.0 | cut -f 1, returnStdout=true}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=bbb +ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + ) + release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + release-manifest-commit-lock.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + release-manifest-commit-lock.sh( + git remote set-url origin https://opensearch-ci:GITHUB_TOKEN@github.com/opensearch-project/opensearch-build + git config user.email "opensearch-infra@amazon.com" + git config user.name "opensearch-ci" + git checkout -b manifest-lock + ) + release-manifest-commit-lock.sh({returnStdout=true, script=git status --porcelain}) + release-manifest-commit-lock.sh( + git add . && git commit -sm "Manifest Commit Lock" + git push origin manifest-lock --force + gh pr create --title '[2.0.0] Manifest Commit Lock, Update to Recent Commits' --body 'Manifest Commit Lock for Release 2.0.0 ' -H manifest-lock -B main + ) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_createPullRequest.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_createPullRequest.txt new file mode 100644 index 0000000000..9d71e7b881 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_createPullRequest.txt @@ -0,0 +1,49 @@ + release-manifest-commit-lock.run() + release-manifest-commit-lock.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + release-manifest-commit-lock.library({identifier=jenkins@6.4.0, retriever=null}) + release-manifest-commit-lock.pipeline(groovy.lang.Closure) + release-manifest-commit-lock.timeout({time=2, unit=HOURS}) + release-manifest-commit-lock.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) + release-manifest-commit-lock.stage(Parameters Check, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.stage(MATCH_BUILD_MANIFEST, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.sh({script=curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/dist/opensearch/manifest.yml, returnStdout=true}) + release-manifest-commit-lock.readYaml({text=bbb +ccc}) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-2.0.0.yml}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=2.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-2.0.0.yml + ) + release-manifest-commit-lock.sh({script=curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/2.0.0/3050/linux/x64/tar/dist/opensearch-dashboards/manifest.yml, returnStdout=true}) + release-manifest-commit-lock.readYaml({text=bbb +ccc}) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=2.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + ) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS) + release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + release-manifest-commit-lock.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + release-manifest-commit-lock.sh( + git remote set-url origin "https://opensearch-ci:GITHUB_TOKEN@github.com/opensearch-project/opensearch-build" + git config user.email "opensearch-infra@amazon.com" + git config user.name "opensearch-ci" + git checkout -b manifest-lock + ) + release-manifest-commit-lock.sh({returnStdout=true, script=git status --porcelain}) + release-manifest-commit-lock.sh( + git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add + git commit -sm "Manifest Commit Lock for Release 2.0.0" + git push origin manifest-lock --force + gh pr create --title '[2.0.0] Manifest Commit Lock with action MATCH_BUILD_MANIFEST' --body 'Manifest Commit Lock for Release 2.0.0 ' -H manifest-lock -B main + ) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_matchBuildManifest.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_matchBuildManifest.txt new file mode 100644 index 0000000000..9d71e7b881 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_matchBuildManifest.txt @@ -0,0 +1,49 @@ + release-manifest-commit-lock.run() + release-manifest-commit-lock.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + release-manifest-commit-lock.library({identifier=jenkins@6.4.0, retriever=null}) + release-manifest-commit-lock.pipeline(groovy.lang.Closure) + release-manifest-commit-lock.timeout({time=2, unit=HOURS}) + release-manifest-commit-lock.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) + release-manifest-commit-lock.stage(Parameters Check, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.stage(MATCH_BUILD_MANIFEST, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.sh({script=curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/dist/opensearch/manifest.yml, returnStdout=true}) + release-manifest-commit-lock.readYaml({text=bbb +ccc}) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-2.0.0.yml}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=2.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-2.0.0.yml + ) + release-manifest-commit-lock.sh({script=curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/2.0.0/3050/linux/x64/tar/dist/opensearch-dashboards/manifest.yml, returnStdout=true}) + release-manifest-commit-lock.readYaml({text=bbb +ccc}) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=2.0, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + ) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS) + release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + release-manifest-commit-lock.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + release-manifest-commit-lock.sh( + git remote set-url origin "https://opensearch-ci:GITHUB_TOKEN@github.com/opensearch-project/opensearch-build" + git config user.email "opensearch-infra@amazon.com" + git config user.name "opensearch-ci" + git checkout -b manifest-lock + ) + release-manifest-commit-lock.sh({returnStdout=true, script=git status --porcelain}) + release-manifest-commit-lock.sh( + git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add + git commit -sm "Manifest Commit Lock for Release 2.0.0" + git push origin manifest-lock --force + gh pr create --title '[2.0.0] Manifest Commit Lock with action MATCH_BUILD_MANIFEST' --body 'Manifest Commit Lock for Release 2.0.0 ' -H manifest-lock -B main + ) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToRecentCommits.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToRecentCommits.txt new file mode 100644 index 0000000000..6c3a2c6101 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testManifestCommitLock_updateToRecentCommits.txt @@ -0,0 +1,47 @@ + release-manifest-commit-lock.run() + release-manifest-commit-lock.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + release-manifest-commit-lock.library({identifier=jenkins@6.4.0, retriever=null}) + release-manifest-commit-lock.pipeline(groovy.lang.Closure) + release-manifest-commit-lock.timeout({time=2, unit=HOURS}) + release-manifest-commit-lock.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) + release-manifest-commit-lock.stage(Parameters Check, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.echo(Skipping stage MATCH_BUILD_MANIFEST) + release-manifest-commit-lock.stage(UPDATE_TO_RECENT_COMMITS, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-2.0.0.yml}) + release-manifest-commit-lock.sh({script=git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.0 | cut -f 1, returnStdout=true}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=bbb +ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-2.0.0.yml + ) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml}) + release-manifest-commit-lock.sh({script=git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.0 | cut -f 1, returnStdout=true}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=bbb +ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + ) + release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + release-manifest-commit-lock.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + release-manifest-commit-lock.sh( + git remote set-url origin "https://opensearch-ci:GITHUB_TOKEN@github.com/opensearch-project/opensearch-build" + git config user.email "opensearch-infra@amazon.com" + git config user.name "opensearch-ci" + git checkout -b manifest-lock + ) + release-manifest-commit-lock.sh({returnStdout=true, script=git status --porcelain}) + release-manifest-commit-lock.sh( + git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add + git commit -sm "Manifest Commit Lock for Release 2.0.0" + git push origin manifest-lock --force + gh pr create --title '[2.0.0] Manifest Commit Lock with action UPDATE_TO_RECENT_COMMITS' --body 'Manifest Commit Lock for Release 2.0.0 ' -H manifest-lock -B main + ) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testMatchBuildManifest.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testMatchBuildManifest.txt new file mode 100644 index 0000000000..031eb83d56 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testMatchBuildManifest.txt @@ -0,0 +1,49 @@ + release-manifest-commit-lock.run() + release-manifest-commit-lock.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + release-manifest-commit-lock.library({identifier=jenkins@6.4.0, retriever=null}) + release-manifest-commit-lock.pipeline(groovy.lang.Closure) + release-manifest-commit-lock.timeout({time=2, unit=HOURS}) + release-manifest-commit-lock.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) + release-manifest-commit-lock.stage(Parameters Check, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.stage(MATCH_BUILD_MANIFEST, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.sh({script=curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/dist/opensearch/manifest.yml, returnStdout=true}) + release-manifest-commit-lock.readYaml({text=bbb +ccc}) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-2.0.0.yml}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.1, build={name=OpenSearch, version=2.0.0, platform=linux, architecture=x64, distribution=tar, location=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/dist/opensearch/opensearch-2.0.0-linux-x64.tar.gz, id=3813}, components=[{name=OpenSearch, repository=https://github.com/opensearch-project/OpenSearch.git, ref=2.0, commit_id=bae3b4e4178c20ac24fece8e82099abe3b2630d0, location=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/builds/opensearch/dist/opensearch-min-2.0.0-linux-x64.tar.gz}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, commit_id=e59ea173af31fd468ce443fc4022649cad306e36}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, commit_id=b5b21097894ecec7a78da622ee96763908b32898, location=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/builds/opensearch/plugins/opensearch-job-scheduler-2.0.0.0.zip}, {name=ml-commons, repository=https://github.com/opensearch-project/ml-commons.git, ref=2.0, commit_id=5c6e4bd4d996cf2d0a9726e1537ef98822d1795f, location=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/builds/opensearch/plugins/opensearch-ml-2.0.0.0.zip}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-2.0.0.yml + ) + release-manifest-commit-lock.sh({script=curl -sSL https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/2.0.0/3050/linux/x64/tar/dist/opensearch-dashboards/manifest.yml, returnStdout=true}) + release-manifest-commit-lock.readYaml({text=bbb +ccc}) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml, data={schema-version=1.1, build={name=OpenSearch, version=2.0.0, platform=linux, architecture=x64, distribution=tar, location=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/dist/opensearch/opensearch-2.0.0-linux-x64.tar.gz, id=3813}, components=[{name=OpenSearch, repository=https://github.com/opensearch-project/OpenSearch.git, ref=2.0, commit_id=bae3b4e4178c20ac24fece8e82099abe3b2630d0, location=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/builds/opensearch/dist/opensearch-min-2.0.0-linux-x64.tar.gz}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, commit_id=e59ea173af31fd468ce443fc4022649cad306e36}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, commit_id=b5b21097894ecec7a78da622ee96763908b32898, location=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/builds/opensearch/plugins/opensearch-job-scheduler-2.0.0.0.zip}, {name=ml-commons, repository=https://github.com/opensearch-project/ml-commons.git, ref=2.0, commit_id=5c6e4bd4d996cf2d0a9726e1537ef98822d1795f, location=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/3813/linux/x64/tar/builds/opensearch/plugins/opensearch-ml-2.0.0.0.zip}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + ) + release-manifest-commit-lock.echo(Skipping stage UPDATE_TO_RECENT_COMMITS) + release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + release-manifest-commit-lock.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + release-manifest-commit-lock.sh( + git remote set-url origin "https://opensearch-ci:GITHUB_TOKEN@github.com/opensearch-project/opensearch-build" + git config user.email "opensearch-infra@amazon.com" + git config user.name "opensearch-ci" + git checkout -b manifest-lock + ) + release-manifest-commit-lock.sh({returnStdout=true, script=git status --porcelain}) + release-manifest-commit-lock.sh( + git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add + git commit -sm "Manifest Commit Lock for Release 2.0.0" + git push origin manifest-lock --force + gh pr create --title '[2.0.0] Manifest Commit Lock with action MATCH_BUILD_MANIFEST' --body 'Manifest Commit Lock for Release 2.0.0 ' -H manifest-lock -B main + ) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit.txt b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit.txt new file mode 100644 index 0000000000..6c3a2c6101 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/release-manifest-commit-lock/testUpdateToRecentCommit.txt @@ -0,0 +1,47 @@ + release-manifest-commit-lock.run() + release-manifest-commit-lock.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + release-manifest-commit-lock.library({identifier=jenkins@6.4.0, retriever=null}) + release-manifest-commit-lock.pipeline(groovy.lang.Closure) + release-manifest-commit-lock.timeout({time=2, unit=HOURS}) + release-manifest-commit-lock.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) + release-manifest-commit-lock.stage(Parameters Check, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.echo(Skipping stage MATCH_BUILD_MANIFEST) + release-manifest-commit-lock.stage(UPDATE_TO_RECENT_COMMITS, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-2.0.0.yml}) + release-manifest-commit-lock.sh({script=git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.0 | cut -f 1, returnStdout=true}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=bbb +ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-2.0.0.yml + ) + release-manifest-commit-lock.readYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml}) + release-manifest-commit-lock.sh({script=git ls-remote https://github.com/opensearch-project/OpenSearch.git 2.0 | cut -f 1, returnStdout=true}) + release-manifest-commit-lock.writeYaml({file=manifests/2.0.0/opensearch-dashboards-2.0.0.yml, data={schema-version=1.0, build={name=OpenSearch, version=2.0.0, qualifier=alpha1}, ci={image={name=opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2, args=-e JAVA_HOME=/opt/java/openjdk-17}}, components=[{name=OpenSearch, ref=bbb +ccc, repository=https://github.com/opensearch-project/OpenSearch.git, checks=[gradle:publish, gradle:properties:version]}, {name=common-utils, repository=https://github.com/opensearch-project/common-utils.git, ref=2.0, checks=[gradle:publish, gradle:properties:version]}, {name=job-scheduler, repository=https://github.com/opensearch-project/job-scheduler.git, ref=2.0, checks=[gradle:properties:version, gradle:dependencies:opensearch.version]}]}, overwrite=true}) + release-manifest-commit-lock.sh( + yq eval -i '.' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + sed -i '1s/^/---\n/' manifests/2.0.0/opensearch-dashboards-2.0.0.yml + ) + release-manifest-commit-lock.stage(Create Pull Request, groovy.lang.Closure) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.usernamePassword({credentialsId=jenkins-github-bot-token, passwordVariable=GITHUB_TOKEN, usernameVariable=GITHUB_USER}) + release-manifest-commit-lock.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) + release-manifest-commit-lock.sh( + git remote set-url origin "https://opensearch-ci:GITHUB_TOKEN@github.com/opensearch-project/opensearch-build" + git config user.email "opensearch-infra@amazon.com" + git config user.name "opensearch-ci" + git checkout -b manifest-lock + ) + release-manifest-commit-lock.sh({returnStdout=true, script=git status --porcelain}) + release-manifest-commit-lock.sh( + git status --porcelain | grep '^ M' | cut -d " " -f3 | xargs git add + git commit -sm "Manifest Commit Lock for Release 2.0.0" + git push origin manifest-lock --force + gh pr create --title '[2.0.0] Manifest Commit Lock with action UPDATE_TO_RECENT_COMMITS' --body 'Manifest Commit Lock for Release 2.0.0 ' -H manifest-lock -B main + ) + release-manifest-commit-lock.script(groovy.lang.Closure) + release-manifest-commit-lock.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})