diff --git a/manifests/2.16.1/opensearch-2.16.1-test.yml b/manifests/2.16.1/opensearch-2.16.1-test.yml new file mode 100644 index 0000000000..c181699bad --- /dev/null +++ b/manifests/2.16.1/opensearch-2.16.1-test.yml @@ -0,0 +1,143 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-21 +components: + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: + - 10.0.0.0/8 + - 127.0.0.1 + bwc-test: + test-configs: + - with-security + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - name: flow-framework + integ-test: + test-configs: + - with-security + - without-security + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - name: cross-cluster-replication + integ-test: + topology: + - cluster_name: leader + data_nodes: 2 + - cluster_name: follower + data_nodes: 2 + test-configs: + - with-security + - without-security + - name: geospatial + integ-test: + test-configs: + - with-security + - without-security + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: + - /tmp + bwc-test: + test-configs: + - with-security + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security + - name: neural-search + integ-test: + test-configs: + - with-security + - without-security + - name: notifications + working-directory: notifications + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - name: opensearch-observability + integ-test: + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + - name: opensearch-reports + integ-test: + test-configs: + - with-security + - without-security + - name: security + integ-test: + test-configs: + - with-security + - name: security-analytics + integ-test: + test-configs: + - with-security + - without-security + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + plugins.query.datasources.encryption.masterkey: 4fc8fee6a3fd7d6ca01772e5 + bwc-test: + test-configs: + - with-security + - name: custom-codecs + integ-test: + test-configs: + - with-security + - without-security + - name: skills + integ-test: + test-configs: + - with-security + - without-security + - name: query-insights + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/3.0.0/opensearch-3.0.0-test.yml b/manifests/3.0.0/opensearch-3.0.0-test.yml index 6a63f55349..15006c19fe 100644 --- a/manifests/3.0.0/opensearch-3.0.0-test.yml +++ b/manifests/3.0.0/opensearch-3.0.0-test.yml @@ -3,7 +3,7 @@ schema-version: '1.0' name: OpenSearch ci: image: - name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 args: -e JAVA_HOME=/opt/java/openjdk-21 components: - name: alerting diff --git a/manifests/3.0.0/opensearch-dashboards-3.0.0-test.yml b/manifests/3.0.0/opensearch-dashboards-3.0.0-test.yml index 7fcfac61a0..f5b041c241 100644 --- a/manifests/3.0.0/opensearch-dashboards-3.0.0-test.yml +++ b/manifests/3.0.0/opensearch-dashboards-3.0.0-test.yml @@ -13,6 +13,12 @@ components: additional-cluster-configs: vis_builder.enabled: true data_source.enabled: true + savedObjects.maxImportPayloadBytes: 10485760 + server.maxPayloadBytes: 1759977 + logging.json: false + data.search.aggs.shardDelay.enabled: true + csp.warnLegacyBrowsers: false + ci-groups: 9 - name: alertingDashboards integ-test: test-configs: diff --git a/manifests/templates/opensearch-dashboards/1.x/manifest-test.yml b/manifests/templates/opensearch-dashboards/1.x/manifest-test.yml new file mode 100644 index 0000000000..f653b3fd30 --- /dev/null +++ b/manifests/templates/opensearch-dashboards/1.x/manifest-test.yml @@ -0,0 +1,12 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v4 +components: + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/templates/opensearch-dashboards/2.x/manifest-test.yml b/manifests/templates/opensearch-dashboards/2.x/manifest-test.yml new file mode 100644 index 0000000000..bc2310e9b1 --- /dev/null +++ b/manifests/templates/opensearch-dashboards/2.x/manifest-test.yml @@ -0,0 +1,26 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-dashboards-integtest-v1 +components: + - name: OpenSearch-Dashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + vis_builder.enabled: true + data_source.enabled: true + savedObjects.maxImportPayloadBytes: 10485760 + server.maxPayloadBytes: 1759977 + logging.json: false + data.search.aggs.shardDelay.enabled: true + csp.warnLegacyBrowsers: false + ci-groups: 9 + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/templates/opensearch-dashboards/3.x/manifest-test.yml b/manifests/templates/opensearch-dashboards/3.x/manifest-test.yml new file mode 100644 index 0000000000..bc2310e9b1 --- /dev/null +++ b/manifests/templates/opensearch-dashboards/3.x/manifest-test.yml @@ -0,0 +1,26 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-dashboards-integtest-v1 +components: + - name: OpenSearch-Dashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + vis_builder.enabled: true + data_source.enabled: true + savedObjects.maxImportPayloadBytes: 10485760 + server.maxPayloadBytes: 1759977 + logging.json: false + data.search.aggs.shardDelay.enabled: true + csp.warnLegacyBrowsers: false + ci-groups: 9 + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/templates/opensearch-dashboards/default/manifest-test.yml b/manifests/templates/opensearch-dashboards/default/manifest-test.yml new file mode 100644 index 0000000000..bc2310e9b1 --- /dev/null +++ b/manifests/templates/opensearch-dashboards/default/manifest-test.yml @@ -0,0 +1,26 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-almalinux8-opensearch-dashboards-integtest-v1 +components: + - name: OpenSearch-Dashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + vis_builder.enabled: true + data_source.enabled: true + savedObjects.maxImportPayloadBytes: 10485760 + server.maxPayloadBytes: 1759977 + logging.json: false + data.search.aggs.shardDelay.enabled: true + csp.warnLegacyBrowsers: false + ci-groups: 9 + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/templates/opensearch/1.x/manifest-test.yml b/manifests/templates/opensearch/1.x/manifest-test.yml new file mode 100644 index 0000000000..17f2390a33 --- /dev/null +++ b/manifests/templates/opensearch/1.x/manifest-test.yml @@ -0,0 +1,20 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + args: -e JAVA_HOME=/opt/java/openjdk-11 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security diff --git a/manifests/templates/opensearch/2.x/manifest-test.yml b/manifests/templates/opensearch/2.x/manifest-test.yml new file mode 100644 index 0000000000..5c481e2d98 --- /dev/null +++ b/manifests/templates/opensearch/2.x/manifest-test.yml @@ -0,0 +1,20 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-21 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security diff --git a/manifests/templates/opensearch/2.x/manifest.yml b/manifests/templates/opensearch/2.x/manifest.yml index ce51bbae19..852db1a3df 100644 --- a/manifests/templates/opensearch/2.x/manifest.yml +++ b/manifests/templates/opensearch/2.x/manifest.yml @@ -5,7 +5,7 @@ build: version: 'replace' ci: image: - name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 args: -e JAVA_HOME=/opt/java/openjdk-21 components: - name: OpenSearch diff --git a/manifests/templates/opensearch/3.x/manifest-test.yml b/manifests/templates/opensearch/3.x/manifest-test.yml new file mode 100644 index 0000000000..5c481e2d98 --- /dev/null +++ b/manifests/templates/opensearch/3.x/manifest-test.yml @@ -0,0 +1,20 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-21 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security diff --git a/manifests/templates/opensearch/3.x/manifest.yml b/manifests/templates/opensearch/3.x/manifest.yml index ce51bbae19..852db1a3df 100644 --- a/manifests/templates/opensearch/3.x/manifest.yml +++ b/manifests/templates/opensearch/3.x/manifest.yml @@ -5,7 +5,7 @@ build: version: 'replace' ci: image: - name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 args: -e JAVA_HOME=/opt/java/openjdk-21 components: - name: OpenSearch diff --git a/manifests/templates/opensearch/default/manifest-test.yml b/manifests/templates/opensearch/default/manifest-test.yml new file mode 100644 index 0000000000..5c481e2d98 --- /dev/null +++ b/manifests/templates/opensearch/default/manifest-test.yml @@ -0,0 +1,20 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 + args: -e JAVA_HOME=/opt/java/openjdk-21 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security diff --git a/manifests/templates/opensearch/default/manifest.yml b/manifests/templates/opensearch/default/manifest.yml index ce51bbae19..852db1a3df 100644 --- a/manifests/templates/opensearch/default/manifest.yml +++ b/manifests/templates/opensearch/default/manifest.yml @@ -5,7 +5,7 @@ build: version: 'replace' ci: image: - name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3 + name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1 args: -e JAVA_HOME=/opt/java/openjdk-21 components: - name: OpenSearch diff --git a/src/manifests_workflow/input_manifests.py b/src/manifests_workflow/input_manifests.py index 187fc200ee..4fb7aa84fd 100644 --- a/src/manifests_workflow/input_manifests.py +++ b/src/manifests_workflow/input_manifests.py @@ -10,13 +10,14 @@ import os import re from abc import abstractmethod -from typing import Dict, List, Type, Union +from typing import Dict, List, Type, Tuple, Union import ruamel.yaml from packaging.version import Version from packaging.version import parse as version_parse from manifests.input_manifest import InputManifest +from manifests.test_manifest import TestManifest from manifests.manifests import Manifests from manifests_workflow.component_opensearch import ComponentOpenSearch from manifests_workflow.component_opensearch_dashboards_min import ComponentOpenSearchDashboardsMin @@ -97,7 +98,6 @@ def update( # check out and build #main, 1.x, etc. all_branches = sorted(min_klass.branches()) branches = [b for b in all_branches if not any(b == o or b.startswith((f"{o}-", f"{o}/")) for o in legacy_branches)] - # TODO: Remove logging.info(f"Checking {self.name} {sorted(branches)} branches") logging.info(f"Ignoring {self.name} {sorted(set(all_branches) - set(branches))} branches as they are legacy") @@ -122,7 +122,7 @@ def update( self.add_to_versionincrement_workflow(new_version_entry) known_versions.append(new_version_entry) - def create_manifest(self, version: str, branch: str, known_versions: List[str]) -> InputManifest: + def create_manifest(self, version: str, branch: str, known_versions: List[str]) -> Tuple[InputManifest, TestManifest]: # If : No known_versions manifests exist or new version smaller than the min(known_versions), create new manifests from the templates # (1.0.0-3.0.0 based on template 1.x-3.x, 4.0.0+ from default.x, previous behavior) # Else: Create new manifests based on the latest version before the new version @@ -132,35 +132,46 @@ def create_manifest(self, version: str, branch: str, known_versions: List[str]) templates_base_path = os.path.join(self.manifests_path(), "templates") template_version_folder = version.split(".")[0] + ".x" template_full_path = os.path.join(templates_base_path, self.prefix, template_version_folder, "manifest.yml") + template_test_full_path = os.path.join(templates_base_path, self.prefix, template_version_folder, "manifest-test.yml") if not os.path.exists(template_full_path): template_full_path = os.path.join(templates_base_path, self.prefix, "default", "manifest.yml") + template_test_full_path = os.path.join(templates_base_path, self.prefix, "default", "manifest-test.yml") else: previous_versions = [v for v in known_versions if Version(v) < Version(version)] base_version = max(previous_versions, key=version_parse) logging.info(f"Base Version: {base_version} is the highest version before {version}") template_full_path = os.path.join(self.manifests_path(), base_version, f"{self.prefix}-{base_version}.yml") + template_test_full_path = os.path.join(self.manifests_path(), base_version, f"{self.prefix}-{base_version}-test.yml") if not os.path.exists(template_full_path): template_full_path = os.path.join(self.legacy_manifests_path(), base_version, f"{self.prefix}-{base_version}.yml") + template_test_full_path = os.path.join(self.legacy_manifests_path(), base_version, f"{self.prefix}-{base_version}-test.yml") - logging.info(f"Using {template_full_path} as the base manifest") - + # Input Manifest + logging.info(f"Using {template_full_path} as the base input manifest") manifest = InputManifest.from_file(open(template_full_path)) - manifest.build.version = version - for component in manifest.components.select(): component.ref = branch # type: ignore - return manifest + # Test Manifest + logging.info(f"Using {template_test_full_path} as the base test manifest") + manifest_test = TestManifest.from_file(open(template_test_full_path)) + + return (manifest, manifest_test) def write_manifest(self, version: str, branch: str, known_versions: List[str]) -> None: logging.info(f"Creating new version: {version}") - manifest = self.create_manifest(version, branch, known_versions) + manifests = self.create_manifest(version, branch, known_versions) manifest_dir = os.path.join(self.manifests_path(), version) os.makedirs(manifest_dir, exist_ok=True) - manifest_path = os.path.join(manifest_dir, f"{self.prefix}-{version}.yml") - manifest.to_file(manifest_path) - logging.info(f"Wrote {manifest_path} as the new manifest") + for manifest in manifests: + if manifest.__class__.__name__ == 'TestManifest': + manifest_path = os.path.join(manifest_dir, f"{self.prefix}-{version}-test.yml") + logging.info(f"Wrote {manifest_path} as the new test manifest") + else: + manifest_path = os.path.join(manifest_dir, f"{self.prefix}-{version}.yml") + logging.info(f"Wrote {manifest_path} as the new input manifest") + manifest.to_file(manifest_path) def add_to_cron(self, version: str) -> None: logging.info(f"Adding new version to cron: {version}")