diff --git a/report.sh b/report.sh new file mode 100755 index 0000000000..90f28e66be --- /dev/null +++ b/report.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +set -e + +DIR="$(dirname "$0")" +"$DIR/run.sh" "$DIR/src/run_test_report.py" $@ diff --git a/src/manifests/test_run_manifest.py b/src/manifests/test_run_manifest.py new file mode 100644 index 0000000000..863629a2a0 --- /dev/null +++ b/src/manifests/test_run_manifest.py @@ -0,0 +1,155 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +from typing import Optional + +from manifests.component_manifest import Component, ComponentManifest, Components + + +class TestRunManifest(ComponentManifest['TestRunManifest', 'TestComponents']): + """ + TestRunManifest contains the aggregated test results for the components. + + The format for schema version 1.0 is: + schema-version: '1.0' + name: name of the product e.g. OpenSearch + test-run: + Command: command to trigger the integ test + TestType: type of test this manifest reports. e.g. integ-test + TestManifest: location of the test manifest used + DistributionManifest: URL or local path of the bundle manifest. + TestID: test id + components: + - name: sql + command: command to trigger the integ test for only sql component + configs: + - name: with-security + status: the status of the test run with this config. e.g. pass/fail + yml: URL or local path to the component yml file + """ + + SCHEMA = { + "schema-version": {"required": True, "type": "string", "allowed": ["1.0"]}, + "name": {"required": True, "type": "string", "allowed": ["OpenSearch", "OpenSearch Dashboards"]}, + "test-run": { + "required": False, + "type": "dict", + "schema": { + "Command": {"required": False, "type": "string"}, + "TestType": {"required": False, "type": "string"}, + "TestManifest": {"required": False, "type": "string"}, + "DistributionManifest": {"required": False, "type": "string"}, + "TestID": {"required": False, "type": "string"} + }, + }, + "components": { + "type": "list", + "schema": { + "type": "dict", + "schema": { + "name": {"required": True, "type": "string"}, + "command": {"type": "string"}, + "configs": { + "type": "list", + "schema": { + "type": "dict", + "schema": { + "name": {"type": "string"}, + "status": {"type": "string"}, + "yml": {"type": "string"}, + } + }, + }, + }, + }, + }, + } + + def __init__(self, data: dict) -> None: + super().__init__(data) + self.name = str(data["name"]) + self.test_run = self.TestRun(data.get("test-run", None)) + self.components = TestComponents(data.get("components", [])) # type: ignore[assignment] + + def __to_dict__(self) -> dict: + return { + "schema-version": "1.0", + "name": self.name, + "test-run": None if self.test_run is None else self.test_run.__to_dict__(), + "components": self.components.__to_dict__() + } + + class TestRun: + def __init__(self, data: dict) -> None: + if data is None: + self.test_run = None + else: + self.command = data["Command"] + self.test_type = data["TestType"] + self.test_manifest = data["TestManifest"] + self.distribution_manifest = data["DistributionManifest"] + self.test_id = data["TestID"] + + def __to_dict__(self) -> Optional[dict]: + if (self.command and self.test_type and self.test_manifest and self.distribution_manifest and self.test_id) is None: + return None + else: + return { + "Command": self.command, + "TestType": self.test_type, + "TestManifest": self.test_manifest, + "DistributionManifest": self.distribution_manifest, + "TestID": self.test_id + } + + +class TestComponents(Components['TestComponent']): + @classmethod + def __create__(self, data: dict) -> 'TestComponent': + return TestComponent(data) + + +class TestComponent(Component): + def __init__(self, data: dict) -> None: + super().__init__(data) + self.command = data["command"] + self.configs = self.TestComponentConfigs(data.get("configs", None)) + + def __to_dict__(self) -> dict: + return { + "name": self.name, + "command": self.command, + "configs": self.configs.__to_list__() + } + + class TestComponentConfigs: + def __init__(self, data: list) -> None: + self.configs = [] + for config in data: + self.configs.append(self.TestComponentConfig(config).__to_dict__()) + + def __to_list__(self) -> list: + return self.configs + + class TestComponentConfig: + def __init__(self, data: dict) -> None: + self.name = data["name"] + self.status = data["status"] + self.yml = data["yml"] + + def __to_dict__(self) -> dict: + return { + "name": self.name, + "status": self.status, + "yml": self.yml + } + + +TestRunManifest.VERSIONS = {"1.0": TestRunManifest} + +TestComponent.__test__ = False # type: ignore[attr-defined] +TestRunManifest.__test__ = False # type: ignore[attr-defined] diff --git a/src/report_workflow/__init__.py b/src/report_workflow/__init__.py new file mode 100644 index 0000000000..153f30145e --- /dev/null +++ b/src/report_workflow/__init__.py @@ -0,0 +1,8 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# This page intentionally left blank. diff --git a/src/report_workflow/report_args.py b/src/report_workflow/report_args.py new file mode 100644 index 0000000000..9b85d5663d --- /dev/null +++ b/src/report_workflow/report_args.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + +import argparse +import logging + +from test_workflow.test_kwargs import TestKwargs + + +class ReportArgs: + test_run_id: str + keep: bool + test_manifest_path: str + artifact_paths: dict + test_type: str + logging_level: int + + def __init__(self) -> None: + parser = argparse.ArgumentParser(description="Generate test report for given test data") + parser.add_argument("test_manifest_path", type=str, help="Specify a test manifest path.") + parser.add_argument("-p", "--artifact-paths", nargs='*', action=TestKwargs, default={}, + help="Specify aritfacts paths for OpenSearch and OpenSearch Dashboards.") + # e.g. --base-path https://ci.opensearch.org/ci/dbc/integ-test/2.7.0/7771/linux/x64/tar/test-results/1234/integ-test use more to save arguments number + parser.add_argument("--base-path", type=str, default="", + help="Specify base paths for the integration test logs.") + parser.add_argument("--test-type", type=str, default="integ-test", help="Specify test type of this.") + parser.add_argument("--output-path", type=str, help="Specify the path location for the test-run manifest.") + parser.add_argument("--test-run-id", type=int, help="The unique execution id for the test") + parser.add_argument("--component", type=str, dest="components", nargs='*', help="Test a specific component or components instead of the entire distribution.") + parser.add_argument( + "-v", "--verbose", help="Show more verbose output.", action="store_const", default=logging.INFO, const=logging.DEBUG, dest="logging_level" + ) + + args = parser.parse_args() + self.test_run_id = args.test_run_id + self.logging_level = args.logging_level + self.test_manifest_path = args.test_manifest_path + self.artifact_paths = args.artifact_paths + self.base_path = args.base_path + self.test_type = args.test_type + self.components = args.components + self.output_path = args.output_path diff --git a/src/report_workflow/test_run_runner.py b/src/report_workflow/test_run_runner.py new file mode 100644 index 0000000000..2af4f7d189 --- /dev/null +++ b/src/report_workflow/test_run_runner.py @@ -0,0 +1,137 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import logging +import os +import urllib.request +from typing import Any +from urllib.error import HTTPError + +import validators +import yaml + +from manifests.test_manifest import TestManifest +from manifests.test_run_manifest import TestRunManifest +from report_workflow.report_args import ReportArgs + + +class TestRunRunner: + args: ReportArgs + test_manifest: TestManifest + tests_dir: str + test_run_manifest: TestRunManifest + test_run_data: dict + + def __init__(self, args: ReportArgs, test_manifest: TestManifest) -> None: + self.args = args + self.base_path = args.base_path + self.test_manifest = test_manifest + self.test_run_data = self.test_run_manifest_data_template("manifest") + self.product_name = test_manifest.__to_dict__().get("name") + self.name = self.product_name.replace(" ", "-").lower() + self.components = self.args.components + self.test_run_id = args.test_run_id + self.test_type = self.args.test_type + self.test_manifest_path = self.args.test_manifest_path + self.artifact_paths = "" + for k, v in self.args.artifact_paths.items(): + self.artifact_paths = " ".join([self.artifact_paths, k + "=" + v]).strip(" ") + + self.dist_manifest = "/".join([self.args.artifact_paths[self.name], "dist", self.name, "manifest.yml"]) if self.args.artifact_paths[self.name].startswith("https://") \ + else os.path.join(self.args.artifact_paths[self.name], "dist", self.name, "manifest.yml") + self.test_components = self.test_manifest.components + + def update_data(self) -> dict: + self.test_run_data["name"] = self.product_name + self.test_run_data["test-run"] = self.update_test_run_data() + for component in self.test_components.select(focus=self.args.components): + self.test_run_data["components"].append(self.component_entry(component.name)) + return self.test_run_data + + def update_test_run_data(self) -> dict: + test_run_data = { + "Command": generate_test_command(self.test_type, self.test_manifest_path, self.artifact_paths), + "TestType": self.test_type, + "TestManifest": self.test_manifest_path, + "DistributionManifest": self.dist_manifest, + "TestID": str(self.test_run_id) + } + return test_run_data + + def generate_report(self, data: dict, output_dir: str) -> Any: + test_run_manifest = TestRunManifest(data) + test_run_manifetest_run_manifest_file = os.path.join(output_dir, "test-run.yml") + logging.info(f"Generating test-run.yml in {output_dir}") + return test_run_manifest.to_file(test_run_manifetest_run_manifest_file) + + def component_entry(self, component_name: str) -> Any: + component = self.test_run_manifest_data_template("component") + component["name"] = component_name + component["command"] = generate_test_command(self.test_type, self.test_manifest_path, self.artifact_paths, component_name) + + test_component = self.test_manifest.components[component_name] + + config_names = [config for config in test_component.__to_dict__().get(self.test_type)["test-configs"]] + logging.info(f"Configs for {component_name} on {self.test_type} are {config_names}") + for config in config_names: + config_dict = { + "name": config, + } + + component_yml_ref = generate_component_yml_ref(self.base_path, str(self.test_run_id), self.test_type, component_name, config) + logging.info(f"Loading {component_yml_ref}") + try: + if validators.url(component_yml_ref): + with urllib.request.urlopen(component_yml_ref) as f: + component_yml = yaml.safe_load(f.read().decode("utf-8")) + test_result = component_yml["test_result"] + else: + with open(component_yml_ref, "r", encoding='utf8') as f: + component_yml = yaml.safe_load(f) + test_result = component_yml["test_result"] + except (FileNotFoundError, HTTPError): + logging.info(f"Component yml file for {component_name} for {config} is missing or the base path is incorrect.") + test_result = "Not Available" + component_yml_ref = "URL not available" + config_dict["yml"] = component_yml_ref + config_dict["status"] = test_result + component["configs"].append(config_dict) + return component + + def test_run_manifest_data_template(self, template_type: str) -> Any: + templates = { + "manifest": { + "schema-version": "1.0", + "name": "", + "test-run": {}, + "components": [] + }, + "component": { + "name": "", + "command": "", + "configs": [] + } + } + return templates[template_type] + + +def generate_component_yml_ref(base_path: str, test_number: str, test_type: str, component_name: str, config: str) -> str: + if base_path.startswith("https://"): + return "/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, f"{component_name}.yml"]) + else: + return os.path.join(base_path, "test-results", test_number, test_type, component_name, config, f"{component_name}.yml") + + +def generate_test_command(test_type: str, test_manifest_path: str, artifacts_path: str, component: str = "") -> str: + command = " ".join(["./test.sh", test_type, test_manifest_path, "--paths", artifacts_path]) + if component: + command = " ".join([command, "--component", component]) + logging.info(command) + return command + + +TestRunRunner.__test__ = False # type:ignore diff --git a/src/run_test_report.py b/src/run_test_report.py new file mode 100644 index 0000000000..f9bd14daa9 --- /dev/null +++ b/src/run_test_report.py @@ -0,0 +1,34 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +import os.path +import sys +from typing import Any + +from manifests.test_manifest import TestManifest +from report_workflow.report_args import ReportArgs +from report_workflow.test_run_runner import TestRunRunner +from system import console + + +def main() -> Any: + args = ReportArgs() + + console.configure(level=args.logging_level) + + test_manifest = TestManifest.from_path(args.test_manifest_path) + + test_run_runner = TestRunRunner(args, test_manifest) + + test_run_data = test_run_runner.update_data() + + test_run = test_run_runner.generate_report(test_run_data, args.output_path or os.getcwd()) + + return test_run + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_run_test_report.py b/tests/test_run_test_report.py new file mode 100644 index 0000000000..fe5a080612 --- /dev/null +++ b/tests/test_run_test_report.py @@ -0,0 +1,46 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + + +import os +import unittest +from typing import Any +from unittest.mock import Mock, patch + +import pytest +from pytest import CaptureFixture + +from run_test_report import main + + +class TestRunTestReport(unittest.TestCase): + + TEST_MANIFEST_PATH = os.path.join( + os.path.dirname(__file__), + "tests_report_workflow", + "data", + "test_manifest.yml" + ) + + @pytest.fixture(autouse=True) + def getCapfd(self, capfd: CaptureFixture) -> None: + self.capfd = capfd + + @patch("argparse._sys.argv", ["run_test_report.py", "--help"]) + def test_usage(self, *mocks: Any) -> None: + with self.assertRaises(SystemExit): + main() + + out, _ = self.capfd.readouterr() + self.assertTrue(out.startswith("usage:")) + + @patch("argparse._sys.argv", ["run_test_report.py", TEST_MANIFEST_PATH, "-p", "opensearch=foo"]) + @patch('run_test_report.TestRunRunner') + def test_main(self, runner_mock: Mock, *mocks: Any) -> None: + + main() + runner_mock.assert_called() diff --git a/tests/tests_manifests/data/test-run.yml b/tests/tests_manifests/data/test-run.yml new file mode 100644 index 0000000000..11ad4e8ab2 --- /dev/null +++ b/tests/tests_manifests/data/test-run.yml @@ -0,0 +1,173 @@ +--- +schema-version: '1.0' +name: OpenSearch +test-run: + Command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + TestType: integ-test + TestManifest: manifests/2.8.0/opensearch-2.8.0-test.yml + DistributionManifest: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar/dist/opensearch/manifest.yml + TestID: '5109' +components: + - name: alerting + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component alerting + configs: + - name: with-security + status: FAIL + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/alerting/with-security/alerting.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/alerting/without-security/alerting.yml + - name: anomaly-detection + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component anomaly-detection + configs: + - name: with-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/anomaly-detection/with-security/anomaly-detection.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/anomaly-detection/without-security/anomaly-detection.yml + - name: asynchronous-search + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component asynchronous-search + configs: + - name: with-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/asynchronous-search/with-security/asynchronous-search.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/asynchronous-search/without-security/asynchronous-search.yml + - name: cross-cluster-replication + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component cross-cluster-replication + configs: + - name: with-security + status: Not Available + yml: URL not available + - name: without-security + status: Not Available + yml: URL not available + - name: geospatial + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component geospatial + configs: + - name: with-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/geospatial.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/without-security/geospatial.yml + - name: index-management + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component index-management + configs: + - name: with-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/index-management/with-security/index-management.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/index-management/without-security/index-management.yml + - name: k-NN + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component k-NN + configs: + - name: with-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/k-NN/with-security/k-NN.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/k-NN/without-security/k-NN.yml + - name: ml-commons + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component ml-commons + configs: + - name: with-security + status: FAIL + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/ml-commons/with-security/ml-commons.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/ml-commons/without-security/ml-commons.yml + - name: neural-search + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component neural-search + configs: + - name: with-security + status: FAIL + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/neural-search/with-security/neural-search.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/neural-search/without-security/neural-search.yml + - name: notifications + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component notifications + configs: + - name: with-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/notifications/with-security/notifications.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/notifications/without-security/notifications.yml + - name: opensearch-observability + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component opensearch-observability + configs: + - name: with-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/opensearch-observability/with-security/opensearch-observability.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/opensearch-observability/without-security/opensearch-observability.yml + - name: opensearch-reports + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component opensearch-reports + configs: + - name: with-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/opensearch-reports/with-security/opensearch-reports.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/opensearch-reports/without-security/opensearch-reports.yml + - name: security + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component security + configs: + - name: with-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/security/with-security/security.yml + - name: security-analytics + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component security-analytics + configs: + - name: with-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/security-analytics/with-security/security-analytics.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/security-analytics/without-security/security-analytics.yml + - name: sql + command: ./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml --paths + opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar + --component sql + configs: + - name: with-security + status: FAIL + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/sql/with-security/sql.yml + - name: without-security + status: PASS + yml: https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/sql/without-security/sql.yml diff --git a/tests/tests_manifests/test_test_run_manifest.py b/tests/tests_manifests/test_test_run_manifest.py new file mode 100644 index 0000000000..203bde3c53 --- /dev/null +++ b/tests/tests_manifests/test_test_run_manifest.py @@ -0,0 +1,51 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import os +import unittest + +import yaml + +from manifests.test_run_manifest import TestRunManifest + + +class TestTestRunManifest(unittest.TestCase): + + def setUp(self) -> None: + self.maxDiff = None + self.data_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "data")) + self.manifest_filename = os.path.join(self.data_path, "test-run.yml") + self.manifest = TestRunManifest.from_path(self.manifest_filename) + + def test_test_run(self) -> None: + self.assertEqual(self.manifest.name, "OpenSearch") + test_run = self.manifest.test_run + self.assertEqual(test_run.command, "./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml " + "--paths opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar") + self.assertEqual(test_run.test_type, "integ-test") + self.assertEqual(test_run.test_manifest, "manifests/2.8.0/opensearch-2.8.0-test.yml") + self.assertEqual(test_run.distribution_manifest, "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar/dist/opensearch/manifest.yml") + self.assertEqual(test_run.test_id, "5109") + + def test_component(self) -> None: + component = self.manifest.components["alerting"] + self.assertEqual(component.name, "alerting") + self.assertEqual(component.command, "./test.sh integ-test manifests/2.8.0/opensearch-2.8.0-test.yml " + "--paths opensearch=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.8.0/7935/linux/x64/tar --component alerting") + self.assertEqual(component.configs.configs[0]["name"], "with-security") + self.assertEqual(component.configs.configs[0]["status"], "FAIL") + self.assertEqual(component.configs.configs[0]["yml"], + "https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/alerting/with-security/alerting.yml") + self.assertEqual(component.configs.configs[1]["name"], "without-security") + self.assertEqual(component.configs.configs[1]["status"], "PASS") + self.assertEqual(component.configs.configs[1]["yml"], + "https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/alerting/without-security/alerting.yml") + + def test_to_dict(self) -> None: + data = self.manifest.to_dict() + with open(self.manifest_filename) as f: + self.assertEqual(yaml.safe_load(f), data) diff --git a/tests/tests_report_workflow/__init__.py b/tests/tests_report_workflow/__init__.py new file mode 100644 index 0000000000..9cf03e4e4e --- /dev/null +++ b/tests/tests_report_workflow/__init__.py @@ -0,0 +1,11 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../src")) diff --git a/tests/tests_report_workflow/data/geospatial.yml b/tests/tests_report_workflow/data/geospatial.yml new file mode 100644 index 0000000000..ac2536e95c --- /dev/null +++ b/tests/tests_report_workflow/data/geospatial.yml @@ -0,0 +1,45 @@ +# yamllint disable-file +component_name: geospatial +run_id: 5109 +test_config: with-security +test_result: PASS +test_result_files: + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/stdout.txt + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/stderr.txt + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/stdout.txt + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/stderr.txt + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/opensearch-service-logs/gc.log.00 + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/opensearch-service-logs/gc.log + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/opensearch-service-logs/opensearchcluster1_deprecation.log + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/opensearch-service-logs/opensearchcluster1_task_detailslog.json + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/opensearch-service-logs/opensearchcluster1.log + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/opensearch-service-logs/opensearchcluster1_index_search_slowlog.json + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/opensearch-service-logs/opensearchcluster1_index_indexing_slowlog.json + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/opensearch-service-logs/opensearchcluster1_server.json + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/opensearch-service-logs/opensearchcluster1_deprecation.json + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/opensearch-service-logs/opensearchcluster1_index_indexing_slowlog.log + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/opensearch-service-logs/opensearchcluster1_index_search_slowlog.log + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/local-cluster-logs/opensearch-service-logs/opensearchcluster1_task_detailslog.log + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/index.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/js/report.js + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/css/style.css + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/css/base-style.css + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.index.mapper.xypoint.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.plugin.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.index.query.xyshape.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.index.mapper.xyshape.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.index.query.xypoint.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.processor.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.rest.action.upload.geojson.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.search.aggregations.bucket.geogrid.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/packages/org.opensearch.geospatial.stats.upload.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.index.mapper.xypoint.XYPointFieldMapperIT.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.plugin.GeospatialPluginIT.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.index.query.xyshape.XYShapeQueryIT.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONActionIT.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.index.mapper.xyshape.XYShapeFieldMapperIT.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.processor.FeatureProcessorIT.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.stats.upload.RestUploadStatsActionIT.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.index.query.xypoint.XYPointQueryIT.html + - https://ci.opensearch.org/ci/dbc/integ-test/2.8.0/7935/linux/x64/tar/test-results/5109/integ-test/geospatial/with-security/opensearch-integ-test/classes/org.opensearch.geospatial.search.aggregations.bucket.geogrid.GeoHexAggregationIT.html +test_type: integ-test diff --git a/tests/tests_report_workflow/data/test-manifest-opensearch-dashboards.yml b/tests/tests_report_workflow/data/test-manifest-opensearch-dashboards.yml new file mode 100644 index 0000000000..1422c00e30 --- /dev/null +++ b/tests/tests_report_workflow/data/test-manifest-opensearch-dashboards.yml @@ -0,0 +1,45 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v3 +components: + - name: OpenSearch-Dashboards + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + vis_builder.enabled: true + data_source.enabled: true + - name: alertingDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: anomalyDetectionDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: ganttChartDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: indexManagementDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: observabilityDashboards + integ-test: + test-configs: + - with-security + - without-security + - name: queryWorkbenchDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/tests/tests_report_workflow/data/test_manifest.yml b/tests/tests_report_workflow/data/test_manifest.yml new file mode 100644 index 0000000000..949c7a9e74 --- /dev/null +++ b/tests/tests_report_workflow/data/test_manifest.yml @@ -0,0 +1,67 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1] + bwc-test: + test-configs: + - with-security + + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + bwc-test: + test-configs: + - with-security + + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + + - name: cross-cluster-replication + integ-test: + topology: + - cluster_name: leader + data_nodes: 1 + cluster_manager_nodes: 0 + - cluster_name: follower + data_nodes: 1 + cluster_manager_nodes: 0 + test-configs: + - with-security + - without-security + + - name: geospatial + integ-test: + test-configs: + - with-security + - without-security + + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + bwc-test: + test-configs: + - with-security diff --git a/tests/tests_report_workflow/test_report_args.py b/tests/tests_report_workflow/test_report_args.py new file mode 100644 index 0000000000..8204cf1145 --- /dev/null +++ b/tests/tests_report_workflow/test_report_args.py @@ -0,0 +1,126 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + + +import logging +import os +import unittest +from unittest.mock import patch + +from report_workflow.report_args import ReportArgs + + +class TestReportArgs(unittest.TestCase): + + ARGS_PY = os.path.realpath( + os.path.join( + os.path.dirname(__file__), "..", "..", "src", "run_test_report.py" + ) + ) + + PATH = os.path.join( + os.path.dirname(__file__), "data" + ) + + TEST_MANIFEST_PATH = os.path.join( + os.path.dirname(__file__), "data", "test_manifest.yml" + ) + + TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH = os.path.join( + os.path.dirname(__file__), "data", "test-manifest-opensearch-dashboards.yml" + ) + + @patch("argparse._sys.argv", [ARGS_PY, TEST_MANIFEST_PATH]) + def test_opensearch_default_with_opensearch_test_manifest(self) -> None: + report_args = ReportArgs() + self.assertFalse(hasattr(report_args, "opensearch")) + self.assertFalse(hasattr(report_args, "opensearch-dashboards")) + + self.assertIsNone(report_args.test_run_id) + self.assertIsNone(report_args.components) + self.assertIsNotNone(report_args.test_type) + self.assertEqual(report_args.logging_level, logging.INFO) + self.assertEqual(report_args.test_manifest_path, self.TEST_MANIFEST_PATH) + + @patch("argparse._sys.argv", [ARGS_PY, TEST_MANIFEST_PATH, "--component", "component1", "component2"]) + def test_components(self) -> None: + report_args = ReportArgs() + self.assertEqual(report_args.components, ["component1", "component2"]) + + @patch("argparse._sys.argv", [ARGS_PY, TEST_MANIFEST_PATH, "--artifact-paths", "opensearch=" + TEST_MANIFEST_PATH]) + def test_opensearch_file_with_opensearch_test_manifest(self) -> None: + report_args = ReportArgs() + self.assertEqual(report_args.artifact_paths.get("opensearch"), os.path.realpath(self.TEST_MANIFEST_PATH)) + self.assertFalse(hasattr(report_args.artifact_paths, "opensearch-dashboards")) + + self.assertIsNone(report_args.test_run_id) + self.assertIsNone(report_args.components) + self.assertEqual(report_args.test_type, "integ-test") + self.assertEqual(report_args.logging_level, logging.INFO) + self.assertEqual(report_args.test_manifest_path, self.TEST_MANIFEST_PATH) + + @patch("argparse._sys.argv", [ARGS_PY, TEST_MANIFEST_PATH, "--artifact-paths", "opensearch=https://ci.opensearch.org/x/y", "--verbose"]) + def test_opensearch_url_with_opensearch_test_manifest(self) -> None: + report_args = ReportArgs() + self.assertEqual(report_args.artifact_paths.get("opensearch"), "https://ci.opensearch.org/x/y") + self.assertFalse(hasattr(report_args.artifact_paths, "opensearch-dashboards")) + self.assertEqual(report_args.test_manifest_path, self.TEST_MANIFEST_PATH) + + self.assertIsNone(report_args.test_run_id) + self.assertIsNone(report_args.components) + self.assertEqual(report_args.test_type, "integ-test") + self.assertEqual(report_args.logging_level, logging.DEBUG) + + @patch("argparse._sys.argv", [ARGS_PY, TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH, "--artifact-paths", "opensearch-dashboards=" + TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH]) + def test_opensearch_dashboards_default_with_opensearch_dashboards_test_manifest(self) -> None: + report_args = ReportArgs() + self.assertFalse(hasattr(report_args.artifact_paths, "opensearch")) + self.assertEqual(report_args.artifact_paths.get("opensearch-dashboards"), self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH) + + self.assertIsNone(report_args.test_run_id) + self.assertIsNone(report_args.components) + self.assertEqual(report_args.test_type, "integ-test") + self.assertEqual(report_args.logging_level, logging.INFO) + self.assertEqual(report_args.test_manifest_path, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH) + + @patch("argparse._sys.argv", [ARGS_PY, TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH, "--artifact-paths", + "opensearch-dashboards=https://ci.opensearch.org/x/y", "opensearch=https://ci.opensearch.org/x/y/z"]) + def test_opensearch_dashboards_url_with_opensearch_dashboards_test_manifest(self) -> None: + report_args = ReportArgs() + self.assertEqual(report_args.artifact_paths.get("opensearch-dashboards"), "https://ci.opensearch.org/x/y") + self.assertEqual(report_args.artifact_paths.get("opensearch"), "https://ci.opensearch.org/x/y/z") + + self.assertIsNone(report_args.test_run_id) + self.assertIsNone(report_args.components) + self.assertEqual(report_args.test_type, "integ-test") + self.assertEqual(report_args.logging_level, logging.INFO) + self.assertEqual(report_args.test_manifest_path, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH) + + @patch("argparse._sys.argv", [ARGS_PY, TEST_MANIFEST_PATH, "--artifact-paths", "opensearch=https://ci.opensearch.org/x/y/z", "--test-run-id", "6"]) + def test_run_id(self) -> None: + report_args = ReportArgs() + self.assertEqual(report_args.test_run_id, 6) + self.assertEqual(report_args.test_manifest_path, self.TEST_MANIFEST_PATH) + self.assertEqual(report_args.artifact_paths.get("opensearch"), "https://ci.opensearch.org/x/y/z") + + @patch( + "argparse._sys.argv", + [ARGS_PY, + TEST_MANIFEST_PATH, + "--artifact-paths", + "opensearch-dashboards=https://ci.opensearch.org/x/y", + "opensearch=https://ci.opensearch.org/x/y/z", + "--base-path", "https://ci.opensearch.org/ci/dbc/integ-test/", + "--test-run-id", "1234"]) + def test_base_path(self) -> None: + report_args = ReportArgs() + self.assertEqual(report_args.base_path, "https://ci.opensearch.org/ci/dbc/integ-test/") + self.assertEqual(report_args.test_manifest_path, self.TEST_MANIFEST_PATH) + self.assertEqual(report_args.artifact_paths.get("opensearch-dashboards"), "https://ci.opensearch.org/x/y") + self.assertEqual(report_args.artifact_paths.get("opensearch"), "https://ci.opensearch.org/x/y/z") + self.assertEqual(report_args.test_type, "integ-test") + self.assertEqual(report_args.test_run_id, 1234) diff --git a/tests/tests_report_workflow/test_test_run_runner.py b/tests/tests_report_workflow/test_test_run_runner.py new file mode 100644 index 0000000000..b8d1a50461 --- /dev/null +++ b/tests/tests_report_workflow/test_test_run_runner.py @@ -0,0 +1,152 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + + +import os +import unittest +from unittest.mock import MagicMock, call, mock_open, patch + +from manifests.test_manifest import TestManifest +from report_workflow.test_run_runner import TestRunRunner + + +class TestTestRunRunner(unittest.TestCase): + TEST_MANIFEST_PATH = os.path.join( + os.path.dirname(__file__), "data", "test_manifest.yml" + ) + + TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH = os.path.join( + os.path.dirname(__file__), "data", "test-manifest-opensearch-dashboards.yml" + ) + + TEST_MANIFEST = TestManifest.from_path(TEST_MANIFEST_PATH) + + TEST_MANIFEST_OPENSEARCH_DASHBOARDS = TestManifest.from_path(TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH) + + @patch("report_workflow.report_args.ReportArgs") + @patch("manifests.test_manifest.TestManifest") + def test_runner_init(self, report_args_mock: MagicMock, test_manifest_mock: MagicMock) -> None: + report_args_mock.test_manifest_path = self.TEST_MANIFEST_PATH + report_args_mock.artifact_paths = {"opensearch": "foo/bar"} + report_args_mock.test_run_id = 123 + report_args_mock.test_type = "integ-test" + + test_run_runner = TestRunRunner(report_args_mock, self.TEST_MANIFEST) + self.assertEqual(test_run_runner.name, "opensearch") + self.assertEqual(test_run_runner.test_run_id, 123) + self.assertEqual(test_run_runner.test_type, "integ-test") + self.assertEqual(test_run_runner.test_manifest_path, self.TEST_MANIFEST_PATH) + + @patch("report_workflow.report_args.ReportArgs") + @patch("manifests.test_manifest.TestManifest") + def test_runner_update_test_run_data_local(self, report_args_mock: MagicMock, test_manifest_mock: MagicMock) -> None: + report_args_mock.test_manifest_path = self.TEST_MANIFEST_PATH + report_args_mock.artifact_paths = {"opensearch": "foo/bar"} + report_args_mock.test_run_id = 123 + report_args_mock.test_type = "integ-test" + + test_run_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data() + self.assertEqual(test_run_dict.get("Command"), " ".join(["./test.sh", "integ-test", self.TEST_MANIFEST_PATH, "--paths", "opensearch=foo/bar"])) + self.assertEqual(test_run_dict.get("TestType"), "integ-test") + self.assertEqual(test_run_dict.get("TestManifest"), self.TEST_MANIFEST_PATH) + self.assertEqual(test_run_dict.get("DistributionManifest"), os.path.join("foo/bar", "dist", "opensearch", "manifest.yml")) + self.assertEqual(test_run_dict.get("TestID"), "123") + + @patch("report_workflow.report_args.ReportArgs") + @patch("manifests.test_manifest.TestManifest") + def test_runner_update_test_run_data_url(self, report_args_mock: MagicMock, test_manifest_mock: MagicMock) -> None: + report_args_mock.test_manifest_path = self.TEST_MANIFEST_PATH + report_args_mock.artifact_paths = {"opensearch": "https://foo/bar"} + report_args_mock.test_run_id = 123 + report_args_mock.test_type = "integ-test" + + test_run_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).update_test_run_data() + self.assertEqual(test_run_dict.get("Command"), " ".join(["./test.sh", "integ-test", self.TEST_MANIFEST_PATH, "--paths", "opensearch=https://foo/bar"])) + self.assertEqual(test_run_dict.get("TestType"), "integ-test") + self.assertEqual(test_run_dict.get("TestManifest"), self.TEST_MANIFEST_PATH) + self.assertEqual(test_run_dict.get("DistributionManifest"), "/".join(["https://foo/bar", "dist", "opensearch", "manifest.yml"])) + self.assertEqual(test_run_dict.get("TestID"), "123") + + @patch("yaml.safe_load") + @patch("urllib.request.urlopen") + @patch("validators.url") + @patch("report_workflow.report_args.ReportArgs") + def test_runner_component_entry_url(self, report_args_mock: MagicMock, validators_mock: MagicMock, urlopen_mock: MagicMock, yaml_safe_load_mock: MagicMock) -> None: + report_args_mock.test_manifest_path = self.TEST_MANIFEST_PATH + report_args_mock.artifact_paths = {"opensearch": "foo/bar"} + report_args_mock.test_run_id = 123 + report_args_mock.base_path = "https://ci.opensearch.org/ci/dbc/mock" + report_args_mock.test_type = "integ-test" + + validators_mock.return_value = True + yaml_safe_load_mock.return_value = {"test_result": "PASS"} + urlopen_mock.return_value = MagicMock() + + test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") + urlopen_mock.assert_has_calls([call('https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml')]) + self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "PASS") + self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security") + self.assertEqual(test_run_component_dict.get("configs")[0]["yml"], "https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml") + + @patch("yaml.safe_load") + @patch("builtins.open", new_callable=mock_open) + @patch("validators.url") + @patch("report_workflow.report_args.ReportArgs") + def test_runner_component_entry_local(self, report_args_mock: MagicMock, validators_mock: MagicMock, mock_open: MagicMock, yaml_safe_load_mock: MagicMock) -> None: + report_args_mock.test_manifest_path = self.TEST_MANIFEST_PATH + report_args_mock.artifact_paths = {"opensearch": "foo/bar"} + report_args_mock.test_run_id = 123 + report_args_mock.base_path = "https://ci.opensearch.org/ci/dbc/mock" + report_args_mock.test_type = "integ-test" + + validators_mock.return_value = False + yaml_safe_load_mock.return_value = {"test_result": "PASS"} + mock_open.return_value = MagicMock() + + test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") + mock_open.assert_has_calls([call('https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml', 'r', encoding='utf8')]) + self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "PASS") + self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security") + self.assertEqual(test_run_component_dict.get("configs")[0]["yml"], "https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml") + + @patch("yaml.safe_load") + @patch("validators.url") + @patch("report_workflow.report_args.ReportArgs") + def test_runner_component_entry_url_invalid(self, report_args_mock: MagicMock, validators_mock: MagicMock, yaml_safe_load_mock: MagicMock) -> None: + report_args_mock.test_manifest_path = self.TEST_MANIFEST_PATH + report_args_mock.artifact_paths = {"opensearch": "foo/bar"} + report_args_mock.test_run_id = 123 + report_args_mock.base_path = "https://ci.opensearch.org/ci/dbc/mock" + report_args_mock.test_type = "integ-test" + + validators_mock.return_value = True + + test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") + self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "Not Available") + self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security") + self.assertEqual(test_run_component_dict.get("configs")[0]["yml"], "URL not available") + + @patch("yaml.safe_load") + @patch("builtins.open", new_callable=mock_open) + @patch("validators.url") + @patch("report_workflow.report_args.ReportArgs") + def test_runner_component_entry_local_invalid(self, report_args_mock: MagicMock, validators_mock: MagicMock, mock_open: MagicMock, yaml_safe_load_mock: MagicMock) -> None: + report_args_mock.test_manifest_path = self.TEST_MANIFEST_PATH + report_args_mock.artifact_paths = {"opensearch": "foo/bar"} + report_args_mock.test_run_id = 123 + report_args_mock.base_path = "https://ci.opensearch.org/ci/dbc/mock" + report_args_mock.test_type = "integ-test" + + validators_mock.return_value = False + yaml_safe_load_mock.return_value = {"test_result": "PASS"} + mock_open.side_effect = FileNotFoundError + + test_run_component_dict = TestRunRunner(report_args_mock, self.TEST_MANIFEST).component_entry("geospatial") + mock_open.assert_has_calls([call('https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/geospatial/with-security/geospatial.yml', 'r', encoding='utf8')]) + self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "Not Available") + self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security") + self.assertEqual(test_run_component_dict.get("configs")[0]["yml"], "URL not available")