Skip to content

Commit

Permalink
Fix the report workflow to work with ci-group (opensearch-project#4960)
Browse files Browse the repository at this point in the history 
Signed-off-by: Zelin Hao <[email protected]>
  • Loading branch information
@zelinh
zelinh authored Aug 21, 2024
1 parent 5965098 commit d52e7a2
Show file tree
Hide file tree
Showing 4 changed files with 85 additions and 56 deletions.
1 change: 1 addition & 0 deletions src/manifests/test_manifest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ class TestManifest(ComponentManifest['TestManifest', 'TestComponents']):
- with-less-security
additional-cluster-configs:
- key : value
ci-group: 6
bwc-test:
test-configs:
- with-security
Expand Down
40 changes: 23 additions & 17 deletions src/report_workflow/test_report_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ class TestReportRunner:
test_manifest: TestManifest
tests_dir: str
test_report_manifest: TestReportManifest
test_run_data: dict
test_report_data: dict
bundle_manifest: BundleManifest

def __init__(self, args: ReportArgs, test_manifest: TestManifest) -> None:
self.args = args
self.base_path = args.base_path
self.test_manifest = test_manifest
self.release_candidate = self.args.release_candidate
self.test_run_data = self.test_report_manifest_data_template("manifest")
self.test_report_data = self.test_report_manifest_data_template("manifest")
self.product_name = test_manifest.__to_dict__().get("name")
self.name = self.product_name.replace(" ", "-").lower()
self.components = self.args.components
Expand All @@ -50,19 +50,24 @@ def __init__(self, args: ReportArgs, test_manifest: TestManifest) -> None:
self.test_components = self.test_manifest.components

def update_data(self) -> dict:
self.test_run_data["name"] = self.product_name
self.test_report_data["name"] = self.product_name
self.bundle_manifest = BundleManifest.from_urlpath(self.dist_manifest)
self.test_run_data["version"] = self.bundle_manifest.build.version
self.test_run_data["platform"] = self.bundle_manifest.build.platform
self.test_run_data["architecture"] = self.bundle_manifest.build.architecture
self.test_run_data["distribution"] = self.bundle_manifest.build.distribution
self.test_run_data["id"] = self.bundle_manifest.build.id
self.test_run_data["rc"] = self.release_candidate
self.test_run_data["test-run"] = self.update_test_run_data()
self.test_report_data["version"] = self.bundle_manifest.build.version
self.test_report_data["platform"] = self.bundle_manifest.build.platform
self.test_report_data["architecture"] = self.bundle_manifest.build.architecture
self.test_report_data["distribution"] = self.bundle_manifest.build.distribution
self.test_report_data["id"] = self.bundle_manifest.build.id
self.test_report_data["rc"] = self.release_candidate
self.test_report_data["test-run"] = self.update_test_run_data()
for component in self.test_components.select(focus=self.args.components):
if self.test_manifest.components[component.name].__to_dict__().get(self.test_type) is not None:
self.test_run_data["components"].append(self.component_entry(component.name))
return self.test_run_data
component_ci_group = getattr(component, self.test_type.replace("-", "_")).get("ci-groups", None)
if component_ci_group:
for i in range(component_ci_group):
self.test_report_data["components"].append(self.component_entry(component.name, i + 1))
else:
self.test_report_data["components"].append(self.component_entry(component.name))
return self.test_report_data

def update_test_run_data(self) -> dict:
test_run_data = {
Expand All @@ -80,9 +85,10 @@ def generate_report(self, data: dict, output_dir: str) -> Any:
logging.info(f"Generating test-report.yml in {output_dir}")
return test_report_manifest.to_file(test_report_manifest_file)

def component_entry(self, component_name: str) -> Any:
def component_entry(self, component_name: str, ci_group: int = None) -> Any:
component = self.test_report_manifest_data_template("component")
component["name"] = component_name
test_report_component_name = component_name if not ci_group else f"{component_name}-ci-group-{ci_group}"
component["name"] = test_report_component_name
component["command"] = generate_test_command(self.test_type, self.test_manifest_path, self.artifact_paths, component_name)

test_component = self.test_manifest.components[component_name]
Expand All @@ -94,7 +100,7 @@ def component_entry(self, component_name: str) -> Any:
"name": config,
}

component_yml_ref = generate_component_yml_ref(self.base_path, str(self.test_run_id), self.test_type, component_name, config)
component_yml_ref = generate_component_yml_ref(self.base_path, str(self.test_run_id), self.test_type, test_report_component_name, config)
logging.info(f"Loading {component_yml_ref}")
try:
if validators.url(component_yml_ref):
Expand All @@ -111,8 +117,8 @@ def component_entry(self, component_name: str) -> Any:
component_yml_ref = "URL not available"
config_dict["yml"] = component_yml_ref
config_dict["status"] = test_result
config_dict["cluster_stdout"] = get_os_cluster_logs(self.base_path, str(self.test_run_id), self.test_type, component_name, config, self.name)[0]
config_dict["cluster_stderr"] = get_os_cluster_logs(self.base_path, str(self.test_run_id), self.test_type, component_name, config, self.name)[1]
config_dict["cluster_stdout"] = get_os_cluster_logs(self.base_path, str(self.test_run_id), self.test_type, test_report_component_name, config, self.name)[0]
config_dict["cluster_stderr"] = get_os_cluster_logs(self.base_path, str(self.test_run_id), self.test_type, test_report_component_name, config, self.name)[1]
component["configs"].append(config_dict)
return component

Expand Down
1 change: 1 addition & 0 deletions tests/tests_report_workflow/data/test-manifest-opensearch-dashboards.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ components:
additional-cluster-configs:
vis_builder.enabled: true
data_source.enabled: true
ci-groups: 3
- name: alertingDashboards
integ-test:
test-configs:
Expand Down
99 changes: 60 additions & 39 deletions tests/tests_report_workflow/test_test_report_runner_dashboards.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,18 @@ def test_runner_dashboards_init(self, report_args_mock: MagicMock) -> None:
report_args_mock.test_type = "integ-test"
report_args_mock.release_candidate = "100"

test_run_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS)
test_run_runner_data = test_run_runner.update_data()
self.assertEqual(test_run_runner.name, "opensearch-dashboards")
self.assertEqual(test_run_runner.test_run_id, 123)
self.assertEqual(test_run_runner.test_type, "integ-test")
self.assertEqual(test_run_runner.test_manifest_path, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH)
self.assertEqual(test_run_runner_data["version"], "1.3.18")
self.assertEqual(test_run_runner_data["platform"], "linux")
self.assertEqual(test_run_runner_data["architecture"], "x64")
self.assertEqual(test_run_runner_data["distribution"], "tar")
self.assertEqual(test_run_runner_data["id"], "7791")
self.assertEqual(test_run_runner_data["rc"], "100")
test_report_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS)
test_report_runner_data = test_report_runner.update_data()
self.assertEqual(test_report_runner.name, "opensearch-dashboards")
self.assertEqual(test_report_runner.test_run_id, 123)
self.assertEqual(test_report_runner.test_type, "integ-test")
self.assertEqual(test_report_runner.test_manifest_path, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH)
self.assertEqual(test_report_runner_data["version"], "1.3.18")
self.assertEqual(test_report_runner_data["platform"], "linux")
self.assertEqual(test_report_runner_data["architecture"], "x64")
self.assertEqual(test_report_runner_data["distribution"], "tar")
self.assertEqual(test_report_runner_data["id"], "7791")
self.assertEqual(test_report_runner_data["rc"], "100")

@patch("report_workflow.report_args.ReportArgs")
def test_generate_file(self, report_args_mock: MagicMock) -> None:
Expand All @@ -51,14 +51,37 @@ def test_generate_file(self, report_args_mock: MagicMock) -> None:
report_args_mock.test_type = "integ-test"
report_args_mock.release_candidate = "100"

test_run_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS)
test_run_runner_data = test_run_runner.update_data()
test_report_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS)
test_report_runner_data = test_report_runner.update_data()

with TemporaryDirectory() as path:
output_path = os.path.join(path.name, "test-report.yml")
test_run_runner.generate_report(test_run_runner_data, path.name)
test_report_runner.generate_report(test_report_runner_data, path.name)
self.assertTrue(os.path.isfile(output_path))

@patch("report_workflow.report_args.ReportArgs")
def test_ci_group(self, report_args_mock: MagicMock) -> None:
report_args_mock.test_manifest_path = self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS_PATH
report_args_mock.artifact_paths = {"opensearch-dashboards": self.DATA_DIR}
report_args_mock.test_run_id = 123
report_args_mock.base_path = self.DATA_DIR
report_args_mock.test_type = "integ-test"
report_args_mock.release_candidate = "100"

test_report_runner = TestReportRunner(report_args_mock, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS)
test_report_runner_data = test_report_runner.update_data()

self.assertEqual(len(test_report_runner_data["components"]), 9)
for i in range(3):
self.assertEqual(test_report_runner_data["components"][i]["name"], f"OpenSearch-Dashboards-ci-group-{i + 1}")

for i in range(len(self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS.components.__to_dict__())):
if self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS.components.__to_dict__()[i]["name"] == "OpenSearch-Dashboards":
continue
else:
self.assertEqual(self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS.components.__to_dict__()[i]["name"],
test_report_runner_data["components"][i + 2]["name"])

@patch("yaml.safe_load")
@patch("urllib.request.urlopen")
@patch("validators.url")
Expand All @@ -75,30 +98,28 @@ def test_runner_component_entry_url(self, report_args_mock: MagicMock, validator
yaml_safe_load_mock.return_value = {"test_result": "PASS"}
urlopen_mock.return_value = MagicMock()

test_run_component_dict = TestReportRunner(report_args_mock,
self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS).component_entry(
"alertingDashboards")
test_report_component_dict = TestReportRunner(report_args_mock, self.TEST_MANIFEST_OPENSEARCH_DASHBOARDS).component_entry("alertingDashboards")
urlopen_mock.assert_has_calls([call(
'https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/alertingDashboards.yml')])
self.assertEqual(test_run_component_dict.get("configs")[0]["status"], "PASS")
self.assertEqual(test_run_component_dict.get("configs")[0]["name"], "with-security")
self.assertEqual(test_run_component_dict.get("configs")[0]["yml"],
self.assertEqual(test_report_component_dict.get("configs")[0]["status"], "PASS")
self.assertEqual(test_report_component_dict.get("configs")[0]["name"], "with-security")
self.assertEqual(test_report_component_dict.get("configs")[0]["yml"],
"https://ci.opensearch.org/ci/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/alertingDashboards.yml")
self.assertEqual(test_run_component_dict.get("configs")[0]["cluster_stdout"][0], "https://ci.opensearch.org/ci"
"/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-0/stdout.txt")
self.assertEqual(test_run_component_dict.get("configs")[0]["cluster_stdout"][1], "https://ci.opensearch.org/ci"
"/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-1/stdout.txt")
self.assertEqual(test_run_component_dict.get("configs")[0]["cluster_stderr"][0], "https://ci.opensearch.org/ci"
"/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-0/stderr.txt")
self.assertEqual(test_run_component_dict.get("configs")[0]["cluster_stderr"][1], "https://ci.opensearch.org/ci"
"/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-1/stderr.txt")

self.assertEqual(test_run_component_dict.get("configs")[1]["name"], "without-security")
self.assertEqual(test_run_component_dict.get("configs")[1]["cluster_stdout"][0], "https://ci.opensearch.org/ci"
"/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-2/stdout.txt")
self.assertEqual(test_run_component_dict.get("configs")[1]["cluster_stdout"][1], "https://ci.opensearch.org/ci"
"/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-3/stdout.txt")
self.assertEqual(test_run_component_dict.get("configs")[1]["cluster_stderr"][0], "https://ci.opensearch.org/ci"
"/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-2/stderr.txt")
self.assertEqual(test_run_component_dict.get("configs")[1]["cluster_stderr"][1], "https://ci.opensearch.org/ci"
"/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-3/stderr.txt")
self.assertEqual(test_report_component_dict.get("configs")[0]["cluster_stdout"][0],
"https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-0/stdout.txt")
self.assertEqual(test_report_component_dict.get("configs")[0]["cluster_stdout"][1],
"https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-1/stdout.txt")
self.assertEqual(test_report_component_dict.get("configs")[0]["cluster_stderr"][0],
"https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-0/stderr.txt")
self.assertEqual(test_report_component_dict.get("configs")[0]["cluster_stderr"][1],
"https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/with-security/local-cluster-logs/id-1/stderr.txt")

self.assertEqual(test_report_component_dict.get("configs")[1]["name"], "without-security")
self.assertEqual(test_report_component_dict.get("configs")[1]["cluster_stdout"][0],
"https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-2/stdout.txt")
self.assertEqual(test_report_component_dict.get("configs")[1]["cluster_stdout"][1],
"https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-3/stdout.txt")
self.assertEqual(test_report_component_dict.get("configs")[1]["cluster_stderr"][0],
"https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-2/stderr.txt")
self.assertEqual(test_report_component_dict.get("configs")[1]["cluster_stderr"][1],
"https://ci.opensearch.org/ci""/dbc/mock/test-results/123/integ-test/alertingDashboards/without-security/local-cluster-logs/id-3/stderr.txt")

0 comments on commit d52e7a2

Please sign in to comment.