From cbf57ab9042943f3c22827b8b2e555dcb93000c9 Mon Sep 17 00:00:00 2001 From: Zelin Hao <87548827+zelinh@users.noreply.github.com> Date: Wed, 27 Jul 2022 12:42:37 -0700 Subject: [PATCH] Add JAVA_HOME arguments in manifest workflow from template manifests. (#2358) * Adding java home arg Signed-off-by: Zelin Hao * Attach tests Signed-off-by: Zelin Hao * Change to get ci data from templates instead of hardcoding Signed-off-by: Zelin Hao * Fix flake8 error Signed-off-by: Zelin Hao * Keep simplifying the workflow. Signed-off-by: Zelin Hao * More changes Signed-off-by: Zelin Hao --- .github/workflows/manifests.yml | 3 +- .../opensearch-dashboards/1.x/manifest.yml | 12 +++++ .../opensearch-dashboards/2.x/manifest.yml | 14 ++++++ .../opensearch-dashboards/3.x/manifest.yml | 12 +++++ .../default/manifest.yml | 14 ++++++ .../templates/opensearch/1.x/manifest.yml | 16 +++++++ .../templates/opensearch/2.x/manifest.yml | 16 +++++++ .../templates/opensearch/3.x/manifest.yml | 16 +++++++ .../templates/opensearch/default/manifest.yml | 16 +++++++ src/manifests_workflow/input_manifests.py | 47 +++++++++---------- .../test_input_manifests.py | 38 ++++++++++++--- .../test_input_manifests_opensearch.py | 9 ++-- ...t_input_manifests_opensearch_dashboards.py | 11 +++-- 13 files changed, 185 insertions(+), 39 deletions(-) create mode 100644 manifests/templates/opensearch-dashboards/1.x/manifest.yml create mode 100644 manifests/templates/opensearch-dashboards/2.x/manifest.yml create mode 100644 manifests/templates/opensearch-dashboards/3.x/manifest.yml create mode 100644 manifests/templates/opensearch-dashboards/default/manifest.yml create mode 100644 manifests/templates/opensearch/1.x/manifest.yml create mode 100644 manifests/templates/opensearch/2.x/manifest.yml create mode 100644 manifests/templates/opensearch/3.x/manifest.yml create mode 100644 manifests/templates/opensearch/default/manifest.yml diff --git a/.github/workflows/manifests.yml b/.github/workflows/manifests.yml index 2166c2fca3..4495916b3b 100644 --- a/.github/workflows/manifests.yml +++ b/.github/workflows/manifests.yml @@ -5,7 +5,8 @@ on: push: pull_request: paths: - - manifests/**/*.yml + - 'manifests/**/*.yml' + - '!manifests/templates/**/' schedule: - cron: 0 0 * * * diff --git a/manifests/templates/opensearch-dashboards/1.x/manifest.yml b/manifests/templates/opensearch-dashboards/1.x/manifest.yml new file mode 100644 index 0000000000..ef01447725 --- /dev/null +++ b/manifests/templates/opensearch-dashboards/1.x/manifest.yml @@ -0,0 +1,12 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch Dashboards + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: main diff --git a/manifests/templates/opensearch-dashboards/2.x/manifest.yml b/manifests/templates/opensearch-dashboards/2.x/manifest.yml new file mode 100644 index 0000000000..9074cf5d0b --- /dev/null +++ b/manifests/templates/opensearch-dashboards/2.x/manifest.yml @@ -0,0 +1,14 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch Dashboards + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: main + checks: + - npm:package:version diff --git a/manifests/templates/opensearch-dashboards/3.x/manifest.yml b/manifests/templates/opensearch-dashboards/3.x/manifest.yml new file mode 100644 index 0000000000..ef01447725 --- /dev/null +++ b/manifests/templates/opensearch-dashboards/3.x/manifest.yml @@ -0,0 +1,12 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch Dashboards + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: main diff --git a/manifests/templates/opensearch-dashboards/default/manifest.yml b/manifests/templates/opensearch-dashboards/default/manifest.yml new file mode 100644 index 0000000000..9074cf5d0b --- /dev/null +++ b/manifests/templates/opensearch-dashboards/default/manifest.yml @@ -0,0 +1,14 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch Dashboards + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: main + checks: + - npm:package:version diff --git a/manifests/templates/opensearch/1.x/manifest.yml b/manifests/templates/opensearch/1.x/manifest.yml new file mode 100644 index 0000000000..a1934229d4 --- /dev/null +++ b/manifests/templates/opensearch/1.x/manifest.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-11 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: main + checks: + - gradle:publish + - gradle:properties:version diff --git a/manifests/templates/opensearch/2.x/manifest.yml b/manifests/templates/opensearch/2.x/manifest.yml new file mode 100644 index 0000000000..d798342efe --- /dev/null +++ b/manifests/templates/opensearch/2.x/manifest.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: main + checks: + - gradle:publish + - gradle:properties:version diff --git a/manifests/templates/opensearch/3.x/manifest.yml b/manifests/templates/opensearch/3.x/manifest.yml new file mode 100644 index 0000000000..d798342efe --- /dev/null +++ b/manifests/templates/opensearch/3.x/manifest.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: main + checks: + - gradle:publish + - gradle:properties:version diff --git a/manifests/templates/opensearch/default/manifest.yml b/manifests/templates/opensearch/default/manifest.yml new file mode 100644 index 0000000000..d798342efe --- /dev/null +++ b/manifests/templates/opensearch/default/manifest.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: main + checks: + - gradle:publish + - gradle:properties:version diff --git a/src/manifests_workflow/input_manifests.py b/src/manifests_workflow/input_manifests.py index e40e3a9e67..f1dee86fd3 100644 --- a/src/manifests_workflow/input_manifests.py +++ b/src/manifests_workflow/input_manifests.py @@ -11,7 +11,7 @@ from abc import abstractmethod from typing import Dict, List, Type, Union -from manifests.input_manifest import InputManifest +from manifests.input_manifest import InputComponents, InputManifest from manifests.manifests import Manifests from manifests_workflow.component_opensearch import ComponentOpenSearch from manifests_workflow.component_opensearch_dashboards_min import ComponentOpenSearchDashboardsMin @@ -48,7 +48,12 @@ def files(self, name: str) -> List: return results @abstractmethod - def update(self, min_klass: Union[Type[ComponentOpenSearchMin], Type[ComponentOpenSearchDashboardsMin]], component_klass: Type[ComponentOpenSearch], keep: bool = False) -> None: + def update( + self, + min_klass: Union[Type[ComponentOpenSearchMin], Type[ComponentOpenSearchDashboardsMin]], + component_klass: Type[ComponentOpenSearch], + keep: bool = False, + ) -> None: known_versions = self.versions logging.info(f"Known versions: {known_versions}") main_versions: Dict = {} @@ -106,30 +111,23 @@ def update(self, min_klass: Union[Type[ComponentOpenSearchMin], Type[ComponentOp self.add_to_cron(release_version) def create_manifest(self, version: str, components: List = []) -> InputManifest: - image_map = { - "opensearch": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2", - "opensearch-dashboards": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2" - } - - data: Dict = { - "schema-version": "1.0", - "build": { - "name": self.name, - "version": version - }, - "ci": { - "image": { - "name": image_map[self.prefix] - } - }, - "components": [], - } + templates_base_path = os.path.join(self.manifests_path(), "templates") + template_version_folder = version.split(".")[0] + ".x" + template_full_path = os.path.join(templates_base_path, self.prefix, template_version_folder, "manifest.yml") + if not os.path.exists(template_full_path): + template_full_path = os.path.join(templates_base_path, self.prefix, "default", "manifest.yml") + + manifest = InputManifest.from_file(open(template_full_path)) + + manifest.build.version = version + manifests_components = [] for component in components: logging.info(f" Adding {component.name}") - data["components"].append(component.to_dict()) + manifests_components.append(component.to_dict()) - return InputManifest(data) + manifest.components = InputComponents(manifests_components) # type: ignore + return manifest def write_manifest(self, version: str, components: List = []) -> None: logging.info(f"Creating new version: {version}") @@ -151,10 +149,7 @@ def add_to_cron(self, version: str) -> None: if cron_entry in data: raise ValueError(f"{jenkinsfile} already contains an entry for {self.prefix} {version}") - data = data.replace( - "parameterizedCron '''\n", - f"parameterizedCron '''\n{' ' * 12}{cron_entry}" - ) + data = data.replace("parameterizedCron '''\n", f"parameterizedCron '''\n{' ' * 12}{cron_entry}") with open(jenkinsfile, "w") as f: f.write(data) diff --git a/tests/tests_manifests_workflow/test_input_manifests.py b/tests/tests_manifests_workflow/test_input_manifests.py index 46393ccd0a..4bfb97ae11 100644 --- a/tests/tests_manifests_workflow/test_input_manifests.py +++ b/tests/tests_manifests_workflow/test_input_manifests.py @@ -17,26 +17,52 @@ def test_manifests_path(self) -> None: self.assertEqual(path, InputManifests.manifests_path()) def test_create_manifest_opensearch(self) -> None: - input_manifests = InputManifests("opensearch") + input_manifests = InputManifests("OpenSearch") input_manifest = input_manifests.create_manifest("1.2.3", []) self.assertEqual( input_manifest.to_dict(), { "schema-version": "1.0", - "build": {"name": "opensearch", "version": "1.2.3"}, - "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2"}}, + "build": {"name": "OpenSearch", "version": "1.2.3"}, + "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2", + "args": "-e JAVA_HOME=/opt/java/openjdk-11"}}, + }, + ) + + def test_create_manifest_opensearch_from_default(self) -> None: + input_manifests = InputManifests("OpenSearch") + input_manifest = input_manifests.create_manifest("0.2.3", []) + self.assertEqual( + input_manifest.to_dict(), + { + "schema-version": "1.0", + "build": {"name": "OpenSearch", "version": "0.2.3"}, + "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2", + "args": "-e JAVA_HOME=/opt/java/openjdk-17"}}, }, ) def test_create_manifest_opensearch_dashboards(self) -> None: - input_manifests = InputManifests("opensearch-dashboards") + input_manifests = InputManifests("OpenSearch Dashboards") input_manifest = input_manifests.create_manifest("1.2.3", []) self.assertEqual( input_manifest.to_dict(), { "schema-version": "1.0", - "build": {"name": "opensearch-dashboards", "version": "1.2.3"}, - "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2"}}, + "build": {"name": "OpenSearch Dashboards", "version": "1.2.3"}, + "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2", }}, + }, + ) + + def test_create_manifest_opensearch_dashboards_from_default(self) -> None: + input_manifests = InputManifests("OpenSearch Dashboards") + input_manifest = input_manifests.create_manifest("4.2.3", []) + self.assertEqual( + input_manifest.to_dict(), + { + "schema-version": "1.0", + "build": {"name": "OpenSearch Dashboards", "version": "4.2.3"}, + "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2", }}, }, ) diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch.py index f130594744..ae2d2b7bce 100644 --- a/tests/tests_manifests_workflow/test_input_manifests_opensearch.py +++ b/tests/tests_manifests_workflow/test_input_manifests_opensearch.py @@ -30,6 +30,8 @@ def test_files(self) -> None: @patch("os.makedirs") @patch("os.chdir") + @patch("manifests_workflow.input_manifests.InputComponents") + @patch("manifests_workflow.input_manifests.InputManifest.from_file") @patch("manifests_workflow.input_manifests.InputManifests.add_to_cron") @patch("manifests_workflow.input_manifests.InputManifest.from_path") @patch("manifests_workflow.input_manifests_opensearch.ComponentOpenSearchMin") @@ -37,7 +39,8 @@ def test_files(self) -> None: @patch("manifests_workflow.input_manifests.InputManifest") def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch: MagicMock, mock_component_opensearch_min: MagicMock, mock_input_manifest_from_path: MagicMock, - mock_add_to_cron: MagicMock, *mocks: MagicMock) -> None: + mock_add_to_cron: MagicMock, mock_input_manifest_from_file: MagicMock, + mock_input_manifest_component: MagicMock, *mocks: MagicMock) -> None: mock_component_opensearch_min.return_value = MagicMock(name="OpenSearch") mock_component_opensearch_min.branches.return_value = ["main", "0.9.0"] mock_component_opensearch_min.checkout.return_value = MagicMock(version="0.9.0") @@ -48,7 +51,7 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch: } manifests = InputManifestsOpenSearch() manifests.update() - self.assertEqual(mock_input_manifest().to_file.call_count, 2) + self.assertEqual(mock_input_manifest_from_file().to_file.call_count, 2) calls = [ call( os.path.join( @@ -65,7 +68,7 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch: ) ), ] - mock_input_manifest().to_file.assert_has_calls(calls) + mock_input_manifest_from_file().to_file.assert_has_calls(calls) mock_add_to_cron.assert_has_calls([ call('0.10.0'), call('0.9.0') diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py index f529884e69..93ff49c6d9 100644 --- a/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py +++ b/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py @@ -29,19 +29,24 @@ def test_files(self) -> None: @patch("os.makedirs") @patch("os.chdir") + @patch("manifests_workflow.input_manifests.InputComponents") + @patch("manifests_workflow.input_manifests.InputManifest.from_file") @patch("manifests_workflow.input_manifests.InputManifests.add_to_cron") @patch("manifests_workflow.input_manifests.InputManifest.from_path") @patch("manifests_workflow.input_manifests_opensearch_dashboards.ComponentOpenSearchDashboardsMin") @patch("manifests_workflow.input_manifests.InputManifest") def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch_min: MagicMock, - mock_input_manifest_from_path: MagicMock, mock_add_to_cron: MagicMock, *mocks: MagicMock) -> None: + mock_input_manifest_from_path: MagicMock, mock_add_to_cron: MagicMock, + mock_input_manifest_from_file: MagicMock, mock_input_manifest_component: MagicMock, + *mocks: MagicMock) -> None: mock_component_opensearch_min.return_value = MagicMock(name="OpenSearch-Dashboards") mock_component_opensearch_min.branches.return_value = ["main", "0.9.0"] mock_component_opensearch_min.checkout.return_value = MagicMock(version="0.9.0") mock_input_manifest_from_path.return_value = MagicMock(components=[]) + manifests = InputManifestsOpenSearchDashboards() manifests.update() - self.assertEqual(mock_input_manifest().to_file.call_count, 1) + self.assertEqual(mock_input_manifest_from_file().to_file.call_count, 1) calls = [ call( os.path.join( @@ -51,7 +56,7 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch_ ) ) ] - mock_input_manifest().to_file.assert_has_calls(calls) + mock_input_manifest_from_file().to_file.assert_has_calls(calls) mock_add_to_cron.assert_has_calls([ call('0.9.0') ])