Build incremental components through build workflow with loading previous build manifest

jenkins/opensearch/distribution-build.jenkinsfile

@@ -138,7 +138,7 @@ pipeline {
                             def snapshotBuild =
                                 build job: 'publish-opensearch-min-snapshots',
                                 propagate: false,
-                                wait: false,                               
+                                wait: false,
                                 parameters: [
                                     string(name: 'INPUT_MANIFEST', value: "${INPUT_MANIFEST}"),
                                 ]
@@ -849,4 +849,4 @@ def markStageUnstableIfPluginsFailedToBuild() {
     if (stageLogs.any{e -> e.contains('Failed plugins are')}) {
         unstable('Some plugins failed to build. See the ./build.sh step for logs and more details')
     }
-}
+}

src/build_workflow/build_incremental.py

@@ -9,8 +9,14 @@
 import os
 from typing import List
 
+from build_workflow.build_args import BuildArgs
+from build_workflow.build_recorder import BuildRecorder
+from build_workflow.build_target import BuildTarget
+from build_workflow.builders import Builders
 from manifests.build_manifest import BuildManifest
 from manifests.input_manifest import InputManifest
+from paths.build_output_dir import BuildOutputDir
+from system.temporary_directory import TemporaryDirectory
 
 
 class BuildIncremental:
@@ -62,3 +68,56 @@ def rebuild_plugins(self, changed_plugins: List, input_manifest: InputManifest)
 
         logging.info(f"Rebuilding list is {rebuild_list}")
         return rebuild_list
+
+    def build_incremental(self, args: BuildArgs, input_manifest: InputManifest, components: List) -> None:
+        build_manifest_path = os.path.join(self.distribution, "builds", input_manifest.build.filename, "manifest.yml")
+        if not os.path.exists(build_manifest_path):
+            logging.error("Previous build manifest is not exists. Throw error.")
+
+        logging.info(f"Build {components} incrementally.")
+
+        build_manifest_data = BuildManifest.from_path(build_manifest_path).__to_dict__()
+
+        output_dir = BuildOutputDir(input_manifest.build.filename, args.distribution).dir
+        failed_plugins = []
+
+        with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
+            logging.info(f"Building in {work_dir.name}")
+            target = BuildTarget(
+                name=input_manifest.build.name,
+                version=input_manifest.build.version,
+                qualifier=input_manifest.build.qualifier,
+                patches=input_manifest.build.patches,
+                snapshot=args.snapshot if args.snapshot is not None else input_manifest.build.snapshot,
+                output_dir=output_dir,
+                distribution=args.distribution,
+                platform=args.platform or input_manifest.build.platform,
+                architecture=args.architecture or input_manifest.build.architecture,
+            )
+
+            build_recorder_incremental = BuildRecorder(target, build_manifest_data)
+
+            logging.info(f"Building {input_manifest.build.name} ({target.architecture}) into {target.output_dir}")
+
+            for component in input_manifest.components.select(focus=components, platform=target.platform):
+                logging.info(f"Rebuilding {component.name}")
+
+                builder = Builders.builder_from(component, target)
+                try:
+                    builder.checkout(work_dir.name)
+                    builder.build(build_recorder_incremental)
+                    builder.export_artifacts(build_recorder_incremental)
+                    logging.info(f"Successfully built {component.name}")
+                except:
+                    logging.error(f"Error incremental building {component.name}.")
+                    if args.continue_on_error and component.name not in ['OpenSearch', 'job-scheduler', 'common-utils', 'OpenSearch-Dashboards']:
+                        failed_plugins.append(component.name)
+                        continue
+                    else:
+                        raise
+
+            build_recorder_incremental.write_manifest()
+
+        if len(failed_plugins) > 0:
+            logging.error(f"Failed plugins are {failed_plugins}")
+        logging.info("Done.")

src/build_workflow/build_recorder.py

@@ -17,8 +17,8 @@
 
 
 class BuildRecorder:
-    def __init__(self, target: BuildTarget) -> None:
-        self.build_manifest = self.BuildManifestBuilder(target)
+    def __init__(self, target: BuildTarget, build_manifest: Dict = None) -> None:
+        self.build_manifest = self.BuildManifestBuilder(target, build_manifest)
         self.target = target
         self.name = target.name
 
@@ -53,18 +53,24 @@ def write_manifest(self) -> None:
         logging.info(f"Created build manifest {manifest_path}")
 
     class BuildManifestBuilder:
-        def __init__(self, target: BuildTarget) -> None:
+        def __init__(self, target: BuildTarget, build_manfiest_data: Dict = None) -> None:
             self.data: Dict[str, Any] = {}
-            self.data["build"] = {}
-            self.data["build"]["id"] = target.build_id
-            self.data["build"]["name"] = target.name
-            self.data["build"]["version"] = target.opensearch_version
-            self.data["build"]["platform"] = target.platform
-            self.data["build"]["architecture"] = target.architecture
-            self.data["build"]["distribution"] = target.distribution if target.distribution else "tar"
-            self.data["schema-version"] = "1.2"
             self.components_hash: Dict[str, Dict[str, Any]] = {}
 
+            if build_manfiest_data:
+                self.data = build_manfiest_data
+                for components_data in build_manfiest_data.get("components"):
+                    self.components_hash[components_data["name"]] = components_data
+            else:
+                self.data["build"] = {}
+                self.data["build"]["id"] = target.build_id
+                self.data["build"]["name"] = target.name
+                self.data["build"]["version"] = target.opensearch_version
+                self.data["build"]["platform"] = target.platform
+                self.data["build"]["architecture"] = target.architecture
+                self.data["build"]["distribution"] = target.distribution if target.distribution else "tar"
+                self.data["schema-version"] = "1.2"
+
         def append_component(self, name: str, version: str, repository_url: str, ref: str, commit_id: str) -> None:
             component = {
                 "name": name,
@@ -75,6 +81,7 @@ def append_component(self, name: str, version: str, repository_url: str, ref: st
                 "version": version,
             }
             self.components_hash[name] = component
+            logging.info(f"Appended {name} component in input manifest.")
 
         def append_artifact(self, component: str, type: str, path: str) -> None:
             artifacts = self.components_hash[component]["artifacts"]

src/run_build.py

@@ -45,7 +45,9 @@ def main() -> int:
     if args.incremental:
         buildIncremental = BuildIncremental(manifest, args.distribution)
         list_of_updated_plugins = buildIncremental.commits_diff(manifest)
-        logging.info(f"Plugins for incremental build: {buildIncremental.rebuild_plugins(list_of_updated_plugins, manifest)}")
+        components = buildIncremental.rebuild_plugins(list_of_updated_plugins, manifest)
+        logging.info(f"Plugins for incremental build: {components}")
+        buildIncremental.build_incremental(args, manifest, components)
         return 0
 
     with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:

tests/test_run_build.py

@@ -229,3 +229,12 @@ def test_failed_plugins_default(self, mock_logging_error: Mock, mock_temp: Mock,
         with pytest.raises(Exception, match="Error during build"):
             main()
         mock_logging_error.assert_called_with(f"Error building common-utils, retry with: run_build.py {self.NON_OPENSEARCH_MANIFEST} --component common-utils")
+
+    @patch("argparse._sys.argv", ["run_build.py", OPENSEARCH_MANIFEST, "--incremental"])
+    @patch("run_build.BuildIncremental")
+    def test_main_incremental(self, mock_build_incremental: Mock, *mocks: Any) -> None:
+        main()
+        self.assertEqual(mock_build_incremental.call_count, 1)
+        mock_build_incremental.return_value.commits_diff.assert_called()
+        mock_build_incremental.return_value.rebuild_plugins.assert_called()
+        mock_build_incremental.return_value.build_incremental.assert_called()

tests/tests_build_workflow/test_build_incremental.py

@@ -6,9 +6,12 @@
 # compatible open source license.
 
 import os
+import tempfile
 import unittest
-from typing import List
-from unittest.mock import MagicMock, patch
+from typing import Any, List
+from unittest.mock import MagicMock, call, patch
+
+import pytest
 
 from build_workflow.build_incremental import BuildIncremental
 from manifests.build_manifest import BuildManifest
@@ -20,6 +23,7 @@ class TestBuildIncremental(unittest.TestCase):
         os.path.join(os.path.dirname(__file__), "data", "opensearch-input-2.12.0.yml"))
     BUILD_MANIFEST = BuildManifest.from_path(
         os.path.join(os.path.dirname(__file__), "data", "opensearch-build-tar-2.12.0.yml"))
+    BUILD_MANIFEST_PATH = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-tar-2.12.0.yml")
     INPUT_MANIFEST_DASHBOARDS = InputManifest.from_path(
         os.path.join(os.path.dirname(__file__), "data", "opensearch-dashboards-input-2.12.0.yml"))
     BUILD_MANIFEST_DASHBOARDS = BuildManifest.from_path(
@@ -158,3 +162,138 @@ def test_rebuild_plugins_with_dashboards(self) -> None:
         self.assertEqual(len(rebuild_list), 2)
         self.assertTrue("OpenSearch-Dashboards" in rebuild_list)
         self.assertTrue("observabilityDashboards" in rebuild_list)
+
+    @patch("os.path.join")
+    @patch("build_workflow.build_incremental.Builders.builder_from", return_value=MagicMock())
+    @patch("build_workflow.build_incremental.BuildRecorder", return_value=MagicMock())
+    @patch("build_workflow.build_incremental.TemporaryDirectory")
+    def test_build_incremental_no_prebuild_manifest(self, mock_temp: MagicMock, mock_recorder: MagicMock, mock_builder: MagicMock, mock_join_path: MagicMock, *mocks: Any) -> None:
+        mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir()
+        args = MagicMock()
+        mock_join_path.return_value = "non_exist_path"
+        try:
+            self.buildIncremental.build_incremental(args, self.INPUT_MANIFEST, ["common-utils"])
+            self.assertRaises(FileNotFoundError)
+        except FileNotFoundError:
+            pass
+
+    @patch("build_workflow.build_incremental.logging.info")
+    @patch("paths.build_output_dir")
+    @patch("os.path.exists")
+    @patch("manifests.build_manifest.BuildManifest.from_path")
+    @patch("build_workflow.build_incremental.Builders.builder_from", return_value=MagicMock())
+    @patch("build_workflow.build_incremental.BuildRecorder", return_value=MagicMock())
+    @patch("build_workflow.build_incremental.TemporaryDirectory")
+    def test_build_incremental_with_prebuild_manifest(self, mock_temp: MagicMock, mock_recorder: MagicMock,
+                                                      mock_builder: MagicMock, mock_build_manifest: MagicMock,
+                                                      mock_path_exist: MagicMock, mock_build_output_dir: MagicMock,
+                                                      mock_logging_info: MagicMock, *mocks: Any) -> None:
+        mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir()
+        args = MagicMock()
+        args.distribution = "tar"
+        args.keep = False
+        args.snapshot = None
+        args.platform = "linux"
+        args.architecture = "x64"
+        mock_path_exist.return_value = True
+        mock_build_manifest.return_value = self.BUILD_MANIFEST
+        self.buildIncremental.build_incremental(args, self.INPUT_MANIFEST, ["common-utils", "opensearch-observability"])
+        mock_build_manifest.assert_called_once()
+        mock_build_manifest.assert_called_with(os.path.join("tar", "builds", "opensearch", "manifest.yml"))
+        self.assertTrue(args.platform == "linux")
+        self.assertNotEqual(mock_builder.return_value.build.call_count, 0)
+        self.assertEqual(mock_builder.return_value.build.call_count, 2)
+        self.assertEqual(mock_builder.return_value.build.call_count, mock_builder.return_value.export_artifacts.call_count)
+
+        mock_logging_info.assert_has_calls([
+            call('Rebuilding common-utils'),
+            call('Rebuilding opensearch-observability'),
+        ], any_order=True)
+
+        mock_recorder.assert_called_once()
+        mock_recorder.return_value.write_manifest.assert_called()
+
+    @patch("build_workflow.build_incremental.logging.error")
+    @patch("build_workflow.build_incremental.logging.info")
+    @patch("paths.build_output_dir")
+    @patch("os.path.exists")
+    @patch("manifests.build_manifest.BuildManifest.from_path")
+    @patch("build_workflow.build_incremental.Builders.builder_from", return_value=MagicMock())
+    @patch("build_workflow.build_incremental.BuildRecorder", return_value=MagicMock())
+    @patch("build_workflow.build_incremental.TemporaryDirectory")
+    def test_build_incremental_continue_on_fail_core(self, mock_temp: MagicMock, mock_recorder: MagicMock,
+                                                     mock_builder_from: MagicMock, mock_build_manifest: MagicMock,
+                                                     mock_path_exist: MagicMock, mock_build_output_dir: MagicMock,
+                                                     mock_logging_info: MagicMock, mock_logging_error: MagicMock,
+                                                     *mocks: Any) -> None:
+        mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir()
+        args = MagicMock()
+        args.distribution = "tar"
+        args.keep = False
+        args.snapshot = None
+        args.platform = "linux"
+        args.architecture = "x64"
+        args.continue_on_error = True
+        mock_path_exist.return_value = True
+        mock_build_manifest.return_value = self.BUILD_MANIFEST
+        mock_builder = MagicMock()
+        mock_builder.build.side_effect = Exception("Error building")
+        mock_builder_from.return_value = mock_builder
+
+        with pytest.raises(Exception, match="Error building"):
+            self.buildIncremental.build_incremental(args, self.INPUT_MANIFEST, ["common-utils", "opensearch-observability"])
+
+        mock_logging_error.assert_called_with("Error incremental building common-utils.")
+        mock_build_manifest.assert_called_once()
+        mock_build_manifest.assert_called_with(os.path.join("tar", "builds", "opensearch", "manifest.yml"))
+        self.assertTrue(args.platform == "linux")
+        self.assertNotEqual(mock_builder.build.call_count, 0)
+        self.assertEqual(mock_builder.build.call_count, 1)
+
+        mock_logging_info.assert_has_calls([
+            call('Rebuilding common-utils')
+        ], any_order=True)
+
+        mock_recorder.assert_called_once()
+        mock_recorder.return_value.write_manifest.assert_not_called()
+
+    @patch("build_workflow.build_incremental.logging.error")
+    @patch("build_workflow.build_incremental.logging.info")
+    @patch("paths.build_output_dir")
+    @patch("os.path.exists")
+    @patch("manifests.build_manifest.BuildManifest.from_path")
+    @patch("build_workflow.build_incremental.Builders.builder_from", return_value=MagicMock())
+    @patch("build_workflow.build_incremental.BuildRecorder", return_value=MagicMock())
+    @patch("build_workflow.build_incremental.TemporaryDirectory")
+    def test_build_incremental_continue_on_fail_plugin(self, mock_temp: MagicMock, mock_recorder: MagicMock, mock_builder_from: MagicMock, mock_build_manifest: MagicMock, mock_path_exist: MagicMock,
+                                                       mock_build_output_dir: MagicMock, mock_logging_info: MagicMock, mock_logging_error: MagicMock, *mocks: Any) -> None:
+        mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir()
+        args = MagicMock()
+        args.distribution = "tar"
+        args.keep = False
+        args.snapshot = None
+        args.platform = "linux"
+        args.architecture = "x64"
+        args.continue_on_error = True
+        mock_path_exist.return_value = True
+        mock_build_manifest.return_value = self.BUILD_MANIFEST
+        mock_builder = MagicMock()
+        mock_builder.build.side_effect = Exception("Error build")
+        mock_builder_from.return_value = mock_builder
+
+        self.buildIncremental.build_incremental(args, self.INPUT_MANIFEST, ["ml-commons", "opensearch-observability"])
+
+        mock_logging_error.assert_called_with("Failed plugins are ['ml-commons', 'opensearch-observability']")
+        mock_build_manifest.assert_called_once()
+        mock_build_manifest.assert_called_with(os.path.join("tar", "builds", "opensearch", "manifest.yml"))
+        self.assertTrue(args.platform == "linux")
+        self.assertNotEqual(mock_builder.build.call_count, 0)
+        self.assertEqual(mock_builder.build.call_count, 2)
+
+        mock_logging_info.assert_has_calls([
+            call('Rebuilding ml-commons'),
+            call('Rebuilding opensearch-observability')
+        ], any_order=True)
+
+        mock_recorder.assert_called_once()
+        mock_recorder.return_value.write_manifest.assert_called()