CorePluginSourceDownloader Migration (#179)

* Implement CorePluginSourceDownloader and SourceBuilder

Signed-off-by: Chase Engelbrecht <[email protected]>

* Use unpacking to get jdk_path

Signed-off-by: Chase Engelbrecht <[email protected]>

* Add tests for CorePluginSourceDownloader

Signed-off-by: Chase Engelbrecht <[email protected]>

* Create binaryBuilder interface, have SourceBinaryBuilder implement it. Remove member class initialization in constructor in favor of passing in as args for factory pattern

Signed-off-by: Chase Engelbrecht <[email protected]>

* Rename build_jdk to build_jdk_version

Signed-off-by: Chase Engelbrecht <[email protected]>

* Replace semi colons with colons in class doc

Signed-off-by: Chase Engelbrecht <[email protected]>

osbenchmark/builder/downloaders/builders/binary_builder.py

@@ -0,0 +1,19 @@
+from abc import ABC, abstractmethod
+
+
+class BinaryBuilder(ABC):
+    """
+    A BinaryBuilder is used to wrap the executor calls necessary for constructing binaries from code
+    """
+
+    @abstractmethod
+    def build(self, host, build_commands, override_source_directory):
+        """
+        Runs the provided commands on the given host to build binaries
+
+        :param host: A host object representing the machine on which to run the commands
+        :param build_commands: A list of strings representing sequential bash commands used to build the binaries
+        :param override_source_directory: A string representing the source directory where the pre-binary code is located
+        :return None
+        """
+        raise NotImplementedError

osbenchmark/builder/downloaders/builders/source_binary_builder.py

@@ -0,0 +1,36 @@
+import logging
+import os
+
+from osbenchmark.builder.downloaders.builders.binary_builder import BinaryBuilder
+from osbenchmark.exceptions import ExecutorError, BuildError
+
+
+class SourceBinaryBuilder(BinaryBuilder):
+    def __init__(self, executor, path_manager, jdk_resolver, source_directory, build_jdk_version, log_directory):
+        self.logger = logging.getLogger(__name__)
+        self.executor = executor
+        self.path_manager = path_manager
+        self.jdk_resolver = jdk_resolver
+
+        self.source_directory = source_directory
+        self.build_jdk_version = build_jdk_version
+        self.log_directory = log_directory
+
+    def build(self, host, build_commands, override_source_directory=None):
+        for build_command in build_commands:
+            self._run_build_command(host, build_command, override_source_directory)
+
+    def _run_build_command(self, host, build_command, override_source_directory):
+        source_directory = self.source_directory if override_source_directory is None else override_source_directory
+
+        self.path_manager.create_path(host, self.log_directory, create_locally=False)
+        log_file = os.path.join(self.log_directory, "build.log")
+
+        _, jdk_path = self.jdk_resolver.resolve_jdk_path(host, self.build_jdk_version)
+        self.executor.execute(host, f"export JAVA_HOME={jdk_path}")
+
+        self.logger.info("Running build command [%s]", build_command)
+        try:
+            self.executor.execute(host, f"{source_directory}/{build_command} > {log_file} 2>&1")
+        except ExecutorError:
+            raise BuildError(f"Executing {build_command} failed. The build log can be found at {log_file}")

osbenchmark/builder/downloaders/core_plugin_source_downloader.py

@@ -0,0 +1,26 @@
+from osbenchmark.builder.downloaders.downloader import Downloader
+
+
+class CorePluginSourceDownloader(Downloader):
+    def __init__(self, plugin, executor, source_repository_provider, binary_builder, opensearch_source_dir):
+        super().__init__(executor)
+        self.plugin = plugin
+        self.source_repository_provider = source_repository_provider
+        self.binary_builder = binary_builder
+        self.opensearch_source_dir = opensearch_source_dir
+
+    def download(self, host):
+        self._fetch(host)
+        self._prepare(host)
+
+        return {self.plugin.name: self._get_zip_path()}
+
+    def _fetch(self, host):
+        return self.source_repository_provider.fetch_repository(host, None, "current", self.opensearch_source_dir)
+
+    def _prepare(self, host):
+        if self.binary_builder:
+            self.binary_builder.build(host, [f"gradlew :plugins:{self.plugin.name}:assemble"])
+
+    def _get_zip_path(self):
+        return f"file://{self.opensearch_source_dir}/plugins/{self.plugin.name}/build/distributions/*.zip"

tests/builder/downloaders/builders/source_binary_builder_test.py

@@ -0,0 +1,47 @@
+from unittest import TestCase, mock
+from unittest.mock import Mock
+
+from osbenchmark.builder.downloaders.builders.source_binary_builder import SourceBinaryBuilder
+from osbenchmark.exceptions import BuildError, ExecutorError
+
+
+class SourceBinaryBuilderTest(TestCase):
+    def setUp(self):
+        self.host = None
+        self.build_commands = ["gradle build"]
+
+        self.executor = Mock()
+        self.path_manager = Mock()
+        self.jdk_resolver = Mock()
+
+        self.os_src_dir = "/fake/src/dir"
+        self.build_jdk_version = 13
+        self.log_dir = "/benchmark/logs"
+
+        self.source_binary_builder = SourceBinaryBuilder(self.executor, self.path_manager, self.jdk_resolver,
+                                                         self.os_src_dir, self.build_jdk_version, self.log_dir)
+
+        self.jdk_resolver.resolve_jdk_path.return_value = (13, "/path/to/jdk")
+
+    def test_build(self):
+        self.source_binary_builder.build(self.host, self.build_commands)
+
+        self.executor.execute.assert_has_calls([
+            mock.call(self.host, "export JAVA_HOME=/path/to/jdk"),
+            mock.call(self.host, "/fake/src/dir/gradle build > /benchmark/logs/build.log 2>&1")
+        ])
+
+    def test_build_with_src_dir_override(self):
+        self.source_binary_builder.build(self.host, self.build_commands, "/override/src")
+
+        self.executor.execute.assert_has_calls([
+            mock.call(self.host, "export JAVA_HOME=/path/to/jdk"),
+            mock.call(self.host, "/override/src/gradle build > /benchmark/logs/build.log 2>&1")
+        ])
+
+    def test_build_failure(self):
+        # Set JAVA_HOME, execute build command
+        self.executor.execute.side_effect = [None, ExecutorError("fake err")]
+
+        with self.assertRaises(BuildError):
+            self.source_binary_builder.build(self.host, self.build_commands)

tests/builder/downloaders/core_plugin_source_downloader_test.py

@@ -0,0 +1,23 @@
+from unittest import TestCase
+from unittest.mock import Mock
+
+from osbenchmark.builder.downloaders.core_plugin_source_downloader import CorePluginSourceDownloader
+from osbenchmark.builder.provision_config import PluginDescriptor
+
+
+class CorePluginSourceDownloaderTest(TestCase):
+    def setUp(self):
+        self.host = None
+
+        self.executor = Mock()
+        self.source_repository_provider = Mock()
+        self.plugin = PluginDescriptor(name="my-plugin")
+        self.builder = Mock()
+        self.opensearch_source_dir = "/fake/path"
+
+        self.source_downloader = CorePluginSourceDownloader(self.plugin, self.executor, self.source_repository_provider,
+                                                            self.builder, self.opensearch_source_dir)
+
+    def test_download(self):
+        plugin_binary = self.source_downloader.download(self.host)
+        self.assertEqual(plugin_binary, {"my-plugin": "file:///fake/path/plugins/my-plugin/build/distributions/*.zip"})