From 2f0f38fcc6606e54fc878298e5a8c1e3f06f7f17 Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Wed, 30 Mar 2022 15:28:57 -0500 Subject: [PATCH 1/7] Initial PluginInstaller migration Signed-off-by: Chase Engelbrecht --- .../builder/installers/plugin_installer.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 osbenchmark/builder/installers/plugin_installer.py diff --git a/osbenchmark/builder/installers/plugin_installer.py b/osbenchmark/builder/installers/plugin_installer.py new file mode 100644 index 000000000..218c68764 --- /dev/null +++ b/osbenchmark/builder/installers/plugin_installer.py @@ -0,0 +1,44 @@ +import logging +import os + +from osbenchmark.builder.installers.installer import Installer +from osbenchmark.builder.provision_config import BootstrapHookHandler +from osbenchmark.builder.utils.config_applier import ConfigApplier +from osbenchmark.builder.utils.path_manager import PathManager +from osbenchmark.builder.utils.template_renderer import TemplateRenderer + + +class PluginInstaller(Installer): + def __init__(self, plugin, executor, hook_handler_class=BootstrapHookHandler): + super().__init__(executor) + self.logger = logging.getLogger(__name__) + self.plugin = plugin + self.hook_handler = hook_handler_class(self.plugin) + if self.hook_handler.can_load(): + self.hook_handler.load() + self.template_renderer = TemplateRenderer() + self.path_manager = PathManager(executor) + self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager) + + def install(self, host, binaries, all_node_ips): + install_cmd = self._get_install_command(host, binaries) + self.executor.execute(host, install_cmd) + + def _get_install_command(self, host, binaries): + installer_binary_path = os.path.join(host.node.binary_path, "bin", "opensearch-plugin") + plugin_binary_path = binaries.get(self.plugin.name) + + if plugin_binary_path: + self.logger.info("Installing [%s] into [%s] from [%s]", self.plugin.name, host.node.binary_path, plugin_binary_path) + return '%s install --batch "%s"' % (installer_binary_path, plugin_binary_path) + else: + self.logger.info("Installing [%s] into [%s]", self.plugin.name, host.node.binary_path) + return '%s install --batch "%s"' % (installer_binary_path, self.plugin.name) + + def get_config_vars(self): + + def invoke_install_hook(self, phase, variables, env): + self.hook_handler.invoke(phase.name, variables=variables, env=env) + + def cleanup(self, host): + pass From c000e33d3299ff9018238a5846c02ea795a6a56b Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Wed, 30 Mar 2022 16:42:23 -0500 Subject: [PATCH 2/7] More PluginInstaller migration Signed-off-by: Chase Engelbrecht --- osbenchmark/builder/installers/plugin_installer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/osbenchmark/builder/installers/plugin_installer.py b/osbenchmark/builder/installers/plugin_installer.py index 218c68764..08c558629 100644 --- a/osbenchmark/builder/installers/plugin_installer.py +++ b/osbenchmark/builder/installers/plugin_installer.py @@ -36,6 +36,7 @@ def _get_install_command(self, host, binaries): return '%s install --batch "%s"' % (installer_binary_path, self.plugin.name) def get_config_vars(self): + return self.plugin.variables def invoke_install_hook(self, phase, variables, env): self.hook_handler.invoke(phase.name, variables=variables, env=env) From c4b80946f13ef7eae502170a0eb58eae7a7f760c Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Thu, 31 Mar 2022 14:40:33 -0500 Subject: [PATCH 3/7] Make config_vars optional param in OpenSearch and PluginInstallers, add unit tests for PluginInstaller Signed-off-by: Chase Engelbrecht --- .../installers/opensearch_installer.py | 17 +++--- .../builder/installers/plugin_installer.py | 6 ++- .../installers/plugin_installer_test.py | 52 +++++++++++++++++++ 3 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 tests/builder/installers/plugin_installer_test.py diff --git a/osbenchmark/builder/installers/opensearch_installer.py b/osbenchmark/builder/installers/opensearch_installer.py index d34ab903c..c80e2297a 100644 --- a/osbenchmark/builder/installers/opensearch_installer.py +++ b/osbenchmark/builder/installers/opensearch_installer.py @@ -26,9 +26,9 @@ def __init__(self, provision_config_instance, executor, hook_handler_class=Boots self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager) self.host_cleaner = HostCleaner(self.path_manager) - def install(self, host, binaries, all_node_ips): + def install(self, host, binaries, all_node_ips, config_vars=None): node = self._create_node() - self._prepare_node(host, node, binaries[OpenSearchInstaller.OPENSEARCH_BINARY_KEY], all_node_ips) + self._prepare_node(host, node, binaries[OpenSearchInstaller.OPENSEARCH_BINARY_KEY], all_node_ips, config_vars) return node @@ -50,14 +50,14 @@ def _create_node(self): data_paths=None, telemetry=None) - def _prepare_node(self, host, node, binary, all_node_ips): + def _prepare_node(self, host, node, binary, all_node_ips, config_vars): self._prepare_directories(host, node) self._extract_opensearch(host, node, binary) self._update_node_binary_path(node) self._set_node_data_paths(node) # we need to immediately delete the prebundled config files as plugins may copy their configuration during installation. self._delete_prebundled_config_files(host, node) - self._prepare_config_files(host, node, all_node_ips) + self._prepare_config_files(host, node, all_node_ips, config_vars) def _prepare_directories(self, host, node): directories_to_create = [node.binary_path, node.log_path, node.heap_dump_path] @@ -79,12 +79,13 @@ def _delete_prebundled_config_files(self, host, node): self.logger.info("Deleting pre-bundled OpenSearch configuration at [%s]", config_path) self.path_manager.delete_path(host, config_path) - def _prepare_config_files(self, host, node, all_node_ips): - config_vars = self.get_config_vars(host, node, all_node_ips) + def _prepare_config_files(self, host, node, all_node_ips, config_vars): + if not config_vars: + config_vars = self.get_config_vars(host, node, all_node_ips) self.config_applier.apply_configs(host, node, self.provision_config_instance.config_paths, config_vars) def get_config_vars(self, host, node, all_node_ips): - provisioner_defaults = { + installer_defaults = { "cluster_name": self.provision_config_instance.variables["cluster_name"], "node_name": node.name, "data_paths": node.data_paths[0], @@ -103,7 +104,7 @@ def get_config_vars(self, host, node, all_node_ips): } config_vars = {} config_vars.update(self.provision_config_instance.variables) - config_vars.update(provisioner_defaults) + config_vars.update(installer_defaults) return config_vars def invoke_install_hook(self, phase, variables, env): diff --git a/osbenchmark/builder/installers/plugin_installer.py b/osbenchmark/builder/installers/plugin_installer.py index 08c558629..671edac60 100644 --- a/osbenchmark/builder/installers/plugin_installer.py +++ b/osbenchmark/builder/installers/plugin_installer.py @@ -20,10 +20,14 @@ def __init__(self, plugin, executor, hook_handler_class=BootstrapHookHandler): self.path_manager = PathManager(executor) self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager) - def install(self, host, binaries, all_node_ips): + def install(self, host, binaries, all_node_ips, config_vars=None): install_cmd = self._get_install_command(host, binaries) self.executor.execute(host, install_cmd) + if not config_vars: + config_vars = self.get_config_vars() + self.config_applier.apply_configs(host, host.node, self.plugin.config_paths, config_vars) + def _get_install_command(self, host, binaries): installer_binary_path = os.path.join(host.node.binary_path, "bin", "opensearch-plugin") plugin_binary_path = binaries.get(self.plugin.name) diff --git a/tests/builder/installers/plugin_installer_test.py b/tests/builder/installers/plugin_installer_test.py new file mode 100644 index 000000000..f44ee71d0 --- /dev/null +++ b/tests/builder/installers/plugin_installer_test.py @@ -0,0 +1,52 @@ +from unittest import TestCase, mock +from unittest.mock import Mock + +from osbenchmark.builder.installers.plugin_installer import PluginInstaller +from osbenchmark.builder.models.host import Host +from osbenchmark.builder.models.node import Node +from osbenchmark.builder.provision_config import PluginDescriptor + + +class PluginInstallerTest(TestCase): + def setUp(self): + self.node = Node(binary_path="/fake_binary_path", data_paths=["/fake1", "/fake2"], name=None, + pid=None, telemetry=None, port=None, root_dir=None, log_path=None, heap_dump_path=None) + self.host = Host(name="fake", address="10.17.22.23", metadata={}, node=self.node) + self.binaries = {"unit-test-plugin": "/data/builds/distributions"} + self.all_node_ips = [] + + self.executor = Mock() + self.plugin = PluginDescriptor(name="unit-test-plugin", config_paths=["default"], variables={"active": True}) + + self.plugin_installer = PluginInstaller(self.plugin, self.executor) + self.plugin_installer.config_applier = Mock() + + def test_plugin_install_with_binary_path(self): + self.plugin_installer.install(self.host, self.binaries, self.all_node_ips) + + self.executor.execute.assert_has_calls([ + mock.call(self.host, "/fake_binary_path/bin/opensearch-plugin install --batch \"/data/builds/distributions\"") + ]) + self.plugin_installer.config_applier.apply_configs.assert_has_calls([ + mock.call(self.host, self.node, ["default"], {"active": True}) + ]) + + def test_plugin_install_without_binary_path(self): + self.plugin_installer.install(self.host, {}, self.all_node_ips) + + self.executor.execute.assert_has_calls([ + mock.call(self.host, "/fake_binary_path/bin/opensearch-plugin install --batch \"unit-test-plugin\"") + ]) + self.plugin_installer.config_applier.apply_configs.assert_has_calls([ + mock.call(self.host, self.node, ["default"], {"active": True}) + ]) + + def test_config_vars_override(self): + self.plugin_installer.install(self.host, {}, self.all_node_ips, {"my": "override"}) + + self.executor.execute.assert_has_calls([ + mock.call(self.host, "/fake_binary_path/bin/opensearch-plugin install --batch \"unit-test-plugin\"") + ]) + self.plugin_installer.config_applier.apply_configs.assert_has_calls([ + mock.call(self.host, self.node, ["default"], {"my": "override"}) + ]) From 61acc8b2751759eb81e2df909ddf0940528c611f Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Thu, 31 Mar 2022 14:56:04 -0500 Subject: [PATCH 4/7] Add disable=arguments-differ for linter Signed-off-by: Chase Engelbrecht --- osbenchmark/builder/installers/opensearch_installer.py | 1 + osbenchmark/builder/installers/plugin_installer.py | 1 + 2 files changed, 2 insertions(+) diff --git a/osbenchmark/builder/installers/opensearch_installer.py b/osbenchmark/builder/installers/opensearch_installer.py index c80e2297a..dab797cfa 100644 --- a/osbenchmark/builder/installers/opensearch_installer.py +++ b/osbenchmark/builder/installers/opensearch_installer.py @@ -26,6 +26,7 @@ def __init__(self, provision_config_instance, executor, hook_handler_class=Boots self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager) self.host_cleaner = HostCleaner(self.path_manager) + # pylint: disable=arguments-differ def install(self, host, binaries, all_node_ips, config_vars=None): node = self._create_node() self._prepare_node(host, node, binaries[OpenSearchInstaller.OPENSEARCH_BINARY_KEY], all_node_ips, config_vars) diff --git a/osbenchmark/builder/installers/plugin_installer.py b/osbenchmark/builder/installers/plugin_installer.py index 671edac60..4bbd7d624 100644 --- a/osbenchmark/builder/installers/plugin_installer.py +++ b/osbenchmark/builder/installers/plugin_installer.py @@ -20,6 +20,7 @@ def __init__(self, plugin, executor, hook_handler_class=BootstrapHookHandler): self.path_manager = PathManager(executor) self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager) + # pylint: disable=arguments-differ def install(self, host, binaries, all_node_ips, config_vars=None): install_cmd = self._get_install_command(host, binaries) self.executor.execute(host, install_cmd) From 6c0b5c4caf45fe0a211bc974c4be5f4862b5c75b Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Fri, 1 Apr 2022 14:41:14 -0500 Subject: [PATCH 5/7] Refactor to use Preparer interface in SingularBareInstaller Signed-off-by: Chase Engelbrecht --- .../builder/installers/preparers/__init__.py | 0 .../opensearch_preparer.py} | 26 ++----- .../plugin_preparer.py} | 24 ++----- .../builder/installers/preparers/preparer.py | 56 +++++++++++++++ .../installers/singular_bare_installer.py | 37 ++++++++++ .../builder/utils/java_home_resolver.py | 32 +++------ .../builder/installers/preparers/__init__.py | 0 .../opensearch_preparer_test.py} | 47 ++++++------ .../plugin_preparer_test.py} | 29 +++----- .../singular_bare_installer_test.py | 71 +++++++++++++++++++ .../builder/utils/java_home_resolver_test.py | 32 +++++---- 11 files changed, 236 insertions(+), 118 deletions(-) create mode 100644 osbenchmark/builder/installers/preparers/__init__.py rename osbenchmark/builder/installers/{opensearch_installer.py => preparers/opensearch_preparer.py} (78%) rename osbenchmark/builder/installers/{plugin_installer.py => preparers/plugin_preparer.py} (54%) create mode 100644 osbenchmark/builder/installers/preparers/preparer.py create mode 100644 osbenchmark/builder/installers/singular_bare_installer.py create mode 100644 tests/builder/installers/preparers/__init__.py rename tests/builder/installers/{opensearch_installer_test.py => preparers/opensearch_preparer_test.py} (56%) rename tests/builder/installers/{plugin_installer_test.py => preparers/plugin_preparer_test.py} (51%) create mode 100644 tests/builder/installers/singular_bare_installer_test.py diff --git a/osbenchmark/builder/installers/preparers/__init__.py b/osbenchmark/builder/installers/preparers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/osbenchmark/builder/installers/opensearch_installer.py b/osbenchmark/builder/installers/preparers/opensearch_preparer.py similarity index 78% rename from osbenchmark/builder/installers/opensearch_installer.py rename to osbenchmark/builder/installers/preparers/opensearch_preparer.py index dab797cfa..e287dcff6 100644 --- a/osbenchmark/builder/installers/opensearch_installer.py +++ b/osbenchmark/builder/installers/preparers/opensearch_preparer.py @@ -2,34 +2,28 @@ import os import uuid -from osbenchmark.builder.installers.installer import Installer +from osbenchmark.builder.installers.preparers.preparer import Preparer from osbenchmark.builder.models.node import Node -from osbenchmark.builder.provision_config import BootstrapHookHandler -from osbenchmark.builder.utils.config_applier import ConfigApplier from osbenchmark.builder.utils.host_cleaner import HostCleaner from osbenchmark.builder.utils.path_manager import PathManager -from osbenchmark.builder.utils.template_renderer import TemplateRenderer -class OpenSearchInstaller(Installer): +class OpenSearchPreparer(Preparer): OPENSEARCH_BINARY_KEY = "opensearch" - def __init__(self, provision_config_instance, executor, hook_handler_class=BootstrapHookHandler): + def __init__(self, provision_config_instance, executor, hook_handler_class): super().__init__(executor) self.logger = logging.getLogger(__name__) self.provision_config_instance = provision_config_instance self.hook_handler = hook_handler_class(self.provision_config_instance) if self.hook_handler.can_load(): self.hook_handler.load() - self.template_renderer = TemplateRenderer() self.path_manager = PathManager(executor) - self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager) self.host_cleaner = HostCleaner(self.path_manager) - # pylint: disable=arguments-differ - def install(self, host, binaries, all_node_ips, config_vars=None): + def prepare(self, host, binaries): node = self._create_node() - self._prepare_node(host, node, binaries[OpenSearchInstaller.OPENSEARCH_BINARY_KEY], all_node_ips, config_vars) + self._prepare_node(host, node, binaries[OpenSearchPreparer.OPENSEARCH_BINARY_KEY]) return node @@ -51,14 +45,13 @@ def _create_node(self): data_paths=None, telemetry=None) - def _prepare_node(self, host, node, binary, all_node_ips, config_vars): + def _prepare_node(self, host, node, binary): self._prepare_directories(host, node) self._extract_opensearch(host, node, binary) self._update_node_binary_path(node) self._set_node_data_paths(node) # we need to immediately delete the prebundled config files as plugins may copy their configuration during installation. self._delete_prebundled_config_files(host, node) - self._prepare_config_files(host, node, all_node_ips, config_vars) def _prepare_directories(self, host, node): directories_to_create = [node.binary_path, node.log_path, node.heap_dump_path] @@ -80,11 +73,6 @@ def _delete_prebundled_config_files(self, host, node): self.logger.info("Deleting pre-bundled OpenSearch configuration at [%s]", config_path) self.path_manager.delete_path(host, config_path) - def _prepare_config_files(self, host, node, all_node_ips, config_vars): - if not config_vars: - config_vars = self.get_config_vars(host, node, all_node_ips) - self.config_applier.apply_configs(host, node, self.provision_config_instance.config_paths, config_vars) - def get_config_vars(self, host, node, all_node_ips): installer_defaults = { "cluster_name": self.provision_config_instance.variables["cluster_name"], @@ -108,7 +96,7 @@ def get_config_vars(self, host, node, all_node_ips): config_vars.update(installer_defaults) return config_vars - def invoke_install_hook(self, phase, variables, env): + def invoke_install_hook(self, host, phase, variables, env): self.hook_handler.invoke(phase.name, variables=variables, env=env) def cleanup(self, host): diff --git a/osbenchmark/builder/installers/plugin_installer.py b/osbenchmark/builder/installers/preparers/plugin_preparer.py similarity index 54% rename from osbenchmark/builder/installers/plugin_installer.py rename to osbenchmark/builder/installers/preparers/plugin_preparer.py index 4bbd7d624..2feba06b2 100644 --- a/osbenchmark/builder/installers/plugin_installer.py +++ b/osbenchmark/builder/installers/preparers/plugin_preparer.py @@ -1,34 +1,22 @@ import logging import os -from osbenchmark.builder.installers.installer import Installer -from osbenchmark.builder.provision_config import BootstrapHookHandler -from osbenchmark.builder.utils.config_applier import ConfigApplier -from osbenchmark.builder.utils.path_manager import PathManager -from osbenchmark.builder.utils.template_renderer import TemplateRenderer +from osbenchmark.builder.installers.preparers.preparer import Preparer -class PluginInstaller(Installer): - def __init__(self, plugin, executor, hook_handler_class=BootstrapHookHandler): +class PluginPreparer(Preparer): + def __init__(self, plugin, executor, hook_handler_class): super().__init__(executor) self.logger = logging.getLogger(__name__) self.plugin = plugin self.hook_handler = hook_handler_class(self.plugin) if self.hook_handler.can_load(): self.hook_handler.load() - self.template_renderer = TemplateRenderer() - self.path_manager = PathManager(executor) - self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager) - # pylint: disable=arguments-differ - def install(self, host, binaries, all_node_ips, config_vars=None): + def prepare(self, host, binaries): install_cmd = self._get_install_command(host, binaries) self.executor.execute(host, install_cmd) - if not config_vars: - config_vars = self.get_config_vars() - self.config_applier.apply_configs(host, host.node, self.plugin.config_paths, config_vars) - def _get_install_command(self, host, binaries): installer_binary_path = os.path.join(host.node.binary_path, "bin", "opensearch-plugin") plugin_binary_path = binaries.get(self.plugin.name) @@ -40,10 +28,10 @@ def _get_install_command(self, host, binaries): self.logger.info("Installing [%s] into [%s]", self.plugin.name, host.node.binary_path) return '%s install --batch "%s"' % (installer_binary_path, self.plugin.name) - def get_config_vars(self): + def get_config_vars(self, host, node, all_node_ips): return self.plugin.variables - def invoke_install_hook(self, phase, variables, env): + def invoke_install_hook(self, host, phase, variables, env): self.hook_handler.invoke(phase.name, variables=variables, env=env) def cleanup(self, host): diff --git a/osbenchmark/builder/installers/preparers/preparer.py b/osbenchmark/builder/installers/preparers/preparer.py new file mode 100644 index 000000000..1d88782e0 --- /dev/null +++ b/osbenchmark/builder/installers/preparers/preparer.py @@ -0,0 +1,56 @@ +from abc import ABC, abstractmethod + + +class Preparer(ABC): + """ + A preparer is used for preparing the installation of a node by setting up the filesystem, binaries, and install hooks + """ + + def __init__(self, executor): + self.executor = executor + + @abstractmethod + def prepare(self, host, binaries): + """ + Prepares the filesystem and binaries on a node + + ;param host: A Host object defining the host on which to prepare the data + ;param binaries: A map of components to download paths on the host + ;return node: A Node object detailing the installation data of the node on the host. May be None if no Node was generated + """ + raise NotImplementedError + + @abstractmethod + def get_config_vars(self, host, node, all_node_ips): + """ + Gets the config file(s) variables associated with the given preparer + + ;param host: A Host object defining a machine within a cluster + ;param node: A Node object defining the node on a host + ;param all_node_ips: A list of the ips for each node in the cluster. Used for cluster formation + ;return dict: A key value pair of the config variables + """ + raise NotImplementedError + + @abstractmethod + def invoke_install_hook(self, host, phase, variables, env): + """ + Invokes the associated install hook + + ;param host: A Host object defining the host on which to invoke the install hook + ;param phase: The BoostrapPhase of install hook + ;param variables: Key value pairs to be passed to the install hook + ;param env: Key value pairs of environment variables to be passed ot the install hook + ;return None + """ + raise NotImplementedError + + @abstractmethod + def cleanup(self, host): + """ + Removes the data that was downloaded, installed, and created on a given host during the test execution + + ;param host: A Host object defining the host on which to remove the data + ;return None + """ + raise NotImplementedError diff --git a/osbenchmark/builder/installers/singular_bare_installer.py b/osbenchmark/builder/installers/singular_bare_installer.py new file mode 100644 index 000000000..1394e74c0 --- /dev/null +++ b/osbenchmark/builder/installers/singular_bare_installer.py @@ -0,0 +1,37 @@ +from osbenchmark.builder.installers.installer import Installer +from osbenchmark.builder.provision_config import BootstrapPhase +from osbenchmark.builder.utils.config_applier import ConfigApplier +from osbenchmark.builder.utils.java_home_resolver import JavaHomeResolver +from osbenchmark.builder.utils.path_manager import PathManager +from osbenchmark.builder.utils.template_renderer import TemplateRenderer + + +class SingularBareInstaller(Installer): + def __init__(self, config, executor, preparer): + super().__init__(executor) + self.config = config + self.preparer = preparer + self.template_renderer = TemplateRenderer() + self.path_manager = PathManager(executor) + self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager) + self.java_home_resolver = JavaHomeResolver(executor) + + def install(self, host, binaries, all_node_ips): + node = self.preparer.prepare(host, binaries) + config_vars = self.preparer.get_config_vars(host, node, all_node_ips) + self.config_applier.apply_configs(host, node, self.config.config_paths, config_vars) + self._install_invoke_hook(host, config_vars) + + return node + + def _install_invoke_hook(self, host, config_vars): + _, java_home = self.java_home_resolver.resolve_java_home(host, self.config) + + env = {} + if java_home: + env["JAVA_HOME"] = java_home + + self.preparer.invoke_install_hook(host, BootstrapPhase.post_install, config_vars.copy(), env) + + def cleanup(self, host): + self.preparer.cleanup(host) diff --git a/osbenchmark/builder/utils/java_home_resolver.py b/osbenchmark/builder/utils/java_home_resolver.py index 2235539fd..75852fce1 100644 --- a/osbenchmark/builder/utils/java_home_resolver.py +++ b/osbenchmark/builder/utils/java_home_resolver.py @@ -10,33 +10,23 @@ def __init__(self, executor): self.executor = executor self.jdk_resolver = JdkResolver(executor) - def resolve_java_home(self, host, provision_config_instance_runtime_jdks, specified_runtime_jdk=None, - provides_bundled_jdk=False): + def resolve_java_home(self, host, provision_config_instance): + is_runtime_jdk_bundled = provision_config_instance.variables["system"]["runtime"]["jdk"]["bundled"] + runtime_jdks = provision_config_instance.variables["system"]["runtime"]["jdk"]["version"] + try: - allowed_runtime_jdks = [int(v) for v in provision_config_instance_runtime_jdks.split(",")] + allowed_runtime_jdks = [int(v) for v in runtime_jdks.split(",")] except ValueError: raise SystemSetupError("ProvisionConfigInstance variable key \"runtime.jdk\" is invalid: \"{}\" (must be int)" - .format(provision_config_instance_runtime_jdks)) - - runtime_jdk_versions = self._determine_runtime_jdks(specified_runtime_jdk, allowed_runtime_jdks) - - if runtime_jdk_versions[0] == "bundled": - return self._handle_bundled_jdk(host, allowed_runtime_jdks, provides_bundled_jdk) - else: - self.logger.info("Allowed JDK versions are %s.", runtime_jdk_versions) - return self._detect_jdk(host, runtime_jdk_versions) + .format(runtime_jdks)) - def _determine_runtime_jdks(self, specified_runtime_jdk, allowed_runtime_jdks): - if specified_runtime_jdk: - return [specified_runtime_jdk] + if is_runtime_jdk_bundled: + return self._handle_bundled_jdk(host, allowed_runtime_jdks) else: - return allowed_runtime_jdks - - def _handle_bundled_jdk(self, host, allowed_runtime_jdks, provides_bundled_jdk): - if not provides_bundled_jdk: - raise SystemSetupError( - "This OpenSearch version does not contain a bundled JDK. Please specify a different runtime JDK.") + self.logger.info("Allowed JDK versions are %s.", allowed_runtime_jdks) + return self._detect_jdk(host, allowed_runtime_jdks) + def _handle_bundled_jdk(self, host, allowed_runtime_jdks): self.logger.info("Using JDK bundled with OpenSearch.") os_check = self.executor.execute(host, "uname", output=True)[0] if os_check == "Windows": diff --git a/tests/builder/installers/preparers/__init__.py b/tests/builder/installers/preparers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/builder/installers/opensearch_installer_test.py b/tests/builder/installers/preparers/opensearch_preparer_test.py similarity index 56% rename from tests/builder/installers/opensearch_installer_test.py rename to tests/builder/installers/preparers/opensearch_preparer_test.py index 169d8dcdd..e3d7c8647 100644 --- a/tests/builder/installers/opensearch_installer_test.py +++ b/tests/builder/installers/preparers/opensearch_preparer_test.py @@ -1,22 +1,24 @@ -# pylint: disable=protected-access - import os from unittest import TestCase, mock from unittest.mock import Mock -from osbenchmark.builder.installers.opensearch_installer import OpenSearchInstaller +from osbenchmark.builder.installers.preparers.opensearch_preparer import OpenSearchPreparer from osbenchmark.builder.models.host import Host +from osbenchmark.builder.models.node import Node from osbenchmark.builder.provision_config import ProvisionConfigInstance -class OpenSearchInstallerTests(TestCase): +class OpenSearchPreparerTests(TestCase): def setUp(self): + self.node_id = "abdefg" + self.node = Node(binary_path="/fake_binary_path", data_paths=["/fake1", "/fake2"], + name=self.node_id, pid=None, telemetry=None, port=9200, root_dir=None, + log_path="/fake/logpath", heap_dump_path="/fake/heap") self.host = Host(name="fake", address="10.17.22.23", metadata={}, node=None) - self.binaries = {OpenSearchInstaller.OPENSEARCH_BINARY_KEY: "/data/builds/distributions"} + self.binaries = {OpenSearchPreparer.OPENSEARCH_BINARY_KEY: "/data/builds/distributions"} self.all_node_ips = ["10.17.22.22", "10.17.22.23"] self.test_execution_root = "fake_root" - self.node_id = "abdefg" self.cluster_name = "my-cluster" self.executor = Mock() @@ -31,50 +33,41 @@ def setUp(self): "cluster_name": self.cluster_name, "node": { "port": "9200" - }, - "preserve_install": False + } } ) - self.installer = OpenSearchInstaller(self.provision_config_instance, self.executor, self.hook_handler_class) - self.installer.path_manager = Mock() - self.installer.config_applier = Mock() + self.preparer = OpenSearchPreparer(self.provision_config_instance, self.executor, self.hook_handler_class) + self.preparer.path_manager = Mock() @mock.patch("uuid.uuid4") - def test_install(self, uuid): + def test_prepare(self, uuid): # extract OpenSearch binary self.executor.execute.side_effect = [None] uuid.return_value = self.node_id - node = self.installer.install(self.host, self.binaries, self.all_node_ips) + node = self.preparer.prepare(self.host, self.binaries) self.assertEqual(node.binary_path, os.path.join(self.test_execution_root, self.node_id, "install/opensearch*")) self.assertEqual(node.data_paths, [os.path.join(self.test_execution_root, self.node_id, "install/opensearch*/data")]) self.assertEqual(node.port, 9200) self.assertEqual(node.root_dir, os.path.join(self.test_execution_root, self.node_id)) self.assertEqual(node.name, self.node_id) - @mock.patch("uuid.uuid4") - def test_config_vars(self, uuid): - # extract OpenSearch binary - self.executor.execute.side_effect = [None] - uuid.return_value = self.node_id - - node = self.installer.install(self.host, self.binaries, self.all_node_ips) - config_vars = self.installer.get_config_vars(self.host, node, self.all_node_ips) + def test_config_vars(self): + config_vars = self.preparer.get_config_vars(self.host, self.node, self.all_node_ips) self.assertEqual({ "cluster_name": self.cluster_name, "node_name": self.node_id, - "data_paths": os.path.join(self.test_execution_root, self.node_id, "install/opensearch*/data"), - "log_path": os.path.join(self.test_execution_root, self.node_id, "logs", "server"), - "heap_dump_path": os.path.join(self.test_execution_root, self.node_id, "heapdump"), + "data_paths": "/fake1", + "log_path": "/fake/logpath", + "heap_dump_path": "/fake/heap", "node_ip": "10.17.22.23", "network_host": "10.17.22.23", "http_port": "9200", "transport_port": "9300", "all_node_ips": "[\"10.17.22.22\",\"10.17.22.23\"]", "minimum_master_nodes": 2, - "install_root_path": os.path.join(self.test_execution_root, self.node_id, "install/opensearch*"), + "install_root_path": "/fake_binary_path", "node": {"port": "9200"}, - "test_execution_root": self.test_execution_root, - "preserve_install": False + "test_execution_root": self.test_execution_root }, config_vars) diff --git a/tests/builder/installers/plugin_installer_test.py b/tests/builder/installers/preparers/plugin_preparer_test.py similarity index 51% rename from tests/builder/installers/plugin_installer_test.py rename to tests/builder/installers/preparers/plugin_preparer_test.py index f44ee71d0..b16ffd29a 100644 --- a/tests/builder/installers/plugin_installer_test.py +++ b/tests/builder/installers/preparers/plugin_preparer_test.py @@ -1,13 +1,13 @@ from unittest import TestCase, mock from unittest.mock import Mock -from osbenchmark.builder.installers.plugin_installer import PluginInstaller +from osbenchmark.builder.installers.preparers.plugin_preparer import PluginPreparer from osbenchmark.builder.models.host import Host from osbenchmark.builder.models.node import Node from osbenchmark.builder.provision_config import PluginDescriptor -class PluginInstallerTest(TestCase): +class PluginPreparerTest(TestCase): def setUp(self): self.node = Node(binary_path="/fake_binary_path", data_paths=["/fake1", "/fake2"], name=None, pid=None, telemetry=None, port=None, root_dir=None, log_path=None, heap_dump_path=None) @@ -16,37 +16,26 @@ def setUp(self): self.all_node_ips = [] self.executor = Mock() + self.hook_handler_class = Mock() self.plugin = PluginDescriptor(name="unit-test-plugin", config_paths=["default"], variables={"active": True}) - self.plugin_installer = PluginInstaller(self.plugin, self.executor) - self.plugin_installer.config_applier = Mock() + self.plugin_preparer = PluginPreparer(self.plugin, self.executor, self.hook_handler_class) def test_plugin_install_with_binary_path(self): - self.plugin_installer.install(self.host, self.binaries, self.all_node_ips) + self.plugin_preparer.prepare(self.host, self.binaries) self.executor.execute.assert_has_calls([ mock.call(self.host, "/fake_binary_path/bin/opensearch-plugin install --batch \"/data/builds/distributions\"") ]) - self.plugin_installer.config_applier.apply_configs.assert_has_calls([ - mock.call(self.host, self.node, ["default"], {"active": True}) - ]) def test_plugin_install_without_binary_path(self): - self.plugin_installer.install(self.host, {}, self.all_node_ips) + self.plugin_preparer.prepare(self.host, {}) self.executor.execute.assert_has_calls([ mock.call(self.host, "/fake_binary_path/bin/opensearch-plugin install --batch \"unit-test-plugin\"") ]) - self.plugin_installer.config_applier.apply_configs.assert_has_calls([ - mock.call(self.host, self.node, ["default"], {"active": True}) - ]) - def test_config_vars_override(self): - self.plugin_installer.install(self.host, {}, self.all_node_ips, {"my": "override"}) + def test_config_vars(self): + config_vars = self.plugin_preparer.get_config_vars(self.host, self.node, self.all_node_ips) - self.executor.execute.assert_has_calls([ - mock.call(self.host, "/fake_binary_path/bin/opensearch-plugin install --batch \"unit-test-plugin\"") - ]) - self.plugin_installer.config_applier.apply_configs.assert_has_calls([ - mock.call(self.host, self.node, ["default"], {"my": "override"}) - ]) + self.assertEqual(config_vars, {"active": True}) diff --git a/tests/builder/installers/singular_bare_installer_test.py b/tests/builder/installers/singular_bare_installer_test.py new file mode 100644 index 000000000..e9d88519b --- /dev/null +++ b/tests/builder/installers/singular_bare_installer_test.py @@ -0,0 +1,71 @@ +from unittest import TestCase, mock +from unittest.mock import Mock + +from osbenchmark.builder.installers.singular_bare_installer import SingularBareInstaller +from osbenchmark.builder.models.host import Host +from osbenchmark.builder.provision_config import ProvisionConfigInstance, BootstrapPhase + + +class SingularBareInstallerTests(TestCase): + def setUp(self): + self.host = Host(name="fake", address="10.17.22.23", metadata={}, node=None) + self.binaries = {} + self.all_node_ips = ["10.17.22.22", "10.17.22.23"] + + self.test_execution_root = "fake_root" + self.node_id = "abdefg" + self.cluster_name = "my-cluster" + + self.executor = Mock() + self.preparer = Mock() + + self.provision_config_instance = ProvisionConfigInstance( + names="defaults", + root_path="fake", + config_paths=["/tmp"], + variables={ + "test_execution_root": self.test_execution_root, + "cluster_name": self.cluster_name, + "node": { + "port": "9200" + }, + "preserve_install": False + } + ) + self.installer = SingularBareInstaller(self.provision_config_instance, self.executor, self.preparer) + self.installer.config_applier = Mock() + self.installer.java_home_resolver = Mock() + + self.preparer.prepare.return_value = "fake node" + self.preparer.get_config_vars.return_value = {"fake": "config"} + + def test_install(self): + self.installer.java_home_resolver.resolve_java_home.return_value = (None, "/path/to/java/home") + + node = self.installer.install(self.host, self.binaries, self.all_node_ips) + self.assertEqual(node, "fake node") + + self.preparer.prepare.assert_has_calls([ + mock.call(self.host, self.binaries) + ]) + self.preparer.get_config_vars.assert_has_calls([ + mock.call(self.host, "fake node", self.all_node_ips) + ]) + self.installer.config_applier.apply_configs.assert_has_calls([ + mock.call(self.host, "fake node", ["/tmp"], {"fake": "config"}) + ]) + self.installer.java_home_resolver.resolve_java_home.assert_has_calls([ + mock.call(self.host, self.provision_config_instance) + ]) + self.preparer.invoke_install_hook.assert_has_calls([ + mock.call(self.host, BootstrapPhase.post_install, {"fake": "config"}, {"JAVA_HOME": "/path/to/java/home"}) + ]) + + def test_install_no_java_home(self): + self.installer.java_home_resolver.resolve_java_home.return_value = (None, None) + + self.installer.install(self.host, self.binaries, self.all_node_ips) + + self.preparer.invoke_install_hook.assert_has_calls([ + mock.call(self.host, BootstrapPhase.post_install, {"fake": "config"}, {}) + ]) diff --git a/tests/builder/utils/java_home_resolver_test.py b/tests/builder/utils/java_home_resolver_test.py index 70f49e46d..992f52875 100644 --- a/tests/builder/utils/java_home_resolver_test.py +++ b/tests/builder/utils/java_home_resolver_test.py @@ -1,6 +1,7 @@ from unittest import TestCase from unittest.mock import Mock +from osbenchmark.builder.provision_config import ProvisionConfigInstance from osbenchmark.builder.utils.java_home_resolver import JavaHomeResolver from osbenchmark.exceptions import SystemSetupError @@ -12,20 +13,32 @@ def setUp(self): self.java_home_resolver = JavaHomeResolver(self.executor) self.java_home_resolver.jdk_resolver = Mock() + self.variables = { + "system": { + "runtime": { + "jdk": { + "version": "12,11,10,9,8", + "bundled": True + } + } + } + } + self.provision_config_instance = ProvisionConfigInstance("fake_provision_config_instance", "/path/to/root", + ["/path/to/config"], variables=self.variables) + def test_resolves_java_home_for_default_runtime_jdk(self): self.executor.execute.return_value = ["Darwin"] self.java_home_resolver.jdk_resolver.resolve_jdk_path.return_value = (12, "/opt/jdk12") - major, java_home = self.java_home_resolver.resolve_java_home(self.host, "12,11,10,9,8", specified_runtime_jdk=None, - provides_bundled_jdk=True) + major, java_home = self.java_home_resolver.resolve_java_home(self.host, self.provision_config_instance) self.assertEqual(major, 12) self.assertEqual(java_home, "/opt/jdk12") def test_resolves_java_home_for_specific_runtime_jdk(self): + self.variables["system"]["runtime"]["jdk"]["version"] = "8" self.executor.execute.return_value = ["Darwin"] self.java_home_resolver.jdk_resolver.resolve_jdk_path.return_value = (8, "/opt/jdk8") - major, java_home = self.java_home_resolver.resolve_java_home(self.host, "12,11,10,9,8", specified_runtime_jdk=8, - provides_bundled_jdk=True) + major, java_home = self.java_home_resolver.resolve_java_home(self.host, self.provision_config_instance) self.assertEqual(major, 8) self.assertEqual(java_home, "/opt/jdk8") @@ -33,8 +46,7 @@ def test_resolves_java_home_for_specific_runtime_jdk(self): def test_resolves_java_home_for_bundled_jdk_on_linux(self): self.executor.execute.return_value = ["Linux"] - major, java_home = self.java_home_resolver.resolve_java_home(self.host, "12,11,10,9,8", specified_runtime_jdk="bundled", - provides_bundled_jdk=True) + major, java_home = self.java_home_resolver.resolve_java_home(self.host, self.provision_config_instance) self.assertEqual(major, 12) self.assertEqual(java_home, None) @@ -42,11 +54,5 @@ def test_resolves_java_home_for_bundled_jdk_on_linux(self): def test_resolves_java_home_for_bundled_jdk_windows(self): self.executor.execute.return_value = ["Windows"] with self.assertRaises(SystemSetupError) as ctx: - self.java_home_resolver.resolve_java_home(self.host, "12,11,10,9", specified_runtime_jdk="bundled", provides_bundled_jdk=True) + self.java_home_resolver.resolve_java_home(self.host, self.provision_config_instance) self.assertEqual("OpenSearch doesn't provide release artifacts for Windows currently.", ctx.exception.args[0]) - - def test_disallowed_bundled_jdk(self): - with self.assertRaises(SystemSetupError) as ctx: - self.java_home_resolver.resolve_java_home(self.host, "12,11,10,9,8", specified_runtime_jdk="bundled") - self.assertEqual( - "This OpenSearch version does not contain a bundled JDK. Please specify a different runtime JDK.", ctx.exception.args[0]) From e639f1bc3a16c1707685b3b9cd297829711ec2f9 Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Fri, 1 Apr 2022 17:13:04 -0500 Subject: [PATCH 6/7] Create BareInstaller Signed-off-by: Chase Engelbrecht --- .../builder/installers/bare_installer.py | 76 +++++++++++ .../preparers/opensearch_preparer.py | 3 + .../installers/preparers/plugin_preparer.py | 6 + .../builder/installers/preparers/preparer.py | 7 + .../installers/singular_bare_installer.py | 37 ------ .../builder/installers/bare_installer_test.py | 125 ++++++++++++++++++ .../singular_bare_installer_test.py | 71 ---------- 7 files changed, 217 insertions(+), 108 deletions(-) create mode 100644 osbenchmark/builder/installers/bare_installer.py delete mode 100644 osbenchmark/builder/installers/singular_bare_installer.py create mode 100644 tests/builder/installers/bare_installer_test.py delete mode 100644 tests/builder/installers/singular_bare_installer_test.py diff --git a/osbenchmark/builder/installers/bare_installer.py b/osbenchmark/builder/installers/bare_installer.py new file mode 100644 index 000000000..0d4a120be --- /dev/null +++ b/osbenchmark/builder/installers/bare_installer.py @@ -0,0 +1,76 @@ +from osbenchmark.builder.installers.installer import Installer +from osbenchmark.builder.installers.preparers.plugin_preparer import PluginPreparer +from osbenchmark.builder.provision_config import BootstrapPhase +from osbenchmark.builder.utils.config_applier import ConfigApplier +from osbenchmark.builder.utils.java_home_resolver import JavaHomeResolver +from osbenchmark.builder.utils.path_manager import PathManager +from osbenchmark.builder.utils.template_renderer import TemplateRenderer + + +class BareInstaller(Installer): + def __init__(self, provision_config_instance, executor, preparers): + super().__init__(executor) + self.provision_config_instance = provision_config_instance + if isinstance(preparers, list): + self.preparers = preparers + else: + self.preparers = [preparers] + self.template_renderer = TemplateRenderer() + self.path_manager = PathManager(executor) + self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager) + self.java_home_resolver = JavaHomeResolver(executor) + + def install(self, host, binaries, all_node_ips): + preparer_to_node = self._prepare_nodes(host, binaries) + config_vars = self._get_config_vars(host, preparer_to_node, all_node_ips) + self._apply_configs(host, preparer_to_node, config_vars) + self._invoke_install_hooks(host, config_vars) + + return self._get_node(preparer_to_node) + + def _prepare_nodes(self, host, binaries): + preparer_to_node = {} + for preparer in self.preparers: + preparer_to_node[preparer] = preparer.prepare(host, binaries) + + return preparer_to_node + + def _get_config_vars(self, host, preparer_to_node, all_node_ips): + config_vars = {} + + for preparer, node in preparer_to_node.items(): + config_vars.update(preparer.get_config_vars(host, node, all_node_ips)) + + plugin_names = [preparer.get_plugin_name() for preparer in self.preparers if isinstance(preparer, PluginPreparer)] + if plugin_names: + # as a safety measure, prevent the cluster to startup if something went wrong during plugin installation + config_vars["cluster_settings"] = {} + config_vars["cluster_settings"]["plugin.mandatory"] = plugin_names + + return config_vars + + def _apply_configs(self, host, preparer_to_node, config_vars): + for preparer, node in preparer_to_node.items(): + self.config_applier.apply_configs(host, node, preparer.get_config_paths(), config_vars) + + def _invoke_install_hooks(self, host, config_vars): + _, java_home = self.java_home_resolver.resolve_java_home(host, self.provision_config_instance) + + env = {} + if java_home: + env["JAVA_HOME"] = java_home + + config_vars_copy = config_vars.copy() + for preparer in self.preparers: + preparer.invoke_install_hook(host, BootstrapPhase.post_install, config_vars_copy, env) + + def _get_node(self, preparer_to_node): + nodes_list = list(filter(lambda node: node is not None, preparer_to_node.values())) + + assert len(nodes_list) == 1, "Exactly one node must be provisioned per host, but found nodes: {}".format(nodes_list) + + return nodes_list[0] + + def cleanup(self, host): + for preparer in self.preparers: + preparer.cleanup(host) \ No newline at end of file diff --git a/osbenchmark/builder/installers/preparers/opensearch_preparer.py b/osbenchmark/builder/installers/preparers/opensearch_preparer.py index e287dcff6..0c66b1444 100644 --- a/osbenchmark/builder/installers/preparers/opensearch_preparer.py +++ b/osbenchmark/builder/installers/preparers/opensearch_preparer.py @@ -96,6 +96,9 @@ def get_config_vars(self, host, node, all_node_ips): config_vars.update(installer_defaults) return config_vars + def get_config_paths(self): + return self.provision_config_instance.config_paths + def invoke_install_hook(self, host, phase, variables, env): self.hook_handler.invoke(phase.name, variables=variables, env=env) diff --git a/osbenchmark/builder/installers/preparers/plugin_preparer.py b/osbenchmark/builder/installers/preparers/plugin_preparer.py index 2feba06b2..1f77042c9 100644 --- a/osbenchmark/builder/installers/preparers/plugin_preparer.py +++ b/osbenchmark/builder/installers/preparers/plugin_preparer.py @@ -31,6 +31,12 @@ def _get_install_command(self, host, binaries): def get_config_vars(self, host, node, all_node_ips): return self.plugin.variables + def get_plugin_name(self): + return self.plugin.name + + def get_config_paths(self): + return self.plugin.config_paths + def invoke_install_hook(self, host, phase, variables, env): self.hook_handler.invoke(phase.name, variables=variables, env=env) diff --git a/osbenchmark/builder/installers/preparers/preparer.py b/osbenchmark/builder/installers/preparers/preparer.py index 1d88782e0..bdc93c5d2 100644 --- a/osbenchmark/builder/installers/preparers/preparer.py +++ b/osbenchmark/builder/installers/preparers/preparer.py @@ -32,6 +32,13 @@ def get_config_vars(self, host, node, all_node_ips): """ raise NotImplementedError + @abstractmethod + def get_config_paths(self): + """ + Returns the config paths list + """ + raise NotImplementedError + @abstractmethod def invoke_install_hook(self, host, phase, variables, env): """ diff --git a/osbenchmark/builder/installers/singular_bare_installer.py b/osbenchmark/builder/installers/singular_bare_installer.py deleted file mode 100644 index 1394e74c0..000000000 --- a/osbenchmark/builder/installers/singular_bare_installer.py +++ /dev/null @@ -1,37 +0,0 @@ -from osbenchmark.builder.installers.installer import Installer -from osbenchmark.builder.provision_config import BootstrapPhase -from osbenchmark.builder.utils.config_applier import ConfigApplier -from osbenchmark.builder.utils.java_home_resolver import JavaHomeResolver -from osbenchmark.builder.utils.path_manager import PathManager -from osbenchmark.builder.utils.template_renderer import TemplateRenderer - - -class SingularBareInstaller(Installer): - def __init__(self, config, executor, preparer): - super().__init__(executor) - self.config = config - self.preparer = preparer - self.template_renderer = TemplateRenderer() - self.path_manager = PathManager(executor) - self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager) - self.java_home_resolver = JavaHomeResolver(executor) - - def install(self, host, binaries, all_node_ips): - node = self.preparer.prepare(host, binaries) - config_vars = self.preparer.get_config_vars(host, node, all_node_ips) - self.config_applier.apply_configs(host, node, self.config.config_paths, config_vars) - self._install_invoke_hook(host, config_vars) - - return node - - def _install_invoke_hook(self, host, config_vars): - _, java_home = self.java_home_resolver.resolve_java_home(host, self.config) - - env = {} - if java_home: - env["JAVA_HOME"] = java_home - - self.preparer.invoke_install_hook(host, BootstrapPhase.post_install, config_vars.copy(), env) - - def cleanup(self, host): - self.preparer.cleanup(host) diff --git a/tests/builder/installers/bare_installer_test.py b/tests/builder/installers/bare_installer_test.py new file mode 100644 index 000000000..e14ee5d5f --- /dev/null +++ b/tests/builder/installers/bare_installer_test.py @@ -0,0 +1,125 @@ +from unittest import TestCase, mock +from unittest.mock import Mock + +from osbenchmark.builder.installers.bare_installer import BareInstaller +from osbenchmark.builder.installers.preparers.plugin_preparer import PluginPreparer +from osbenchmark.builder.models.host import Host +from osbenchmark.builder.provision_config import ProvisionConfigInstance, BootstrapPhase + + +class BareInstallerTests(TestCase): + def setUp(self): + self.host = Host(name="fake", address="10.17.22.23", metadata={}, node=None) + self.binaries = {} + self.all_node_ips = ["10.17.22.22", "10.17.22.23"] + + self.test_execution_root = "fake_root" + self.node_id = "abdefg" + self.cluster_name = "my-cluster" + + self.executor = Mock() + self.preparer = Mock() + self.preparer2 = Mock() + + self.provision_config_instance = ProvisionConfigInstance( + names="defaults", + root_path="fake", + config_paths=["/tmp"], + variables={ + "test_execution_root": self.test_execution_root, + "cluster_name": self.cluster_name, + "node": { + "port": "9200" + }, + "preserve_install": False + } + ) + self.installer = BareInstaller(self.provision_config_instance, self.executor, self.preparer) + self.installer.config_applier = Mock() + self.installer.java_home_resolver = Mock() + + self.preparer.prepare.return_value = "fake node" + self.preparer.get_config_vars.return_value = {"fake": "config"} + self.preparer.get_config_paths.return_value = ["/tmp"] + self.preparer2.prepare.return_value = "second node" + self.preparer2.get_config_vars.return_value = {"new": "var"} + self.preparer2.get_config_paths.return_value = ["/fake"] + self.installer.java_home_resolver.resolve_java_home.return_value = (None, "/path/to/java/home") + + def test_install_only_opensearch(self): + node = self.installer.install(self.host, self.binaries, self.all_node_ips) + self.assertEqual(node, "fake node") + + self.preparer.prepare.assert_has_calls([ + mock.call(self.host, self.binaries) + ]) + self.preparer.get_config_vars.assert_has_calls([ + mock.call(self.host, "fake node", self.all_node_ips) + ]) + self.installer.config_applier.apply_configs.assert_has_calls([ + mock.call(self.host, "fake node", ["/tmp"], {"fake": "config"}) + ]) + self.installer.java_home_resolver.resolve_java_home.assert_has_calls([ + mock.call(self.host, self.provision_config_instance) + ]) + self.preparer.invoke_install_hook.assert_has_calls([ + mock.call(self.host, BootstrapPhase.post_install, {"fake": "config"}, {"JAVA_HOME": "/path/to/java/home"}) + ]) + + def test_install_no_java_home(self): + self.installer.java_home_resolver.resolve_java_home.return_value = (None, None) + + self.installer.install(self.host, self.binaries, self.all_node_ips) + + self.preparer.invoke_install_hook.assert_has_calls([ + mock.call(self.host, BootstrapPhase.post_install, {"fake": "config"}, {}) + ]) + + def test_multiple_nodes_installed(self): + self.installer.preparers = [self.preparer, self.preparer2] + + with self.assertRaises(AssertionError): + self.installer.install(self.host, self.binaries, self.all_node_ips) + + def test_no_nodes_installed(self): + self.preparer.prepare.return_value = None + + with self.assertRaises(AssertionError): + self.installer.install(self.host, self.binaries, self.all_node_ips) + + def test_opensearch_and_plugin_installation(self): + self.preparer2.prepare.return_value = None + self.preparer2.get_plugin_name.return_value = "my-plugin" + self.preparer2.mock_add_spec(PluginPreparer) + self.installer.preparers = [self.preparer, self.preparer2] + + node = self.installer.install(self.host, self.binaries, self.all_node_ips) + self.assertEqual(node, "fake node") + + self.preparer.prepare.assert_has_calls([ + mock.call(self.host, self.binaries) + ]) + self.preparer2.prepare.assert_has_calls([ + mock.call(self.host, self.binaries) + ]) + self.preparer.get_config_vars.assert_has_calls([ + mock.call(self.host, "fake node", self.all_node_ips) + ]) + self.preparer2.get_config_vars.assert_has_calls([ + mock.call(self.host, None, self.all_node_ips) + ]) + + expected_config_vars = {"fake": "config", "new": "var", "cluster_settings": {"plugin.mandatory": ["my-plugin"]}} + self.installer.config_applier.apply_configs.assert_has_calls([ + mock.call(self.host, "fake node", ["/tmp"], expected_config_vars), + mock.call(self.host, None, ["/fake"], expected_config_vars) + ]) + self.installer.java_home_resolver.resolve_java_home.assert_has_calls([ + mock.call(self.host, self.provision_config_instance) + ]) + self.preparer.invoke_install_hook.assert_has_calls([ + mock.call(self.host, BootstrapPhase.post_install, expected_config_vars, {"JAVA_HOME": "/path/to/java/home"}) + ]) + self.preparer2.invoke_install_hook.assert_has_calls([ + mock.call(self.host, BootstrapPhase.post_install, expected_config_vars, {"JAVA_HOME": "/path/to/java/home"}) + ]) diff --git a/tests/builder/installers/singular_bare_installer_test.py b/tests/builder/installers/singular_bare_installer_test.py deleted file mode 100644 index e9d88519b..000000000 --- a/tests/builder/installers/singular_bare_installer_test.py +++ /dev/null @@ -1,71 +0,0 @@ -from unittest import TestCase, mock -from unittest.mock import Mock - -from osbenchmark.builder.installers.singular_bare_installer import SingularBareInstaller -from osbenchmark.builder.models.host import Host -from osbenchmark.builder.provision_config import ProvisionConfigInstance, BootstrapPhase - - -class SingularBareInstallerTests(TestCase): - def setUp(self): - self.host = Host(name="fake", address="10.17.22.23", metadata={}, node=None) - self.binaries = {} - self.all_node_ips = ["10.17.22.22", "10.17.22.23"] - - self.test_execution_root = "fake_root" - self.node_id = "abdefg" - self.cluster_name = "my-cluster" - - self.executor = Mock() - self.preparer = Mock() - - self.provision_config_instance = ProvisionConfigInstance( - names="defaults", - root_path="fake", - config_paths=["/tmp"], - variables={ - "test_execution_root": self.test_execution_root, - "cluster_name": self.cluster_name, - "node": { - "port": "9200" - }, - "preserve_install": False - } - ) - self.installer = SingularBareInstaller(self.provision_config_instance, self.executor, self.preparer) - self.installer.config_applier = Mock() - self.installer.java_home_resolver = Mock() - - self.preparer.prepare.return_value = "fake node" - self.preparer.get_config_vars.return_value = {"fake": "config"} - - def test_install(self): - self.installer.java_home_resolver.resolve_java_home.return_value = (None, "/path/to/java/home") - - node = self.installer.install(self.host, self.binaries, self.all_node_ips) - self.assertEqual(node, "fake node") - - self.preparer.prepare.assert_has_calls([ - mock.call(self.host, self.binaries) - ]) - self.preparer.get_config_vars.assert_has_calls([ - mock.call(self.host, "fake node", self.all_node_ips) - ]) - self.installer.config_applier.apply_configs.assert_has_calls([ - mock.call(self.host, "fake node", ["/tmp"], {"fake": "config"}) - ]) - self.installer.java_home_resolver.resolve_java_home.assert_has_calls([ - mock.call(self.host, self.provision_config_instance) - ]) - self.preparer.invoke_install_hook.assert_has_calls([ - mock.call(self.host, BootstrapPhase.post_install, {"fake": "config"}, {"JAVA_HOME": "/path/to/java/home"}) - ]) - - def test_install_no_java_home(self): - self.installer.java_home_resolver.resolve_java_home.return_value = (None, None) - - self.installer.install(self.host, self.binaries, self.all_node_ips) - - self.preparer.invoke_install_hook.assert_has_calls([ - mock.call(self.host, BootstrapPhase.post_install, {"fake": "config"}, {}) - ]) From dd56c2d338192ec9baefee7cf7a4ca2783ff4424 Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Fri, 1 Apr 2022 17:18:12 -0500 Subject: [PATCH 7/7] Add space at end of file for linter Signed-off-by: Chase Engelbrecht --- osbenchmark/builder/installers/bare_installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osbenchmark/builder/installers/bare_installer.py b/osbenchmark/builder/installers/bare_installer.py index 0d4a120be..191e272ec 100644 --- a/osbenchmark/builder/installers/bare_installer.py +++ b/osbenchmark/builder/installers/bare_installer.py @@ -73,4 +73,4 @@ def _get_node(self, preparer_to_node): def cleanup(self, host): for preparer in self.preparers: - preparer.cleanup(host) \ No newline at end of file + preparer.cleanup(host)