Plugin installer migration

osbenchmark/builder/installers/bare_installer.py

@@ -0,0 +1,76 @@
+from osbenchmark.builder.installers.installer import Installer
+from osbenchmark.builder.installers.preparers.plugin_preparer import PluginPreparer
+from osbenchmark.builder.provision_config import BootstrapPhase
+from osbenchmark.builder.utils.config_applier import ConfigApplier
+from osbenchmark.builder.utils.java_home_resolver import JavaHomeResolver
+from osbenchmark.builder.utils.path_manager import PathManager
+from osbenchmark.builder.utils.template_renderer import TemplateRenderer
+
+
+class BareInstaller(Installer):
+    def __init__(self, provision_config_instance, executor, preparers):
+        super().__init__(executor)
+        self.provision_config_instance = provision_config_instance
+        if isinstance(preparers, list):
+            self.preparers = preparers
+        else:
+            self.preparers = [preparers]
+        self.template_renderer = TemplateRenderer()
+        self.path_manager = PathManager(executor)
+        self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager)
+        self.java_home_resolver = JavaHomeResolver(executor)
+
+    def install(self, host, binaries, all_node_ips):
+        preparer_to_node = self._prepare_nodes(host, binaries)
+        config_vars = self._get_config_vars(host, preparer_to_node, all_node_ips)
+        self._apply_configs(host, preparer_to_node, config_vars)
+        self._invoke_install_hooks(host, config_vars)
+
+        return self._get_node(preparer_to_node)
+
+    def _prepare_nodes(self, host, binaries):
+        preparer_to_node = {}
+        for preparer in self.preparers:
+            preparer_to_node[preparer] = preparer.prepare(host, binaries)
+
+        return preparer_to_node
+
+    def _get_config_vars(self, host, preparer_to_node, all_node_ips):
+        config_vars = {}
+
+        for preparer, node in preparer_to_node.items():
+            config_vars.update(preparer.get_config_vars(host, node, all_node_ips))
+
+        plugin_names = [preparer.get_plugin_name() for preparer in self.preparers if isinstance(preparer, PluginPreparer)]
+        if plugin_names:
+            # as a safety measure, prevent the cluster to startup if something went wrong during plugin installation
+            config_vars["cluster_settings"] = {}
+            config_vars["cluster_settings"]["plugin.mandatory"] = plugin_names
+
+        return config_vars
+
+    def _apply_configs(self, host, preparer_to_node, config_vars):
+        for preparer, node in preparer_to_node.items():
+            self.config_applier.apply_configs(host, node, preparer.get_config_paths(), config_vars)
+
+    def _invoke_install_hooks(self, host, config_vars):
+        _, java_home = self.java_home_resolver.resolve_java_home(host, self.provision_config_instance)
+
+        env = {}
+        if java_home:
+            env["JAVA_HOME"] = java_home
+
+        config_vars_copy = config_vars.copy()
+        for preparer in self.preparers:
+            preparer.invoke_install_hook(host, BootstrapPhase.post_install, config_vars_copy, env)
+
+    def _get_node(self, preparer_to_node):
+        nodes_list = list(filter(lambda node: node is not None, preparer_to_node.values()))
+
+        assert len(nodes_list) == 1, "Exactly one node must be provisioned per host, but found nodes: {}".format(nodes_list)
+
+        return nodes_list[0]
+
+    def cleanup(self, host):
+        for preparer in self.preparers:
+            preparer.cleanup(host)

osbenchmark/builder/installers/opensearch_installer.py → osbenchmark/builder/installers/preparers/opensearch_preparer.py

@@ -2,33 +2,28 @@
 import os
 import uuid
 
-from osbenchmark.builder.installers.installer import Installer
+from osbenchmark.builder.installers.preparers.preparer import Preparer
 from osbenchmark.builder.models.node import Node
-from osbenchmark.builder.provision_config import BootstrapHookHandler
-from osbenchmark.builder.utils.config_applier import ConfigApplier
 from osbenchmark.builder.utils.host_cleaner import HostCleaner
 from osbenchmark.builder.utils.path_manager import PathManager
-from osbenchmark.builder.utils.template_renderer import TemplateRenderer
 
 
-class OpenSearchInstaller(Installer):
+class OpenSearchPreparer(Preparer):
     OPENSEARCH_BINARY_KEY = "opensearch"
 
-    def __init__(self, provision_config_instance, executor, hook_handler_class=BootstrapHookHandler):
+    def __init__(self, provision_config_instance, executor, hook_handler_class):
         super().__init__(executor)
         self.logger = logging.getLogger(__name__)
         self.provision_config_instance = provision_config_instance
         self.hook_handler = hook_handler_class(self.provision_config_instance)
         if self.hook_handler.can_load():
             self.hook_handler.load()
-        self.template_renderer = TemplateRenderer()
         self.path_manager = PathManager(executor)
-        self.config_applier = ConfigApplier(executor, self.template_renderer, self.path_manager)
         self.host_cleaner = HostCleaner(self.path_manager)
 
-    def install(self, host, binaries, all_node_ips):
+    def prepare(self, host, binaries):
         node = self._create_node()
-        self._prepare_node(host, node, binaries[OpenSearchInstaller.OPENSEARCH_BINARY_KEY], all_node_ips)
+        self._prepare_node(host, node, binaries[OpenSearchPreparer.OPENSEARCH_BINARY_KEY])
 
         return node
 
@@ -50,14 +45,13 @@ def _create_node(self):
                     data_paths=None,
                     telemetry=None)
 
-    def _prepare_node(self, host, node, binary, all_node_ips):
+    def _prepare_node(self, host, node, binary):
         self._prepare_directories(host, node)
         self._extract_opensearch(host, node, binary)
         self._update_node_binary_path(node)
         self._set_node_data_paths(node)
         # we need to immediately delete the prebundled config files as plugins may copy their configuration during installation.
         self._delete_prebundled_config_files(host, node)
-        self._prepare_config_files(host, node, all_node_ips)
 
     def _prepare_directories(self, host, node):
         directories_to_create = [node.binary_path, node.log_path, node.heap_dump_path]
@@ -79,12 +73,8 @@ def _delete_prebundled_config_files(self, host, node):
         self.logger.info("Deleting pre-bundled OpenSearch configuration at [%s]", config_path)
         self.path_manager.delete_path(host, config_path)
 
-    def _prepare_config_files(self, host, node, all_node_ips):
-        config_vars = self.get_config_vars(host, node, all_node_ips)
-        self.config_applier.apply_configs(host, node, self.provision_config_instance.config_paths, config_vars)
-
     def get_config_vars(self, host, node, all_node_ips):
-        provisioner_defaults = {
+        installer_defaults = {
             "cluster_name": self.provision_config_instance.variables["cluster_name"],
             "node_name": node.name,
             "data_paths": node.data_paths[0],
@@ -103,10 +93,13 @@ def get_config_vars(self, host, node, all_node_ips):
         }
         config_vars = {}
         config_vars.update(self.provision_config_instance.variables)
-        config_vars.update(provisioner_defaults)
+        config_vars.update(installer_defaults)
         return config_vars
 
-    def invoke_install_hook(self, phase, variables, env):
+    def get_config_paths(self):
+        return self.provision_config_instance.config_paths
+
+    def invoke_install_hook(self, host, phase, variables, env):
         self.hook_handler.invoke(phase.name, variables=variables, env=env)
 
     def cleanup(self, host):

osbenchmark/builder/installers/preparers/plugin_preparer.py

@@ -0,0 +1,44 @@
+import logging
+import os
+
+from osbenchmark.builder.installers.preparers.preparer import Preparer
+
+
+class PluginPreparer(Preparer):
+    def __init__(self, plugin, executor, hook_handler_class):
+        super().__init__(executor)
+        self.logger = logging.getLogger(__name__)
+        self.plugin = plugin
+        self.hook_handler = hook_handler_class(self.plugin)
+        if self.hook_handler.can_load():
+            self.hook_handler.load()
+
+    def prepare(self, host, binaries):
+        install_cmd = self._get_install_command(host, binaries)
+        self.executor.execute(host, install_cmd)
+
+    def _get_install_command(self, host, binaries):
+        installer_binary_path = os.path.join(host.node.binary_path, "bin", "opensearch-plugin")
+        plugin_binary_path = binaries.get(self.plugin.name)
+
+        if plugin_binary_path:
+            self.logger.info("Installing [%s] into [%s] from [%s]", self.plugin.name, host.node.binary_path, plugin_binary_path)
+            return '%s install --batch "%s"' % (installer_binary_path, plugin_binary_path)
+        else:
+            self.logger.info("Installing [%s] into [%s]", self.plugin.name, host.node.binary_path)
+            return '%s install --batch "%s"' % (installer_binary_path, self.plugin.name)
+
+    def get_config_vars(self, host, node, all_node_ips):
+        return self.plugin.variables
+
+    def get_plugin_name(self):
+        return self.plugin.name
+
+    def get_config_paths(self):
+        return self.plugin.config_paths
+
+    def invoke_install_hook(self, host, phase, variables, env):
+        self.hook_handler.invoke(phase.name, variables=variables, env=env)
+
+    def cleanup(self, host):
+        pass

osbenchmark/builder/installers/preparers/preparer.py

@@ -0,0 +1,63 @@
+from abc import ABC, abstractmethod
+
+
+class Preparer(ABC):
+    """
+    A preparer is used for preparing the installation of a node by setting up the filesystem, binaries, and install hooks
+    """
+
+    def __init__(self, executor):
+        self.executor = executor
+
+    @abstractmethod
+    def prepare(self, host, binaries):
+        """
+        Prepares the filesystem and binaries on a node
+
+        ;param host: A Host object defining the host on which to prepare the data
+        ;param binaries: A map of components to download paths on the host
+        ;return node: A Node object detailing the installation data of the node on the host. May be None if no Node was generated
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_config_vars(self, host, node, all_node_ips):
+        """
+        Gets the config file(s) variables associated with the given preparer
+
+        ;param host: A Host object defining a machine within a cluster
+        ;param node: A Node object defining the node on a host
+        ;param all_node_ips: A list of the ips for each node in the cluster. Used for cluster formation
+        ;return dict: A key value pair of the config variables
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_config_paths(self):
+        """
+        Returns the config paths list
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def invoke_install_hook(self, host, phase, variables, env):
+        """
+        Invokes the associated install hook
+
+        ;param host: A Host object defining the host on which to invoke the install hook
+        ;param phase: The BoostrapPhase of install hook
+        ;param variables: Key value pairs to be passed to the install hook
+        ;param env: Key value pairs of environment variables to be passed ot the install hook
+        ;return None
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def cleanup(self, host):
+        """
+        Removes the data that was downloaded, installed, and created on a given host during the test execution
+
+        ;param host: A Host object defining the host on which to remove the data
+        ;return None
+        """
+        raise NotImplementedError

osbenchmark/builder/utils/java_home_resolver.py

@@ -10,33 +10,23 @@ def __init__(self, executor):
         self.executor = executor
         self.jdk_resolver = JdkResolver(executor)
 
-    def resolve_java_home(self, host, provision_config_instance_runtime_jdks, specified_runtime_jdk=None,
-                          provides_bundled_jdk=False):
+    def resolve_java_home(self, host, provision_config_instance):
+        is_runtime_jdk_bundled = provision_config_instance.variables["system"]["runtime"]["jdk"]["bundled"]
+        runtime_jdks = provision_config_instance.variables["system"]["runtime"]["jdk"]["version"]
+
         try:
-            allowed_runtime_jdks = [int(v) for v in provision_config_instance_runtime_jdks.split(",")]
+            allowed_runtime_jdks = [int(v) for v in runtime_jdks.split(",")]
         except ValueError:
             raise SystemSetupError("ProvisionConfigInstance variable key \"runtime.jdk\" is invalid: \"{}\" (must be int)"
-                                   .format(provision_config_instance_runtime_jdks))
-
-        runtime_jdk_versions = self._determine_runtime_jdks(specified_runtime_jdk, allowed_runtime_jdks)
-
-        if runtime_jdk_versions[0] == "bundled":
-            return self._handle_bundled_jdk(host, allowed_runtime_jdks, provides_bundled_jdk)
-        else:
-            self.logger.info("Allowed JDK versions are %s.", runtime_jdk_versions)
-            return self._detect_jdk(host, runtime_jdk_versions)
+                                   .format(runtime_jdks))
 
-    def _determine_runtime_jdks(self, specified_runtime_jdk, allowed_runtime_jdks):
-        if specified_runtime_jdk:
-            return [specified_runtime_jdk]
+        if is_runtime_jdk_bundled:
+            return self._handle_bundled_jdk(host, allowed_runtime_jdks)
         else:
-            return allowed_runtime_jdks
-
-    def _handle_bundled_jdk(self, host, allowed_runtime_jdks, provides_bundled_jdk):
-        if not provides_bundled_jdk:
-            raise SystemSetupError(
-                "This OpenSearch version does not contain a bundled JDK. Please specify a different runtime JDK.")
+            self.logger.info("Allowed JDK versions are %s.", allowed_runtime_jdks)
+            return self._detect_jdk(host, allowed_runtime_jdks)
 
+    def _handle_bundled_jdk(self, host, allowed_runtime_jdks):
         self.logger.info("Using JDK bundled with OpenSearch.")
         os_check = self.executor.execute(host, "uname", output=True)[0]
         if os_check == "Windows":