Create Repository helper classes to be consumed by the Downloaders

osbenchmark/builder/downloaders/repositories/opensearch_distribution_repository_provider.py

@@ -0,0 +1,34 @@
+import logging
+
+from osbenchmark.builder.downloaders.repositories.repository_url_provider import RepositoryUrlProvider
+from osbenchmark.utils import convert
+
+
+class OpenSearchDistributionRepositoryProvider:
+    def __init__(self, provision_config_instance, executor):
+        self.logger = logging.getLogger(__name__)
+        self.provision_config_instance = provision_config_instance
+        self.executor = executor
+        self.repository_url_provider = RepositoryUrlProvider(executor)
+
+    def get_download_url(self, host):
+        is_runtime_jdk_bundled = self.provision_config_instance.variables["system"]["runtime"]["jdk"]["bundled"]
+        distribution_repository = self.provision_config_instance.variables["distribution"]["repository"]
+
+        self.logger.info("runtime_jdk_bundled? [%s]", is_runtime_jdk_bundled)
+        if is_runtime_jdk_bundled:
+            url_key = "distribution.jdk.bundled.{}_url".format(distribution_repository)
+        else:
+            url_key = "distribution.jdk.unbundled.{}_url".format(distribution_repository)
+
+        self.logger.info("key: [%s]", url_key)
+        return self.repository_url_provider.render_url_for_key(host, self.provision_config_instance.variables, url_key)
+
+    def get_file_name_from_download_url(self, download_url):
+        return download_url[download_url.rfind("/") + 1:]
+
+    def is_cache_enabled(self):
+        distribution_repository = self.provision_config_instance.variables["distribution"]["repository"]
+        is_cache_enabled = self.provision_config_instance.variables["distribution"][distribution_repository]["cache"]
+
+        return convert.to_bool(is_cache_enabled)

osbenchmark/builder/downloaders/repositories/plugin_distribution_repository_provider.py

@@ -0,0 +1,13 @@
+from osbenchmark.builder.downloaders.repositories.repository_url_provider import RepositoryUrlProvider
+
+
+class PluginDistributionRepositoryProvider:
+    def __init__(self, plugin, executor):
+        self.plugin = plugin
+        self.repository_url_provider = RepositoryUrlProvider(executor)
+
+    def get_download_url(self, host):
+        distribution_repository = self.plugin.variables["distribution"]["repository"]
+
+        default_key = "plugin.{}.{}.url".format(self.plugin.name, distribution_repository)
+        return self.repository_url_provider.render_url_for_key(host, self.plugin.variables, default_key, mandatory=False)

osbenchmark/builder/downloaders/repositories/repository_url_provider.py

@@ -0,0 +1,43 @@
+from functools import reduce
+
+from osbenchmark.builder.utils.template_renderer import TemplateRenderer
+from osbenchmark.exceptions import SystemSetupError
+
+ARCH_MAPPINGS = {
+    "x86_64": "x64",
+    "aarch64": "arm64"
+}
+
+
+class RepositoryUrlProvider:
+    def __init__(self, executor):
+        self.executor = executor
+        self.template_renderer = TemplateRenderer()
+
+    def render_url_for_key(self, host, config_variables, key, mandatory=True):
+        try:
+            url_template = self._get_value_from_dot_notation_key(config_variables, key)
+        except TypeError:
+            if mandatory:
+                raise SystemSetupError("Config key [{}] is not defined.".format(key))
+            else:
+                return None
+        return self.template_renderer.render_template_string(url_template, self._get_url_template_variables(host, config_variables))
+
+    def _get_value_from_dot_notation_key(self, dict_object, key):
+        return reduce(dict.get, key.split("."), dict_object)
+
+    def _get_url_template_variables(self, host, config_variables):
+        return {
+            "VERSION": config_variables["distribution"]["version"],
+            "OSNAME": self._get_os_name(host),
+            "ARCH": self._get_arch(host)
+        }
+
+    def _get_os_name(self, host):
+        os_name = self.executor.execute(host, "uname", output=True)[0]
+        return os_name.lower()
+
+    def _get_arch(self, host):
+        arch = self.executor.execute(host, "uname -m", output=True)[0]
+        return ARCH_MAPPINGS[arch.lower()]

osbenchmark/builder/installers/docker_installer.py

@@ -108,7 +108,7 @@ def _add_if_defined_for_provision_config_instance(self, variables, key):
 
     def _render_template_from_docker_file(self, variables):
         compose_file = os.path.join(paths.benchmark_root(), "resources", "docker-compose.yml.j2")
-        return self.template_renderer.render_template(io.dirname(compose_file), variables, compose_file)
+        return self.template_renderer.render_template_file(io.dirname(compose_file), variables, compose_file)
 
     def cleanup(self, host):
         self.host_cleaner.cleanup(host, self.provision_config_instance.variables["preserve_install"])

osbenchmark/builder/utils/config_applier.py

@@ -34,7 +34,7 @@ def _apply_config(self, host, source_root_path, target_root_path, config_vars):
                 if io.is_plain_text(source_file):
                     self.logger.info("Reading config template file [%s] and writing to [%s].", source_file, target_file)
                     with open(target_file, mode="a", encoding="utf-8") as f:
-                        f.write(self.template_renderer.render_template(root, config_vars, source_file))
+                        f.write(self.template_renderer.render_template_file(root, config_vars, source_file))
 
                     self.executor.execute(host, "cp {0} {0}".format(target_file))
                 else:

osbenchmark/builder/utils/template_renderer.py

@@ -6,13 +6,28 @@
 
 
 class TemplateRenderer:
-    def render_template(self, root_path, variables, file_name):
+    def render_template_file(self, root_path, variables, file_name):
+        return self._handle_template_rendering_exceptions(self._render_template_file, root_path, variables, file_name)
+
+    def _render_template_file(self, root_path, variables, file_name):
+        env = jinja2.Environment(loader=jinja2.FileSystemLoader(root_path), autoescape=select_autoescape(['html', 'xml']))
+        template = env.get_template(io.basename(file_name))
+        # force a new line at the end. Jinja seems to remove it.
+        return template.render(variables) + "\n"
+
+    def render_template_string(self, template_string, variables):
+        return self._handle_template_rendering_exceptions(self._render_template_string, template_string, variables)
+
+    def _render_template_string(self, template_string, variables):
+        env = jinja2.Environment(loader=jinja2.BaseLoader, autoescape=select_autoescape(['html', 'xml']))
+        template = env.from_string(template_string)
+
+        return template.render(variables)
+
+    def _handle_template_rendering_exceptions(self, render_func, *args):
         try:
-            env = jinja2.Environment(loader=jinja2.FileSystemLoader(root_path), autoescape=select_autoescape(['html', 'xml']))
-            template = env.get_template(io.basename(file_name))
-            # force a new line at the end. Jinja seems to remove it.
-            return template.render(variables) + "\n"
+            return render_func(*args)
         except jinja2.exceptions.TemplateSyntaxError as e:
-            raise InvalidSyntax("%s in %s" % (str(e), file_name))
+            raise InvalidSyntax("%s" % str(e))
         except BaseException as e:
-            raise SystemSetupError("%s in %s" % (str(e), file_name))
+            raise SystemSetupError("%s" % str(e))

tests/builder/downloaders/repositories/opensearch_distribution_repository_provider_test.py

@@ -0,0 +1,59 @@
+from unittest import TestCase, mock
+from unittest.mock import Mock
+
+from osbenchmark.builder.downloaders.repositories.opensearch_distribution_repository_provider import \
+    OpenSearchDistributionRepositoryProvider
+from osbenchmark.builder.provision_config import ProvisionConfigInstance
+
+
+class OpenSearchDistributionRepositoryProviderTest(TestCase):
+    def setUp(self):
+        self.executor = Mock()
+
+        self.host = None
+        self.provision_config_instance = ProvisionConfigInstance(names=None, config_paths=None, root_path=None, variables={
+            "system": {
+                "runtime": {
+                    "jdk": {
+                        "bundled": True
+                    }
+                }
+            },
+            "distribution": {
+                "repository": "release",
+                "release": {
+                    "cache": True
+                }
+            }
+        })
+        self.os_distro_repo_provider = OpenSearchDistributionRepositoryProvider(self.provision_config_instance, self.executor)
+        self.os_distro_repo_provider.repository_url_provider = Mock()
+
+    def test_get_url_bundled_jdk(self):
+        self.os_distro_repo_provider.get_download_url(self.host)
+        self.os_distro_repo_provider.repository_url_provider.render_url_for_key.assert_has_calls([
+            mock.call(None, self.provision_config_instance.variables, "distribution.jdk.bundled.release_url")
+        ])
+
+    def test_get_url_unbundled_jdk(self):
+        self.provision_config_instance.variables["system"]["runtime"]["jdk"]["bundled"] = False
+
+        self.os_distro_repo_provider.get_download_url(self.host)
+        self.os_distro_repo_provider.repository_url_provider.render_url_for_key.assert_has_calls([
+            mock.call(None, self.provision_config_instance.variables, "distribution.jdk.unbundled.release_url")
+        ])
+
+    def test_get_file_name(self):
+        file_name = self.os_distro_repo_provider.get_file_name_from_download_url(
+            "https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.3/opensearch-1.2.3-linux-arm64.tar.gz")
+
+        self.assertEqual(file_name, "opensearch-1.2.3-linux-arm64.tar.gz")
+
+    def test_is_cache_enabled_true(self):
+        is_cache_enabled = self.os_distro_repo_provider.is_cache_enabled()
+        self.assertEqual(is_cache_enabled, True)
+
+    def test_is_cache_enabled_false(self):
+        self.provision_config_instance.variables["distribution"]["release"]["cache"] = False
+        is_cache_enabled = self.os_distro_repo_provider.is_cache_enabled()
+        self.assertEqual(is_cache_enabled, False)

tests/builder/downloaders/repositories/plugin_distribution_repository_provider_test.py

@@ -0,0 +1,22 @@
+from unittest import TestCase, mock
+from unittest.mock import Mock
+
+from osbenchmark.builder.downloaders.repositories.plugin_distribution_repository_provider import \
+    PluginDistributionRepositoryProvider
+from osbenchmark.builder.provision_config import PluginDescriptor
+
+
+class PluginDistributionRepositoryProviderTest(TestCase):
+    def setUp(self):
+        self.executor = Mock()
+
+        self.host = None
+        self.plugin = PluginDescriptor(name="my-plugin", variables={"distribution": {"repository": "release"}})
+        self.plugin_distro_repo_provider = PluginDistributionRepositoryProvider(self.plugin, self.executor)
+        self.plugin_distro_repo_provider.repository_url_provider = Mock()
+
+    def test_get_plugin_url(self):
+        self.plugin_distro_repo_provider.get_download_url(self.host)
+        self.plugin_distro_repo_provider.repository_url_provider.render_url_for_key.assert_has_calls([
+            mock.call(None, {"distribution": {"repository": "release"}}, "plugin.my-plugin.release.url", mandatory=False)
+        ])

tests/builder/downloaders/repositories/repository_url_provider_test.py

@@ -0,0 +1,43 @@
+from unittest import TestCase
+from unittest.mock import Mock
+
+from osbenchmark.builder.downloaders.repositories.repository_url_provider import RepositoryUrlProvider
+from osbenchmark.exceptions import SystemSetupError
+
+
+class RepositoryUrlProviderTest(TestCase):
+    def setUp(self):
+        self.executor = Mock()
+
+        self.host = None
+        self.variables = {
+            "distribution": {
+                "version": "1.2.3"
+            },
+            "fake": {
+                "url": "opensearch/{{VERSION}}/opensearch-{{VERSION}}-{{OSNAME}}-{{ARCH}}.tar.gz"
+            }
+        }
+        self.url_key = "fake.url"
+
+        self.repo_url_provider = RepositoryUrlProvider(self.executor)
+
+    def test_get_url_aarch64(self):
+        self.executor.execute.side_effect = [["Linux"], ["aarch64"]]
+
+        url = self.repo_url_provider.render_url_for_key(self.host, self.variables, self.url_key)
+        self.assertEqual(url, "opensearch/1.2.3/opensearch-1.2.3-linux-arm64.tar.gz")
+
+    def test_get_url_x86(self):
+        self.executor.execute.side_effect = [["Linux"], ["x86_64"]]
+
+        url = self.repo_url_provider.render_url_for_key(self.host, self.variables, self.url_key)
+        self.assertEqual(url, "opensearch/1.2.3/opensearch-1.2.3-linux-x64.tar.gz")
+
+    def test_no_url_template_found(self):
+        with self.assertRaises(SystemSetupError):
+            self.repo_url_provider.render_url_for_key(self.host, self.variables, "not.real")
+
+    def test_no_url_template_found_not_mandatory(self):
+        url = self.repo_url_provider.render_url_for_key(self.host, self.variables, "not.real", False)
+        self.assertEqual(url, None)

tests/builder/utils/config_applier_test.py

@@ -32,7 +32,7 @@ def test_apply_config_binary_file(self, is_plain_text, os_walk):
         self.path_manager.create_path.assert_has_calls([
             mock.call(self.host, "/fake_binary_path/sub_fake_config_path")
         ])
-        self.template_renderer.render_template.assert_has_calls([])
+        self.template_renderer.render_template_file.assert_has_calls([])
         self.executor.execute.assert_has_calls([
             mock.call(self.host, "cp /fake_config_path/sub_fake_config_path/fake_file /fake_binary_path/sub_fake_config_path/fake_file")
         ])
@@ -52,7 +52,7 @@ def test_apply_config_plaintext_file(self, is_plain_text, os_walk):
             self.path_manager.create_path.assert_has_calls([
                 mock.call(self.host, "/fake_binary_path/sub_fake_config_path")
             ])
-            self.template_renderer.render_template.assert_has_calls([
+            self.template_renderer.render_template_file.assert_has_calls([
                 mock.call("/fake_config_path/sub_fake_config_path", self.config_vars, "/fake_config_path/sub_fake_config_path/fake_file")
             ])
             self.executor.execute.assert_has_calls([

tests/builder/utils/template_renderer_test.py

@@ -20,14 +20,14 @@ def test_successful_render(self, get_template):
         get_template.return_value = template
         template.render.return_value = "template as string"
 
-        self.template_renderer.render_template(self.root_path, self.variables, self.file_name)
+        self.template_renderer.render_template_file(self.root_path, self.variables, self.file_name)
 
     @mock.patch('jinja2.Environment.get_template')
     def test_template_syntax_error(self, get_template):
         get_template.side_effect = TemplateSyntaxError("fake", 12)
 
         with self.assertRaises(InvalidSyntax):
-            self.template_renderer.render_template(self.root_path, self.variables, self.file_name)
+            self.template_renderer.render_template_file(self.root_path, self.variables, self.file_name)
 
     @mock.patch('jinja2.Environment.get_template')
     def test_unknown_error(self, get_template):
@@ -36,4 +36,4 @@ def test_unknown_error(self, get_template):
         template.render.side_effect = RuntimeError()
 
         with self.assertRaises(SystemSetupError):
-            self.template_renderer.render_template(self.root_path, self.variables, self.file_name)
+            self.template_renderer.render_template_file(self.root_path, self.variables, self.file_name)