diff --git a/osbenchmark/builder/downloaders/repositories/__init__.py b/osbenchmark/builder/downloaders/repositories/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/osbenchmark/builder/downloaders/repositories/opensearch_distribution_repository_provider.py b/osbenchmark/builder/downloaders/repositories/opensearch_distribution_repository_provider.py new file mode 100644 index 000000000..2360b2345 --- /dev/null +++ b/osbenchmark/builder/downloaders/repositories/opensearch_distribution_repository_provider.py @@ -0,0 +1,34 @@ +import logging + +from osbenchmark.builder.downloaders.repositories.repository_url_provider import RepositoryUrlProvider +from osbenchmark.utils import convert + + +class OpenSearchDistributionRepositoryProvider: + def __init__(self, provision_config_instance, executor): + self.logger = logging.getLogger(__name__) + self.provision_config_instance = provision_config_instance + self.executor = executor + self.repository_url_provider = RepositoryUrlProvider(executor) + + def get_download_url(self, host): + is_runtime_jdk_bundled = self.provision_config_instance.variables["system"]["runtime"]["jdk"]["bundled"] + distribution_repository = self.provision_config_instance.variables["distribution"]["repository"] + + self.logger.info("runtime_jdk_bundled? [%s]", is_runtime_jdk_bundled) + if is_runtime_jdk_bundled: + url_key = "distribution.jdk.bundled.{}_url".format(distribution_repository) + else: + url_key = "distribution.jdk.unbundled.{}_url".format(distribution_repository) + + self.logger.info("key: [%s]", url_key) + return self.repository_url_provider.render_url_for_key(host, self.provision_config_instance.variables, url_key) + + def get_file_name_from_download_url(self, download_url): + return download_url[download_url.rfind("/") + 1:] + + def is_cache_enabled(self): + distribution_repository = self.provision_config_instance.variables["distribution"]["repository"] + is_cache_enabled = self.provision_config_instance.variables["distribution"][distribution_repository]["cache"] + + return convert.to_bool(is_cache_enabled) diff --git a/osbenchmark/builder/downloaders/repositories/plugin_distribution_repository_provider.py b/osbenchmark/builder/downloaders/repositories/plugin_distribution_repository_provider.py new file mode 100644 index 000000000..1a3cfe811 --- /dev/null +++ b/osbenchmark/builder/downloaders/repositories/plugin_distribution_repository_provider.py @@ -0,0 +1,13 @@ +from osbenchmark.builder.downloaders.repositories.repository_url_provider import RepositoryUrlProvider + + +class PluginDistributionRepositoryProvider: + def __init__(self, plugin, executor): + self.plugin = plugin + self.repository_url_provider = RepositoryUrlProvider(executor) + + def get_download_url(self, host): + distribution_repository = self.plugin.variables["distribution"]["repository"] + + default_key = "plugin.{}.{}.url".format(self.plugin.name, distribution_repository) + return self.repository_url_provider.render_url_for_key(host, self.plugin.variables, default_key, mandatory=False) diff --git a/osbenchmark/builder/downloaders/repositories/repository_url_provider.py b/osbenchmark/builder/downloaders/repositories/repository_url_provider.py new file mode 100644 index 000000000..996441e1b --- /dev/null +++ b/osbenchmark/builder/downloaders/repositories/repository_url_provider.py @@ -0,0 +1,43 @@ +from functools import reduce + +from osbenchmark.builder.utils.template_renderer import TemplateRenderer +from osbenchmark.exceptions import SystemSetupError + +ARCH_MAPPINGS = { + "x86_64": "x64", + "aarch64": "arm64" +} + + +class RepositoryUrlProvider: + def __init__(self, executor): + self.executor = executor + self.template_renderer = TemplateRenderer() + + def render_url_for_key(self, host, config_variables, key, mandatory=True): + try: + url_template = self._get_value_from_dot_notation_key(config_variables, key) + except TypeError: + if mandatory: + raise SystemSetupError("Config key [{}] is not defined.".format(key)) + else: + return None + return self.template_renderer.render_template_string(url_template, self._get_url_template_variables(host, config_variables)) + + def _get_value_from_dot_notation_key(self, dict_object, key): + return reduce(dict.get, key.split("."), dict_object) + + def _get_url_template_variables(self, host, config_variables): + return { + "VERSION": config_variables["distribution"]["version"], + "OSNAME": self._get_os_name(host), + "ARCH": self._get_arch(host) + } + + def _get_os_name(self, host): + os_name = self.executor.execute(host, "uname", output=True)[0] + return os_name.lower() + + def _get_arch(self, host): + arch = self.executor.execute(host, "uname -m", output=True)[0] + return ARCH_MAPPINGS[arch.lower()] diff --git a/osbenchmark/builder/installers/docker_installer.py b/osbenchmark/builder/installers/docker_installer.py index 954795d2a..b8b722449 100644 --- a/osbenchmark/builder/installers/docker_installer.py +++ b/osbenchmark/builder/installers/docker_installer.py @@ -108,7 +108,7 @@ def _add_if_defined_for_provision_config_instance(self, variables, key): def _render_template_from_docker_file(self, variables): compose_file = os.path.join(paths.benchmark_root(), "resources", "docker-compose.yml.j2") - return self.template_renderer.render_template(io.dirname(compose_file), variables, compose_file) + return self.template_renderer.render_template_file(io.dirname(compose_file), variables, compose_file) def cleanup(self, host): self.host_cleaner.cleanup(host, self.provision_config_instance.variables["preserve_install"]) diff --git a/osbenchmark/builder/utils/config_applier.py b/osbenchmark/builder/utils/config_applier.py index 79ee509c8..859b2b9ce 100644 --- a/osbenchmark/builder/utils/config_applier.py +++ b/osbenchmark/builder/utils/config_applier.py @@ -34,7 +34,7 @@ def _apply_config(self, host, source_root_path, target_root_path, config_vars): if io.is_plain_text(source_file): self.logger.info("Reading config template file [%s] and writing to [%s].", source_file, target_file) with open(target_file, mode="a", encoding="utf-8") as f: - f.write(self.template_renderer.render_template(root, config_vars, source_file)) + f.write(self.template_renderer.render_template_file(root, config_vars, source_file)) self.executor.execute(host, "cp {0} {0}".format(target_file)) else: diff --git a/osbenchmark/builder/utils/template_renderer.py b/osbenchmark/builder/utils/template_renderer.py index 73bf8c03f..ee4821a9d 100644 --- a/osbenchmark/builder/utils/template_renderer.py +++ b/osbenchmark/builder/utils/template_renderer.py @@ -6,13 +6,28 @@ class TemplateRenderer: - def render_template(self, root_path, variables, file_name): + def render_template_file(self, root_path, variables, file_name): + return self._handle_template_rendering_exceptions(self._render_template_file, root_path, variables, file_name) + + def _render_template_file(self, root_path, variables, file_name): + env = jinja2.Environment(loader=jinja2.FileSystemLoader(root_path), autoescape=select_autoescape(['html', 'xml'])) + template = env.get_template(io.basename(file_name)) + # force a new line at the end. Jinja seems to remove it. + return template.render(variables) + "\n" + + def render_template_string(self, template_string, variables): + return self._handle_template_rendering_exceptions(self._render_template_string, template_string, variables) + + def _render_template_string(self, template_string, variables): + env = jinja2.Environment(loader=jinja2.BaseLoader, autoescape=select_autoescape(['html', 'xml'])) + template = env.from_string(template_string) + + return template.render(variables) + + def _handle_template_rendering_exceptions(self, render_func, *args): try: - env = jinja2.Environment(loader=jinja2.FileSystemLoader(root_path), autoescape=select_autoescape(['html', 'xml'])) - template = env.get_template(io.basename(file_name)) - # force a new line at the end. Jinja seems to remove it. - return template.render(variables) + "\n" + return render_func(*args) except jinja2.exceptions.TemplateSyntaxError as e: - raise InvalidSyntax("%s in %s" % (str(e), file_name)) + raise InvalidSyntax("%s" % str(e)) except BaseException as e: - raise SystemSetupError("%s in %s" % (str(e), file_name)) + raise SystemSetupError("%s" % str(e)) diff --git a/tests/builder/downloaders/__init__.py b/tests/builder/downloaders/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/builder/downloaders/repositories/__init__.py b/tests/builder/downloaders/repositories/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/builder/downloaders/repositories/opensearch_distribution_repository_provider_test.py b/tests/builder/downloaders/repositories/opensearch_distribution_repository_provider_test.py new file mode 100644 index 000000000..82c3b66fd --- /dev/null +++ b/tests/builder/downloaders/repositories/opensearch_distribution_repository_provider_test.py @@ -0,0 +1,59 @@ +from unittest import TestCase, mock +from unittest.mock import Mock + +from osbenchmark.builder.downloaders.repositories.opensearch_distribution_repository_provider import \ + OpenSearchDistributionRepositoryProvider +from osbenchmark.builder.provision_config import ProvisionConfigInstance + + +class OpenSearchDistributionRepositoryProviderTest(TestCase): + def setUp(self): + self.executor = Mock() + + self.host = None + self.provision_config_instance = ProvisionConfigInstance(names=None, config_paths=None, root_path=None, variables={ + "system": { + "runtime": { + "jdk": { + "bundled": True + } + } + }, + "distribution": { + "repository": "release", + "release": { + "cache": True + } + } + }) + self.os_distro_repo_provider = OpenSearchDistributionRepositoryProvider(self.provision_config_instance, self.executor) + self.os_distro_repo_provider.repository_url_provider = Mock() + + def test_get_url_bundled_jdk(self): + self.os_distro_repo_provider.get_download_url(self.host) + self.os_distro_repo_provider.repository_url_provider.render_url_for_key.assert_has_calls([ + mock.call(None, self.provision_config_instance.variables, "distribution.jdk.bundled.release_url") + ]) + + def test_get_url_unbundled_jdk(self): + self.provision_config_instance.variables["system"]["runtime"]["jdk"]["bundled"] = False + + self.os_distro_repo_provider.get_download_url(self.host) + self.os_distro_repo_provider.repository_url_provider.render_url_for_key.assert_has_calls([ + mock.call(None, self.provision_config_instance.variables, "distribution.jdk.unbundled.release_url") + ]) + + def test_get_file_name(self): + file_name = self.os_distro_repo_provider.get_file_name_from_download_url( + "https://artifacts.opensearch.org/releases/bundle/opensearch/1.2.3/opensearch-1.2.3-linux-arm64.tar.gz") + + self.assertEqual(file_name, "opensearch-1.2.3-linux-arm64.tar.gz") + + def test_is_cache_enabled_true(self): + is_cache_enabled = self.os_distro_repo_provider.is_cache_enabled() + self.assertEqual(is_cache_enabled, True) + + def test_is_cache_enabled_false(self): + self.provision_config_instance.variables["distribution"]["release"]["cache"] = False + is_cache_enabled = self.os_distro_repo_provider.is_cache_enabled() + self.assertEqual(is_cache_enabled, False) diff --git a/tests/builder/downloaders/repositories/plugin_distribution_repository_provider_test.py b/tests/builder/downloaders/repositories/plugin_distribution_repository_provider_test.py new file mode 100644 index 000000000..cb0e52784 --- /dev/null +++ b/tests/builder/downloaders/repositories/plugin_distribution_repository_provider_test.py @@ -0,0 +1,22 @@ +from unittest import TestCase, mock +from unittest.mock import Mock + +from osbenchmark.builder.downloaders.repositories.plugin_distribution_repository_provider import \ + PluginDistributionRepositoryProvider +from osbenchmark.builder.provision_config import PluginDescriptor + + +class PluginDistributionRepositoryProviderTest(TestCase): + def setUp(self): + self.executor = Mock() + + self.host = None + self.plugin = PluginDescriptor(name="my-plugin", variables={"distribution": {"repository": "release"}}) + self.plugin_distro_repo_provider = PluginDistributionRepositoryProvider(self.plugin, self.executor) + self.plugin_distro_repo_provider.repository_url_provider = Mock() + + def test_get_plugin_url(self): + self.plugin_distro_repo_provider.get_download_url(self.host) + self.plugin_distro_repo_provider.repository_url_provider.render_url_for_key.assert_has_calls([ + mock.call(None, {"distribution": {"repository": "release"}}, "plugin.my-plugin.release.url", mandatory=False) + ]) diff --git a/tests/builder/downloaders/repositories/repository_url_provider_test.py b/tests/builder/downloaders/repositories/repository_url_provider_test.py new file mode 100644 index 000000000..5fabc71d9 --- /dev/null +++ b/tests/builder/downloaders/repositories/repository_url_provider_test.py @@ -0,0 +1,43 @@ +from unittest import TestCase +from unittest.mock import Mock + +from osbenchmark.builder.downloaders.repositories.repository_url_provider import RepositoryUrlProvider +from osbenchmark.exceptions import SystemSetupError + + +class RepositoryUrlProviderTest(TestCase): + def setUp(self): + self.executor = Mock() + + self.host = None + self.variables = { + "distribution": { + "version": "1.2.3" + }, + "fake": { + "url": "opensearch/{{VERSION}}/opensearch-{{VERSION}}-{{OSNAME}}-{{ARCH}}.tar.gz" + } + } + self.url_key = "fake.url" + + self.repo_url_provider = RepositoryUrlProvider(self.executor) + + def test_get_url_aarch64(self): + self.executor.execute.side_effect = [["Linux"], ["aarch64"]] + + url = self.repo_url_provider.render_url_for_key(self.host, self.variables, self.url_key) + self.assertEqual(url, "opensearch/1.2.3/opensearch-1.2.3-linux-arm64.tar.gz") + + def test_get_url_x86(self): + self.executor.execute.side_effect = [["Linux"], ["x86_64"]] + + url = self.repo_url_provider.render_url_for_key(self.host, self.variables, self.url_key) + self.assertEqual(url, "opensearch/1.2.3/opensearch-1.2.3-linux-x64.tar.gz") + + def test_no_url_template_found(self): + with self.assertRaises(SystemSetupError): + self.repo_url_provider.render_url_for_key(self.host, self.variables, "not.real") + + def test_no_url_template_found_not_mandatory(self): + url = self.repo_url_provider.render_url_for_key(self.host, self.variables, "not.real", False) + self.assertEqual(url, None) diff --git a/tests/builder/utils/config_applier_test.py b/tests/builder/utils/config_applier_test.py index b308f63dd..3e35f3f79 100644 --- a/tests/builder/utils/config_applier_test.py +++ b/tests/builder/utils/config_applier_test.py @@ -32,7 +32,7 @@ def test_apply_config_binary_file(self, is_plain_text, os_walk): self.path_manager.create_path.assert_has_calls([ mock.call(self.host, "/fake_binary_path/sub_fake_config_path") ]) - self.template_renderer.render_template.assert_has_calls([]) + self.template_renderer.render_template_file.assert_has_calls([]) self.executor.execute.assert_has_calls([ mock.call(self.host, "cp /fake_config_path/sub_fake_config_path/fake_file /fake_binary_path/sub_fake_config_path/fake_file") ]) @@ -52,7 +52,7 @@ def test_apply_config_plaintext_file(self, is_plain_text, os_walk): self.path_manager.create_path.assert_has_calls([ mock.call(self.host, "/fake_binary_path/sub_fake_config_path") ]) - self.template_renderer.render_template.assert_has_calls([ + self.template_renderer.render_template_file.assert_has_calls([ mock.call("/fake_config_path/sub_fake_config_path", self.config_vars, "/fake_config_path/sub_fake_config_path/fake_file") ]) self.executor.execute.assert_has_calls([ diff --git a/tests/builder/utils/template_renderer_test.py b/tests/builder/utils/template_renderer_test.py index 050746a5d..566b7f274 100644 --- a/tests/builder/utils/template_renderer_test.py +++ b/tests/builder/utils/template_renderer_test.py @@ -20,14 +20,14 @@ def test_successful_render(self, get_template): get_template.return_value = template template.render.return_value = "template as string" - self.template_renderer.render_template(self.root_path, self.variables, self.file_name) + self.template_renderer.render_template_file(self.root_path, self.variables, self.file_name) @mock.patch('jinja2.Environment.get_template') def test_template_syntax_error(self, get_template): get_template.side_effect = TemplateSyntaxError("fake", 12) with self.assertRaises(InvalidSyntax): - self.template_renderer.render_template(self.root_path, self.variables, self.file_name) + self.template_renderer.render_template_file(self.root_path, self.variables, self.file_name) @mock.patch('jinja2.Environment.get_template') def test_unknown_error(self, get_template): @@ -36,4 +36,4 @@ def test_unknown_error(self, get_template): template.render.side_effect = RuntimeError() with self.assertRaises(SystemSetupError): - self.template_renderer.render_template(self.root_path, self.variables, self.file_name) + self.template_renderer.render_template_file(self.root_path, self.variables, self.file_name)