From 22316694b89883a4bb267e0bd4bc8c54582a4481 Mon Sep 17 00:00:00 2001 From: Chase <62891993+engechas@users.noreply.github.com> Date: Wed, 6 Apr 2022 12:16:27 -0500 Subject: [PATCH] Migrate the OpenSearch and PluginDistributionDownloaders (#176) * Implement OpenSearchDistributionDownloader and unit tests Signed-off-by: Chase Engelbrecht * Implement PluginDistributionDownloader and unit tests Signed-off-by: Chase Engelbrecht --- .../opensearch_distribution_downloader.py | 66 ++++++++++++++++++ .../plugin_distribution_downloader.py | 14 ++++ ...opensearch_distribution_downloader_test.py | 67 +++++++++++++++++++ .../plugin_distribution_downloader_test.py | 28 ++++++++ 4 files changed, 175 insertions(+) create mode 100644 osbenchmark/builder/downloaders/opensearch_distribution_downloader.py create mode 100644 osbenchmark/builder/downloaders/plugin_distribution_downloader.py create mode 100644 tests/builder/downloaders/opensearch_distribution_downloader_test.py create mode 100644 tests/builder/downloaders/repositories/plugin_distribution_downloader_test.py diff --git a/osbenchmark/builder/downloaders/opensearch_distribution_downloader.py b/osbenchmark/builder/downloaders/opensearch_distribution_downloader.py new file mode 100644 index 000000000..56ec6b02f --- /dev/null +++ b/osbenchmark/builder/downloaders/opensearch_distribution_downloader.py @@ -0,0 +1,66 @@ +import logging +import os.path + +from osbenchmark.builder.downloaders.downloader import Downloader +from osbenchmark.builder.downloaders.repositories.opensearch_distribution_repository_provider import \ + OpenSearchDistributionRepositoryProvider +from osbenchmark.builder.utils.path_manager import PathManager +from osbenchmark.exceptions import ExecutorError + + +class OpenSearchDistributionDownloader(Downloader): + BINARY_KEY = "opensearch" + + def __init__(self, provision_config_instance, executor): + super().__init__(executor) + self.logger = logging.getLogger(__name__) + self.provision_config_instance = provision_config_instance + self.path_manager = PathManager(executor) + self.distribution_repository_provider = OpenSearchDistributionRepositoryProvider(provision_config_instance, executor) + + def download(self, host): + binary_path = self._fetch_binary(host) + return {OpenSearchDistributionDownloader.BINARY_KEY: binary_path} + + def _fetch_binary(self, host): + download_url = self.distribution_repository_provider.get_download_url(host) + distribution_path = self._create_distribution_path(host, download_url) + opensearch_version = self.provision_config_instance.variables["distribution"]["version"] + + is_binary_present = self._is_binary_present(host, distribution_path) + is_cache_enabled = self.distribution_repository_provider.is_cache_enabled() + + if is_binary_present and is_cache_enabled: + self.logger.info("Skipping download for version [%s]. Found existing binary at [%s].", opensearch_version, + distribution_path) + else: + self._download_opensearch(host, distribution_path, download_url, opensearch_version) + + return distribution_path + + def _create_distribution_path(self, host, download_url): + distribution_root_path = os.path.join(self.provision_config_instance.variables["node"]["root"]["dir"], "distributions") + self.path_manager.create_path(host, distribution_root_path) + + distribution_binary_name = self.distribution_repository_provider.get_file_name_from_download_url(download_url) + return os.path.join(distribution_root_path, distribution_binary_name) + + def _is_binary_present(self, host, distribution_path): + try: + self.executor.execute(host, "test -f {}".format(distribution_path)) + return True + except ExecutorError: + return False + + def _download_opensearch(self, host, distribution_path, download_url, opensearch_version): + self.logger.info("Resolved download URL [%s] for version [%s]", download_url, opensearch_version) + self.logger.info("Starting download of OpenSearch [%s]", opensearch_version) + + try: + self.executor.execute(host, "curl -o {} {}".format(distribution_path, download_url)) + except ExecutorError as e: + self.logger.exception("Exception downloading OpenSearch distribution for version [%s] from [%s].", + opensearch_version, download_url) + raise e + + self.logger.info("Successfully downloaded OpenSearch [%s].", opensearch_version) diff --git a/osbenchmark/builder/downloaders/plugin_distribution_downloader.py b/osbenchmark/builder/downloaders/plugin_distribution_downloader.py new file mode 100644 index 000000000..5b0436985 --- /dev/null +++ b/osbenchmark/builder/downloaders/plugin_distribution_downloader.py @@ -0,0 +1,14 @@ +from osbenchmark.builder.downloaders.downloader import Downloader +from osbenchmark.builder.downloaders.repositories.plugin_distribution_repository_provider import \ + PluginDistributionRepositoryProvider + + +class PluginDistributionDownloader(Downloader): + def __init__(self, plugin, executor): + super().__init__(executor) + self.plugin = plugin + self.distribution_repository_provider = PluginDistributionRepositoryProvider(plugin, executor) + + def download(self, host): + plugin_url = self.distribution_repository_provider.get_download_url(host) + return {self.plugin.name: plugin_url} if plugin_url else {} diff --git a/tests/builder/downloaders/opensearch_distribution_downloader_test.py b/tests/builder/downloaders/opensearch_distribution_downloader_test.py new file mode 100644 index 000000000..751b38367 --- /dev/null +++ b/tests/builder/downloaders/opensearch_distribution_downloader_test.py @@ -0,0 +1,67 @@ +from unittest import TestCase, mock +from unittest.mock import Mock + +from osbenchmark.builder.downloaders.opensearch_distribution_downloader import OpenSearchDistributionDownloader +from osbenchmark.builder.provision_config import ProvisionConfigInstance +from osbenchmark.exceptions import ExecutorError + + +class OpenSearchDistributionDownloaderTest(TestCase): + def setUp(self): + self.host = None + + self.executor = Mock() + self.provision_config_instance = ProvisionConfigInstance(names="fake", root_path="also fake", config_paths="fake2", variables={ + "node": { + "root": { + "dir": "/fake/dir/for/download" + } + }, + "distribution": { + "version": "1.2.3" + } + }) + + self.os_distro_downloader = OpenSearchDistributionDownloader(self.provision_config_instance, self.executor) + self.os_distro_downloader.path_manager = Mock() + self.os_distro_downloader.distribution_repository_provider = Mock() + + self.os_distro_downloader.distribution_repository_provider.get_download_url.return_value = "https://fake/download.tar.gz" + self.os_distro_downloader.distribution_repository_provider.get_file_name_from_download_url.return_value = "my-distro" + self.os_distro_downloader.distribution_repository_provider.is_cache_enabled.return_value = True + + def test_download_distro(self): + # Check if file exists, download via curl + self.executor.execute.side_effect = [ExecutorError("file doesn't exist"), None] + + binary_map = self.os_distro_downloader.download(self.host) + self.assertEqual(binary_map, {"opensearch": "/fake/dir/for/download/distributions/my-distro"}) + + self.executor.execute.assert_has_calls([ + mock.call(self.host, "test -f /fake/dir/for/download/distributions/my-distro"), + mock.call(self.host, "curl -o /fake/dir/for/download/distributions/my-distro https://fake/download.tar.gz") + ]) + + def test_download_distro_exists_and_cache_enabled(self): + # Check if file exists, download via curl + self.executor.execute.side_effect = [None] + + binary_map = self.os_distro_downloader.download(self.host) + self.assertEqual(binary_map, {"opensearch": "/fake/dir/for/download/distributions/my-distro"}) + + self.executor.execute.assert_has_calls([ + mock.call(self.host, "test -f /fake/dir/for/download/distributions/my-distro") + ]) + + def test_download_distro_exists_and_cache_disabled(self): + self.os_distro_downloader.distribution_repository_provider.is_cache_enabled.return_value = False + # Check if file exists, download via curl + self.executor.execute.side_effect = [None, None] + + binary_map = self.os_distro_downloader.download(self.host) + self.assertEqual(binary_map, {"opensearch": "/fake/dir/for/download/distributions/my-distro"}) + + self.executor.execute.assert_has_calls([ + mock.call(self.host, "test -f /fake/dir/for/download/distributions/my-distro"), + mock.call(self.host, "curl -o /fake/dir/for/download/distributions/my-distro https://fake/download.tar.gz") + ]) diff --git a/tests/builder/downloaders/repositories/plugin_distribution_downloader_test.py b/tests/builder/downloaders/repositories/plugin_distribution_downloader_test.py new file mode 100644 index 000000000..5b883970f --- /dev/null +++ b/tests/builder/downloaders/repositories/plugin_distribution_downloader_test.py @@ -0,0 +1,28 @@ +from unittest import TestCase +from unittest.mock import Mock + +from osbenchmark.builder.downloaders.plugin_distribution_downloader import PluginDistributionDownloader +from osbenchmark.builder.provision_config import PluginDescriptor + + +class PluginDistributionDownloaderTest(TestCase): + def setUp(self): + self.host = None + + self.executor = Mock() + self.plugin = PluginDescriptor(name="my plugin") + + self.plugin_distro_downloader = PluginDistributionDownloader(self.plugin, self.executor) + self.plugin_distro_downloader.distribution_repository_provider = Mock() + + def test_plugin_url_exists(self): + self.plugin_distro_downloader.distribution_repository_provider.get_download_url.return_value = "https://fake" + + binaries_map = self.plugin_distro_downloader.download(self.host) + self.assertEqual(binaries_map, {"my plugin": "https://fake"}) + + def test_plugin_url_does_not_exist(self): + self.plugin_distro_downloader.distribution_repository_provider.get_download_url.return_value = None + + binaries_map = self.plugin_distro_downloader.download(self.host) + self.assertEqual(binaries_map, {})