Migrate the OpenSearch and PluginDistributionDownloaders (#176)

* Implement OpenSearchDistributionDownloader and unit tests

Signed-off-by: Chase Engelbrecht <[email protected]>

* Implement PluginDistributionDownloader and unit tests

Signed-off-by: Chase Engelbrecht <[email protected]>

osbenchmark/builder/downloaders/opensearch_distribution_downloader.py

@@ -0,0 +1,66 @@
+import logging
+import os.path
+
+from osbenchmark.builder.downloaders.downloader import Downloader
+from osbenchmark.builder.downloaders.repositories.opensearch_distribution_repository_provider import \
+    OpenSearchDistributionRepositoryProvider
+from osbenchmark.builder.utils.path_manager import PathManager
+from osbenchmark.exceptions import ExecutorError
+
+
+class OpenSearchDistributionDownloader(Downloader):
+    BINARY_KEY = "opensearch"
+
+    def __init__(self, provision_config_instance, executor):
+        super().__init__(executor)
+        self.logger = logging.getLogger(__name__)
+        self.provision_config_instance = provision_config_instance
+        self.path_manager = PathManager(executor)
+        self.distribution_repository_provider = OpenSearchDistributionRepositoryProvider(provision_config_instance, executor)
+
+    def download(self, host):
+        binary_path = self._fetch_binary(host)
+        return {OpenSearchDistributionDownloader.BINARY_KEY: binary_path}
+
+    def _fetch_binary(self, host):
+        download_url = self.distribution_repository_provider.get_download_url(host)
+        distribution_path = self._create_distribution_path(host, download_url)
+        opensearch_version = self.provision_config_instance.variables["distribution"]["version"]
+
+        is_binary_present = self._is_binary_present(host, distribution_path)
+        is_cache_enabled = self.distribution_repository_provider.is_cache_enabled()
+
+        if is_binary_present and is_cache_enabled:
+            self.logger.info("Skipping download for version [%s]. Found existing binary at [%s].", opensearch_version,
+                             distribution_path)
+        else:
+            self._download_opensearch(host, distribution_path, download_url, opensearch_version)
+
+        return distribution_path
+
+    def _create_distribution_path(self, host, download_url):
+        distribution_root_path = os.path.join(self.provision_config_instance.variables["node"]["root"]["dir"], "distributions")
+        self.path_manager.create_path(host, distribution_root_path)
+
+        distribution_binary_name = self.distribution_repository_provider.get_file_name_from_download_url(download_url)
+        return os.path.join(distribution_root_path, distribution_binary_name)
+
+    def _is_binary_present(self, host, distribution_path):
+        try:
+            self.executor.execute(host, "test -f {}".format(distribution_path))
+            return True
+        except ExecutorError:
+            return False
+
+    def _download_opensearch(self, host, distribution_path, download_url, opensearch_version):
+        self.logger.info("Resolved download URL [%s] for version [%s]", download_url, opensearch_version)
+        self.logger.info("Starting download of OpenSearch [%s]", opensearch_version)
+
+        try:
+            self.executor.execute(host, "curl -o {} {}".format(distribution_path, download_url))
+        except ExecutorError as e:
+            self.logger.exception("Exception downloading OpenSearch distribution for version [%s] from [%s].",
+                                  opensearch_version, download_url)
+            raise e
+
+        self.logger.info("Successfully downloaded OpenSearch [%s].", opensearch_version)

osbenchmark/builder/downloaders/plugin_distribution_downloader.py

@@ -0,0 +1,14 @@
+from osbenchmark.builder.downloaders.downloader import Downloader
+from osbenchmark.builder.downloaders.repositories.plugin_distribution_repository_provider import \
+    PluginDistributionRepositoryProvider
+
+
+class PluginDistributionDownloader(Downloader):
+    def __init__(self, plugin, executor):
+        super().__init__(executor)
+        self.plugin = plugin
+        self.distribution_repository_provider = PluginDistributionRepositoryProvider(plugin, executor)
+
+    def download(self, host):
+        plugin_url = self.distribution_repository_provider.get_download_url(host)
+        return {self.plugin.name: plugin_url} if plugin_url else {}

tests/builder/downloaders/opensearch_distribution_downloader_test.py

@@ -0,0 +1,67 @@
+from unittest import TestCase, mock
+from unittest.mock import Mock
+
+from osbenchmark.builder.downloaders.opensearch_distribution_downloader import OpenSearchDistributionDownloader
+from osbenchmark.builder.provision_config import ProvisionConfigInstance
+from osbenchmark.exceptions import ExecutorError
+
+
+class OpenSearchDistributionDownloaderTest(TestCase):
+    def setUp(self):
+        self.host = None
+
+        self.executor = Mock()
+        self.provision_config_instance = ProvisionConfigInstance(names="fake", root_path="also fake", config_paths="fake2", variables={
+            "node": {
+                "root": {
+                    "dir": "/fake/dir/for/download"
+                }
+            },
+            "distribution": {
+                "version": "1.2.3"
+            }
+        })
+
+        self.os_distro_downloader = OpenSearchDistributionDownloader(self.provision_config_instance, self.executor)
+        self.os_distro_downloader.path_manager = Mock()
+        self.os_distro_downloader.distribution_repository_provider = Mock()
+
+        self.os_distro_downloader.distribution_repository_provider.get_download_url.return_value = "https://fake/download.tar.gz"
+        self.os_distro_downloader.distribution_repository_provider.get_file_name_from_download_url.return_value = "my-distro"
+        self.os_distro_downloader.distribution_repository_provider.is_cache_enabled.return_value = True
+
+    def test_download_distro(self):
+        # Check if file exists, download via curl
+        self.executor.execute.side_effect = [ExecutorError("file doesn't exist"), None]
+
+        binary_map = self.os_distro_downloader.download(self.host)
+        self.assertEqual(binary_map, {"opensearch": "/fake/dir/for/download/distributions/my-distro"})
+
+        self.executor.execute.assert_has_calls([
+            mock.call(self.host, "test -f /fake/dir/for/download/distributions/my-distro"),
+            mock.call(self.host, "curl -o /fake/dir/for/download/distributions/my-distro https://fake/download.tar.gz")
+        ])
+
+    def test_download_distro_exists_and_cache_enabled(self):
+        # Check if file exists, download via curl
+        self.executor.execute.side_effect = [None]
+
+        binary_map = self.os_distro_downloader.download(self.host)
+        self.assertEqual(binary_map, {"opensearch": "/fake/dir/for/download/distributions/my-distro"})
+
+        self.executor.execute.assert_has_calls([
+            mock.call(self.host, "test -f /fake/dir/for/download/distributions/my-distro")
+        ])
+
+    def test_download_distro_exists_and_cache_disabled(self):
+        self.os_distro_downloader.distribution_repository_provider.is_cache_enabled.return_value = False
+        # Check if file exists, download via curl
+        self.executor.execute.side_effect = [None, None]
+
+        binary_map = self.os_distro_downloader.download(self.host)
+        self.assertEqual(binary_map, {"opensearch": "/fake/dir/for/download/distributions/my-distro"})
+
+        self.executor.execute.assert_has_calls([
+            mock.call(self.host, "test -f /fake/dir/for/download/distributions/my-distro"),
+            mock.call(self.host, "curl -o /fake/dir/for/download/distributions/my-distro https://fake/download.tar.gz")
+        ])

tests/builder/downloaders/repositories/plugin_distribution_downloader_test.py

@@ -0,0 +1,28 @@
+from unittest import TestCase
+from unittest.mock import Mock
+
+from osbenchmark.builder.downloaders.plugin_distribution_downloader import PluginDistributionDownloader
+from osbenchmark.builder.provision_config import PluginDescriptor
+
+
+class PluginDistributionDownloaderTest(TestCase):
+    def setUp(self):
+        self.host = None
+
+        self.executor = Mock()
+        self.plugin = PluginDescriptor(name="my plugin")
+
+        self.plugin_distro_downloader = PluginDistributionDownloader(self.plugin, self.executor)
+        self.plugin_distro_downloader.distribution_repository_provider = Mock()
+
+    def test_plugin_url_exists(self):
+        self.plugin_distro_downloader.distribution_repository_provider.get_download_url.return_value = "https://fake"
+
+        binaries_map = self.plugin_distro_downloader.download(self.host)
+        self.assertEqual(binaries_map, {"my plugin": "https://fake"})
+
+    def test_plugin_url_does_not_exist(self):
+        self.plugin_distro_downloader.distribution_repository_provider.get_download_url.return_value = None
+
+        binaries_map = self.plugin_distro_downloader.download(self.host)
+        self.assertEqual(binaries_map, {})