From 7aebc908495995acdf9ad73fa219eed873a82c17 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 14 Nov 2022 21:49:19 +0000 Subject: [PATCH 1/3] Resolve Window IntegTest copy and startup issues Signed-off-by: Your Name --- src/test_workflow/dependency_installer.py | 4 ++-- src/test_workflow/integ_test/distribution_zip.py | 4 ++-- .../test_integ_workflow/integ_test/test_distribution_zip.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test_workflow/dependency_installer.py b/src/test_workflow/dependency_installer.py index 55af11a437..1750456386 100644 --- a/src/test_workflow/dependency_installer.py +++ b/src/test_workflow/dependency_installer.py @@ -38,8 +38,8 @@ def download_dist(self, dest: str) -> str: return self.download_or_copy(self.bundle_manifest.build.location, local_path) def __source_dest(self, path: str, category: str, dest: str) -> Tuple[str, str]: - source = "/".join([self.root_url, category, self.build_manifest.build.filename, path]) - dest = os.path.realpath(os.path.join(dest, "/".join(path.split("/")[1:]))) + source = os.path.join(self.root_url, category, self.build_manifest.build.filename, path) + dest = os.path.realpath(os.path.join(dest, os.path.join(*path.split(os.sep)[1:]))) return (source, dest) def download(self, paths: List[str], category: str, dest: str) -> None: diff --git a/src/test_workflow/integ_test/distribution_zip.py b/src/test_workflow/integ_test/distribution_zip.py index 39089c9886..ecad250c11 100644 --- a/src/test_workflow/integ_test/distribution_zip.py +++ b/src/test_workflow/integ_test/distribution_zip.py @@ -33,8 +33,8 @@ def install(self, bundle_name: str) -> None: @property def start_cmd(self) -> str: start_cmd_map = { - "opensearch": "./opensearch-windows-install.bat", - "opensearch-dashboards": "./opensearch-dashboards.bat", + "opensearch": ".\\opensearch-windows-install.bat", + "opensearch-dashboards": ".\\opensearch-dashboards.bat", } return start_cmd_map[self.filename] diff --git a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_zip.py b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_zip.py index 538ddcd726..7506abb6a3 100644 --- a/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_zip.py +++ b/tests/tests_test_workflow/test_integ_workflow/integ_test/test_distribution_zip.py @@ -43,7 +43,7 @@ def test_install(self) -> None: mock_zipfile_extractall.assert_called_with(self.work_dir) def test_start_cmd(self) -> None: - self.assertEqual(self.distribution_zip.start_cmd, "./opensearch-windows-install.bat") + self.assertEqual(self.distribution_zip.start_cmd, ".\\opensearch-windows-install.bat") @patch("subprocess.check_call") def test_uninstall(self, check_call_mock: Mock) -> None: @@ -85,7 +85,7 @@ def test_install(self) -> None: mock_zipfile_extractall.assert_called_with(self.work_dir) def test_start_cmd(self) -> None: - self.assertEqual(self.distribution_zip.start_cmd, "./opensearch-dashboards.bat") + self.assertEqual(self.distribution_zip.start_cmd, ".\\opensearch-dashboards.bat") @patch("subprocess.check_call") def test_uninstall(self, check_call_mock: Mock) -> None: From 38b2a7192d6a08aff8a2f4d6ec076cb1343522f2 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 15 Nov 2022 01:13:19 +0000 Subject: [PATCH 2/3] Resolve http url downloading with wrong slash Signed-off-by: Your Name --- src/test_workflow/dependency_installer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/test_workflow/dependency_installer.py b/src/test_workflow/dependency_installer.py index 1750456386..4b00196bb0 100644 --- a/src/test_workflow/dependency_installer.py +++ b/src/test_workflow/dependency_installer.py @@ -11,6 +11,7 @@ import os import shutil import urllib +import pathlib from typing import List, Tuple import validators # type:ignore @@ -39,7 +40,7 @@ def download_dist(self, dest: str) -> str: def __source_dest(self, path: str, category: str, dest: str) -> Tuple[str, str]: source = os.path.join(self.root_url, category, self.build_manifest.build.filename, path) - dest = os.path.realpath(os.path.join(dest, os.path.join(*path.split(os.sep)[1:]))) + dest = os.path.realpath(os.path.join(dest, os.path.join(*pathlib.Path(path).parts[1:]))) return (source, dest) def download(self, paths: List[str], category: str, dest: str) -> None: @@ -54,9 +55,13 @@ def download(self, paths: List[str], category: str, dest: str) -> None: def download_or_copy(self, source: str, dest: str) -> str: os.makedirs(os.path.dirname(dest), exist_ok=True) - if validators.url(source): - logging.info(f"Downloading {source} into {dest} ...") - urllib.request.urlretrieve(source, dest) + # Required to make sure validators.url is checking an actual url with '/' + # else '\\' will fail the check even if it is a valid url format + # Ex: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.4.0/6452/windows/x64/zip\builds\opensearch\maven\org\opensearch\client\client-benchmarks\maven-metadata.xml + source_url = source.replace('\\', '/') + if validators.url(source_url): + logging.info(f"Downloading {source_url} into {dest} ...") + urllib.request.urlretrieve(source_url, dest) else: logging.info(f"Copying {source} into {dest} ...") source = os.path.realpath(source) From 6fb4d473cd1a3d278156fb511a556653c98711cf Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 15 Nov 2022 01:18:15 +0000 Subject: [PATCH 3/3] Resolve unittest failures on the checks linting Signed-off-by: Your Name --- src/test_workflow/dependency_installer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test_workflow/dependency_installer.py b/src/test_workflow/dependency_installer.py index 4b00196bb0..b896238611 100644 --- a/src/test_workflow/dependency_installer.py +++ b/src/test_workflow/dependency_installer.py @@ -9,9 +9,9 @@ import concurrent.futures import logging import os +import pathlib import shutil import urllib -import pathlib from typing import List, Tuple import validators # type:ignore @@ -57,7 +57,7 @@ def download_or_copy(self, source: str, dest: str) -> str: os.makedirs(os.path.dirname(dest), exist_ok=True) # Required to make sure validators.url is checking an actual url with '/' # else '\\' will fail the check even if it is a valid url format - # Ex: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.4.0/6452/windows/x64/zip\builds\opensearch\maven\org\opensearch\client\client-benchmarks\maven-metadata.xml + # Ex: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.4.0/6452/windows/x64/zip\builds\opensearch\maven\org\opensearch\client\client-benchmarks\maven-metadata.xml source_url = source.replace('\\', '/') if validators.url(source_url): logging.info(f"Downloading {source_url} into {dest} ...")