From 1075d9d4347685185c986712fc8dcb06e786cc4b Mon Sep 17 00:00:00 2001 From: piglei Date: Thu, 21 Mar 2024 19:29:18 +0800 Subject: [PATCH] Improve tests for bkrepo storate(with name), cover more names; Update tox.ini. --- sdks/bk-storages/pyproject.toml | 14 ++-- sdks/bk-storages/requirements-tests.txt | 5 -- sdks/bk-storages/setup.py | 54 --------------- sdks/bk-storages/tests/test_bkrepo.py | 92 +++++++++++++++---------- sdks/bk-storages/tox.ini | 28 +++----- 5 files changed, 74 insertions(+), 119 deletions(-) delete mode 100644 sdks/bk-storages/requirements-tests.txt delete mode 100644 sdks/bk-storages/setup.py diff --git a/sdks/bk-storages/pyproject.toml b/sdks/bk-storages/pyproject.toml index 4b7b207c..ba556657 100644 --- a/sdks/bk-storages/pyproject.toml +++ b/sdks/bk-storages/pyproject.toml @@ -6,15 +6,19 @@ readme = "README.md" authors = ["blueking "] license = "MIT" classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Framework :: Django", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Framework :: Django", ] [project.urls] Homepage = "https://github.com/TencentBlueKing/bkpaas-python-sdk/" Repository = "https://github.com/TencentBlueKing/bkpaas-python-sdk/" +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + [tool.poetry.dependencies] python = ">=3.6.2,<3.11" boto3 = ">=1.4.1" @@ -44,7 +48,7 @@ types-six = "0.1.7" types-toml = "0.1.3" # isort isort = "^5.9.2" -moto = {extras = ["s3"], version = "^3.1.3"} +moto = { extras = ["s3"], version = "^3.1.3" } [tool.black] line-length = 119 @@ -70,7 +74,7 @@ max-line-length = 119 max-complexity = 12 format = "pylint" # ignore example - exclude = "*.pyc,.git,__pycache__,*/dist/*" +exclude = "*.pyc,.git,__pycache__,*/dist/*" [tool.mypy] ignore_missing_imports = true diff --git a/sdks/bk-storages/requirements-tests.txt b/sdks/bk-storages/requirements-tests.txt deleted file mode 100644 index 8bc5f1f0..00000000 --- a/sdks/bk-storages/requirements-tests.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Requirements modules for running tests -pytest>=3.0.4 -pytest-django>=3.1.2 -six -boto3 diff --git a/sdks/bk-storages/setup.py b/sdks/bk-storages/setup.py deleted file mode 100644 index 9a7f78f6..00000000 --- a/sdks/bk-storages/setup.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -""" - * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-蓝鲸 PaaS 平台(BlueKing-PaaS) available. - * Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved. - * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://opensource.org/licenses/MIT - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. -""" - -# DO NOT EDIT THIS FILE! -# This file has been autogenerated by dephell <3 -# https://github.com/dephell/dephell - -try: - from setuptools import setup -except ImportError: - from distutils.core import setup - -readme = '' - -setup( - long_description=readme, - name='bkstorages', - version='1.1.0', - description='File storage backends for blueking PaaS platform', - python_requires='<3.11,>=3.6.2', - author='blueking', - author_email='blueking@tencent.com', - classifiers=['Programming Language :: Python', 'Programming Language :: Python :: 3', 'Framework :: Django'], - packages=['bkstorages', 'bkstorages.backends'], - package_dir={"": "."}, - package_data={}, - install_requires=['boto3>=1.4.1', 'curlify==2.*,>=2.2.1', 'moto[s3]==3.*,>=3.1.3', 'requests', 'six>=1.14'], - extras_require={ - "dev": [ - "black==21.7b0", - "django<3.0.0,>=1.7", - "flake8-comprehensions==3.*,>=3.5.0", - "isort==5.*,>=5.9.2", - "mypy==0.910", - "pyproject-flake8==0.*,>=0.0.1.a2", - "pytest==6.2.5", - "pytest-django==4.1.0", - "requests-mock==1.*,>=1.8.0", - "toml==0.10.1", - "types-requests==2.25.0", - "types-setuptools==57.0.0", - "types-six==0.1.7", - "types-toml==0.1.3", - ] - }, -) diff --git a/sdks/bk-storages/tests/test_bkrepo.py b/sdks/bk-storages/tests/test_bkrepo.py index 088fbef3..11075595 100644 --- a/sdks/bk-storages/tests/test_bkrepo.py +++ b/sdks/bk-storages/tests/test_bkrepo.py @@ -15,29 +15,29 @@ import pytest import requests import requests_mock +from bkstorages.backends.bkrepo import BKGenericRepoClient, BKRepoFile, BKRepoStorage, parse_gmt_datetime from django.core.files.base import File from six.moves.urllib_parse import urljoin -from bkstorages.backends.bkrepo import BKGenericRepoClient, BKRepoFile, BKRepoStorage, parse_gmt_datetime from tests.utils import generate_random_string -@pytest.fixture +@pytest.fixture() def username(): return generate_random_string() -@pytest.fixture +@pytest.fixture() def password(): return generate_random_string() -@pytest.fixture +@pytest.fixture() def endpoint(): - return "http://dummy.com" + return "http://example.com" -@pytest.fixture +@pytest.fixture() def session(): return requests.session() @@ -45,20 +45,20 @@ def session(): @pytest.fixture() def adapter(session): adapter = requests_mock.Adapter() - session.mount('http://', adapter) + session.mount("http://", adapter) return adapter -@pytest.fixture -def bk_repo_cli(session, username, password, endpoint): - bk_repo_cli = BKGenericRepoClient( +@pytest.fixture() +def bk_repo_client(session, username, password, endpoint): + bk_repo_client = BKGenericRepoClient( bucket="dummy-bucket", username=username, password=password, project="dummy-project", endpoint_url=endpoint ) - with mock.patch.object(bk_repo_cli, "get_client", lambda: session): - yield bk_repo_cli + with mock.patch.object(bk_repo_client, "get_client", return_value=session): + yield bk_repo_client -@pytest.fixture +@pytest.fixture() def make_dummy_file(adapter, endpoint): def core(key, content): def fake_upload(request): @@ -66,11 +66,11 @@ def fake_upload(request): return True g = adapter.register_uri( - 'GET', urljoin(endpoint, f'/generic/dummy-project/dummy-bucket/{key}'), body=io.BytesIO(content) + "GET", urljoin(endpoint, f"/generic/dummy-project/dummy-bucket/{key}"), body=io.BytesIO(content) ) adapter.register_uri( - 'PUT', - urljoin(endpoint, f'/generic/dummy-project/dummy-bucket/{key}'), + "PUT", + urljoin(endpoint, f"/generic/dummy-project/dummy-bucket/{key}"), json={"code": 0, "message": ""}, additional_matcher=fake_upload, ) @@ -78,10 +78,10 @@ def fake_upload(request): return core -@pytest.fixture -def bk_repo_storage(bk_repo_cli): +@pytest.fixture() +def bk_repo_storage(bk_repo_client): storage = BKRepoStorage() - storage.client = bk_repo_cli + storage.client = bk_repo_client return storage @@ -115,7 +115,7 @@ def file(name): class TestBKRepoStorage: @pytest.mark.parametrize( - "mock_responses, expect_directories, expect_files", + ("mock_responses", "expect_directories", "expect_files"), [ (([folder("d-a"), folder("d-b"), file("f-a")],), ["d-a", "d-b"], ["f-a"]), ( @@ -129,11 +129,11 @@ def test_list_dir(self, bk_repo_storage, adapter, endpoint, mock_responses, expe path = "dummy-key" for idx, records in enumerate(mock_responses): adapter.register_uri( - 'GET', + "GET", urljoin( endpoint, - f'/repository/api/node/page/dummy-project/dummy-bucket/' - f'{path}?pageSize=1000&PageNumber={idx + 1}&includeFolder=True', + f"/repository/api/node/page/dummy-project/dummy-bucket/" + f"{path}?pageSize=1000&PageNumber={idx + 1}&includeFolder=True", ), json={"code": 0, "message": 0, "data": {"totalPages": len(mock_responses), "records": records}}, ) @@ -148,30 +148,48 @@ def test_save_without_name(self, bk_repo_storage, adapter, endpoint): content = b"aaa" fh = File(io.BytesIO(content)) adapter.register_uri( - "PUT", urljoin(endpoint, f'/generic/dummy-project/dummy-bucket/{filename}'), json={"code": 0} + "PUT", urljoin(endpoint, f"/generic/dummy-project/dummy-bucket/{filename}"), json={"code": 0} ) assert bk_repo_storage.save(filename, fh) assert adapter.called assert adapter.last_request.headers["Content-Length"] == str(len(content)) assert fh.name == filename - def test_save_with_name(self, bk_repo_storage, adapter, endpoint): - key = "foo/bar" - filename = "baz.py" - content = b"aaa" - fh = File(io.BytesIO(content), name=filename) - adapter.register_uri("PUT", urljoin(endpoint, f'/generic/dummy-project/dummy-bucket/{key}'), json={"code": 0}) - assert bk_repo_storage.save(key, fh) - assert adapter.called - assert adapter.last_request.headers["Content-Length"] == str(len(content)) - assert fh.name == filename + @pytest.mark.parametrize( + ("root_path", "name", "has_error", "expected_key"), + [ + ("", "foo/bar", False, "foo/bar"), + ("", "/foo/bar", False, "foo/bar"), + ("/foo-prefix", "/bar/baz", False, "foo-prefix/bar/baz"), + # Invalid paths + ("/foo-prefix", "../../../bar", True, ""), + ], + ) + def test_save_with_name(self, root_path, name, has_error, expected_key, bk_repo_client, adapter, endpoint): + storage = BKRepoStorage(root_path=root_path) + storage.client = bk_repo_client + + filename = "foobar.py" + content = b"import this" + fh = File(io.BytesIO(content), name="foobar.py") + adapter.register_uri( + "PUT", urljoin(endpoint, f"/generic/dummy-project/dummy-bucket/{expected_key}"), json={"code": 0} + ) + if has_error: + with pytest.raises(ValueError, match="joined path"): + storage.save(name, fh) + else: + assert storage.save(name, fh) + assert adapter.called + assert adapter.last_request.headers["Content-Length"] == str(len(content)) + assert fh.name == filename @pytest.mark.parametrize( - "gmt, expected", + ("gmt", "expected"), [ - ('Fri, 03 Dec 2021 10:55:04 GMT', datetime.datetime(2021, 12, 3, 10, 55, 4)), - ('Thu, 15 Aug 2019 03:02:38 GMT', datetime.datetime(2019, 8, 15, 3, 2, 38)), + ("Fri, 03 Dec 2021 10:55:04 GMT", datetime.datetime(2021, 12, 3, 10, 55, 4)), + ("Thu, 15 Aug 2019 03:02:38 GMT", datetime.datetime(2019, 8, 15, 3, 2, 38)), ], ) def test_parse_gmt_datetime(gmt, expected): diff --git a/sdks/bk-storages/tox.ini b/sdks/bk-storages/tox.ini index 7ca871a4..89ad71b7 100644 --- a/sdks/bk-storages/tox.ini +++ b/sdks/bk-storages/tox.ini @@ -4,27 +4,19 @@ # and then run "tox" from this directory. [tox] -envlist = py{27,36}-django{18,111} +envlist = py{38}-django{111,22,32} +isolated_build = True skip_missing_interpreters = True +requires = + tox-poetry-installer[poetry] == 0.10.3 [testenv] -# Required env vars: -# RGW_ENDPOINT_URL=http://s3.com/ -# RGW_STORAGE_BUCKET_NAME=bucket_name -# RGW_ACCESS_KEY_ID=id -# RGW_SECRET_ACCESS_KEY=key -passenv = - RGW_ENDPOINT_URL - RGW_STORAGE_BUCKET_NAME - RGW_ACCESS_KEY_ID - RGW_SECRET_ACCESS_KEY +install_dev_deps = True +poetry_dep_groups = + dev deps = - pytest - pytest-django - requests - requests_mock - - django18: django==1.8 - django111: django==1.11 + django111: Django>=1.11,<1.12 + django22: Django>=2.2,<2.3 + django32: Django>=3.2,<3.3 commands = pytest -s --maxfail 1