From bb39dd539d3dfb3c64ab4558d3cfdcd2cb4ccc72 Mon Sep 17 00:00:00 2001 From: theodu Date: Mon, 28 Aug 2023 14:24:29 +0200 Subject: [PATCH] feat: update tests to delete QueryAssets --- .../entrypoints/mutations/asset/__init__.py | 4 +- .../entrypoints/queries/label/__init__.py | 2 +- .../kili_api_gateway/asset/__init__.py | 2 +- src/kili/services/asset_import/base.py | 2 +- src/kili/services/copy_project/__init__.py | 4 +- src/kili/services/export/tools.py | 10 +- src/kili/services/helpers.py | 2 +- src/kili/use_cases/asset/__init__.py | 2 +- .../entrypoints/cli/project/test_project.py | 4 +- .../queries/test_user_facing_queries.py | 22 ++-- tests/integration/use_cases/test_asset.py | 57 +++++--- .../asset_import/test_import_common.py | 21 ++- .../asset_import/test_import_image.py | 23 ++-- .../services/asset_import/test_import_pdf.py | 17 +-- .../services/asset_import/test_import_text.py | 21 ++- .../asset_import/test_import_video.py | 13 +- tests/unit/services/export/test_export.py | 10 +- .../import_labels/test_import_from_dict.py | 12 +- .../label_data_parsing/test_label_utils.py | 124 ------------------ tests/unit/services/test_helpers.py | 6 +- 20 files changed, 116 insertions(+), 242 deletions(-) diff --git a/src/kili/entrypoints/mutations/asset/__init__.py b/src/kili/entrypoints/mutations/asset/__init__.py index 6dee82ad3..d06e6b15a 100644 --- a/src/kili/entrypoints/mutations/asset/__init__.py +++ b/src/kili/entrypoints/mutations/asset/__init__.py @@ -516,8 +516,8 @@ def verify_last_batch(last_batch: Dict, results: List) -> None: # it happens when no assets have been sent to review if isinstance(result, dict) and "id" in result: assets_in_review = self.kili_api_gateway.list_assets( - ["id"], AssetWhere(project_id=result["id"], asset_id_in=asset_ids, status_in=["TO_REVIEW"]), + ["id"], QueryOptions(disable_tqdm=True), None, ) @@ -598,8 +598,8 @@ def verify_last_batch(last_batch: Dict, results: List) -> None: result = self.format_result("data", results[0]) if isinstance(result, dict) and "id" in result: assets_in_queue = self.kili_api_gateway.list_assets( - ["id"], AssetWhere(project_id=result["id"], asset_id_in=asset_ids, status_in=["ONGOING"]), + ["id"], QueryOptions(disable_tqdm=True), None, ) diff --git a/src/kili/entrypoints/queries/label/__init__.py b/src/kili/entrypoints/queries/label/__init__.py index 9627408c2..62c11cb6e 100644 --- a/src/kili/entrypoints/queries/label/__init__.py +++ b/src/kili/entrypoints/queries/label/__init__.py @@ -640,8 +640,8 @@ def export_labels_as_df( """ services.get_project(self, project_id, ["id"]) assets_gen = self.kili_api_gateway.list_assets( - asset_fields + ["labels." + field for field in fields], AssetWhere(project_id=project_id), + asset_fields + ["labels." + field for field in fields], QueryOptions(disable_tqdm=False), None, ) diff --git a/src/kili/gateways/kili_api_gateway/asset/__init__.py b/src/kili/gateways/kili_api_gateway/asset/__init__.py index dab732d92..eabed0739 100644 --- a/src/kili/gateways/kili_api_gateway/asset/__init__.py +++ b/src/kili/gateways/kili_api_gateway/asset/__init__.py @@ -25,8 +25,8 @@ class AssetOperationMixin: def list_assets( self, - fields: List[str], where: AssetWhere, + fields: List[str], options: QueryOptions, post_call_function: Optional[Callable], ): diff --git a/src/kili/services/asset_import/base.py b/src/kili/services/asset_import/base.py index 72507f3f6..77046c9ef 100644 --- a/src/kili/services/asset_import/base.py +++ b/src/kili/services/asset_import/base.py @@ -445,8 +445,8 @@ def filter_duplicate_external_ids(self, assets): if len(assets) == 0: raise ImportValidationError("No assets to import") assets_in_project = self.kili.kili_api_gateway.list_assets( - ["externalId"], AssetWhere(project_id=self.project_params.project_id), + ["externalId"], QueryOptions(disable_tqdm=True), None, ) diff --git a/src/kili/services/copy_project/__init__.py b/src/kili/services/copy_project/__init__.py index a8cd235ed..66b47a78c 100644 --- a/src/kili/services/copy_project/__init__.py +++ b/src/kili/services/copy_project/__init__.py @@ -176,7 +176,7 @@ def download_and_upload_assets(assets): return self._upload_assets(new_project_id, downloaded_assets) asset_gen = self.kili.kili_api_gateway.list_assets( - fields, where, options, download_and_upload_assets + where, fields, options, download_and_upload_assets ) # Generator needs to be iterated over to actually fetch assets for _ in asset_gen: @@ -237,8 +237,8 @@ def _upload_assets(self, new_project_id, assets): # pylint: disable=too-many-locals def _copy_labels(self, from_project_id: str, new_project_id: str) -> None: assets_new_project = self.kili.kili_api_gateway.list_assets( - ["id", "externalId"], AssetWhere(project_id=new_project_id), + ["id", "externalId"], QueryOptions(disable_tqdm=True), None, ) diff --git a/src/kili/services/export/tools.py b/src/kili/services/export/tools.py index 0a5467991..5a2ed854b 100644 --- a/src/kili/services/export/tools.py +++ b/src/kili/services/export/tools.py @@ -4,8 +4,8 @@ import requests -from kili.core.graphql.operations.asset.queries import AssetQuery, AssetWhere from kili.core.helpers import validate_category_search_query +from kili.gateways.kili_api_gateway.asset.types import AssetWhere from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.services.export.types import ExportType from kili.use_cases.asset.media_downloader import get_download_assets_function @@ -139,7 +139,7 @@ def fetch_assets( where = AssetWhere(**asset_where_params) if download_media: - count = AssetQuery(kili.graphql_client, kili.http_client).count(where) + count = kili.kili_api_gateway.count_assets(where) if count > THRESHOLD_WARN_MANY_ASSETS: warnings.warn( f"Downloading many assets ({count}). This might take a while. Consider" @@ -151,11 +151,7 @@ def fetch_assets( post_call_function, fields = get_download_assets_function( kili.kili_api_gateway, download_media, fields, project_id, local_media_dir ) - assets = list( - AssetQuery(kili.graphql_client, kili.http_client)( - where, fields, options, post_call_function - ) - ) + assets = list(kili.kili_api_gateway.list_assets(where, fields, options, post_call_function)) attach_name_to_assets_labels_author(assets, export_type) return assets diff --git a/src/kili/services/helpers.py b/src/kili/services/helpers.py index d30321435..1defdc57b 100644 --- a/src/kili/services/helpers.py +++ b/src/kili/services/helpers.py @@ -93,8 +93,8 @@ def _build_id_map(kili: Kili, asset_external_ids, project_id): for external_ids_batch in pagination.BatchIteratorBuilder(asset_external_ids, 1000): assets_generators.append( kili.kili_api_gateway.list_assets( - ["id", "externalId"], AssetWhere(project_id, external_id_strictly_in=external_ids_batch), + ["id", "externalId"], QueryOptions(disable_tqdm=True), None, ) diff --git a/src/kili/use_cases/asset/__init__.py b/src/kili/use_cases/asset/__init__.py index 451611ea0..361f2b2d3 100644 --- a/src/kili/use_cases/asset/__init__.py +++ b/src/kili/use_cases/asset/__init__.py @@ -36,7 +36,7 @@ def list_assets( post_call_function, fields = get_download_assets_function( self._kili_api_gateway, download_media, fields, where.project_id, local_media_dir ) - assets_gen = self._kili_api_gateway.list_assets(fields, where, options, post_call_function) + assets_gen = self._kili_api_gateway.list_assets(where, fields, options, post_call_function) if label_output_format == "parsed_label": project: LabelParsingProject = self._kili_api_gateway.get_project( diff --git a/tests/integration/entrypoints/cli/project/test_project.py b/tests/integration/entrypoints/cli/project/test_project.py index 2d84d663b..14c2a8e20 100644 --- a/tests/integration/entrypoints/cli/project/test_project.py +++ b/tests/integration/entrypoints/cli/project/test_project.py @@ -10,13 +10,13 @@ import pytest_mock from click.testing import CliRunner -from kili.core.graphql.operations.asset.queries import AssetQuery from kili.core.graphql.operations.project.queries import ProjectQuery from kili.entrypoints.cli.project.create import create_project from kili.entrypoints.cli.project.describe import describe_project from kili.entrypoints.cli.project.export import export_labels from kili.entrypoints.cli.project.import_ import import_assets from kili.entrypoints.cli.project.list_ import list_projects +from kili.gateways.kili_api_gateway.asset import AssetOperationMixin from tests.integration.entrypoints.cli.helpers import debug_subprocess_pytest from .mocks.assets import mocked__project_assets @@ -454,7 +454,7 @@ def test_import( def test_export(name: str, test_case: List[str], mocker: pytest_mock.MockerFixture): mocker.patch.dict("os.environ", {"KILI_API_KEY": "toto", "KILI_SDK_SKIP_CHECKS": "True"}) mocker.patch.object(ProjectQuery, "__call__", side_effect=mocked__ProjectQuery) - mocker.patch.object(AssetQuery, "__call__", side_effect=mocked__project_assets) + mocker.patch.object(AssetOperationMixin, "list_assets", side_effect=mocked__project_assets) mocker.patch( "kili.services.export.format.base.AbstractExporter._has_data_connection", return_value=False ) diff --git a/tests/integration/entrypoints/client/queries/test_user_facing_queries.py b/tests/integration/entrypoints/client/queries/test_user_facing_queries.py index ea25b7344..eabe94af5 100644 --- a/tests/integration/entrypoints/client/queries/test_user_facing_queries.py +++ b/tests/integration/entrypoints/client/queries/test_user_facing_queries.py @@ -2,15 +2,14 @@ from typing import Dict, Generator, List -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from typeguard import check_type -from kili.core.graphql.operations.asset.queries import AssetQuery from kili.core.graphql.operations.user.queries import UserQuery -from kili.entrypoints.queries.asset import QueriesAsset from kili.entrypoints.queries.user import QueriesUser +from kili.presentation.client.asset import AssetClientMethods @pytest.mark.parametrize( @@ -38,16 +37,19 @@ def test_users_query_return_type(mocker, args, kwargs, expected_return_type): (("project-id",), {}, List[Dict]), (("project-id",), {"as_generator": False}, List[Dict]), (("project-id",), {"as_generator": True}, Generator[Dict, None, None]), + (("project-id",), {"label_output_format": "parsed_label"}, List[Dict]), + ( + ("project-id",), + {"label_output_format": "parsed_label", "as_generator": True}, + Generator[Dict, None, None], + ), ((), {"project_id": "project-id"}, List[Dict]), ((), {"project_id": "project-id", "as_generator": True}, Generator[Dict, None, None]), ((), {"project_id": "project-id", "as_generator": False}, List[Dict]), ], ) -@patch.object(AssetQuery, "__call__") -def test_assets_query_return_type(mocker, args, kwargs, expected_return_type): - kili = QueriesAsset() - kili.graphql_client = mocker.MagicMock() - kili.http_client = mocker.MagicMock() - - result = kili.assets(*args, **kwargs) +def test_assets_query_return_type(kili_api_gateway, args, kwargs, expected_return_type): + asset_client_methods = AssetClientMethods() + asset_client_methods.kili_api_gateway = MagicMock() + result = asset_client_methods.assets(*args, **kwargs) assert check_type("result", result, expected_return_type) is None diff --git a/tests/integration/use_cases/test_asset.py b/tests/integration/use_cases/test_asset.py index b07a79e7e..6e88b82a7 100644 --- a/tests/integration/use_cases/test_asset.py +++ b/tests/integration/use_cases/test_asset.py @@ -38,25 +38,29 @@ def test_given_query_parameters_I_can_query_assets_and_get_their_labels_parsed( kili_api_gateway: KiliAPIGateway, ): # mocking - json_response = {"JOB_0": {"categories": [{"name": "CATGORY_A"}]}} + label = { + "author": { + "id": "cldbnzmmq00go0jwc20fq1jkl", + "email": "john.doe@kili-technology.com", + }, + "createdAt": "2023-05-11T16:01:48.093Z", + "id": "clhjbhrul015m0k7hct21drz4", + "jsonResponse": {"JOB_0": {"text": "some text abc"}}, + } asset = { + "labels": [label], + "latestLabel": label, + "content": "https://storage.googleapis.com/label-backend-staging/", + "createdAt": "2023-05-11T15:55:01.134Z", + "externalId": "4bad2303e43bfefa0169d890c68f5c9d--cherry-blossom-tree-blossom-trees.jpg", "id": "asset_id", - "labels": [{"jsonResponse": json_response}], - "latestLabel": {"jsonResponse": json_response}, + "isHoneypot": False, + "jsonMetadata": {}, + "skipped": False, + "status": "LABELED", } json_interface = { - "jobs": { - "JOB_0": { - "mlTask": "CLASSIFICATION", - "isChild": False, - "content": { - "categories": { - "CATGORY_A": {"children": [], "name": "category A", "id": "category30"}, - }, - "input": "checkbox", - }, - } - } + "jobs": {"JOB_0": {"mlTask": "TRANSCRIPTION", "required": 1, "isChild": False}} } kili_api_gateway.list_assets.return_value = (asset for asset in [asset]) kili_api_gateway.get_project.return_value = { @@ -67,7 +71,22 @@ def test_given_query_parameters_I_can_query_assets_and_get_their_labels_parsed( # given parameters to query assets asset_use_cases = AssetUseCases(kili_api_gateway) where = AssetWhere(project_id="project_id") - fields = ["id", "label.jsonResponse", "latestLabel.jsonresponse"] + fields = [ + "content", + "createdAt", + "externalId", + "id", + "isHoneypot", + "jsonMetadata", + "labels.author.id", + "labels.author.email", + "labels.createdAt", + "labels.id", + "labels.jsonResponse", + "skipped", + "status", + "latestLabel.jsonResponse", + ] options = QueryOptions(disable_tqdm=False) # when creating query assets @@ -79,12 +98,16 @@ def test_given_query_parameters_I_can_query_assets_and_get_their_labels_parsed( local_media_dir=None, label_output_format="parsed_label", ) - returned_asset = list(asset_gen)[0] # then + returned_assets = list(asset_gen) + assert len(returned_assets) == 1 + returned_asset = returned_assets[0] assert returned_asset == asset assert isinstance(returned_asset["latestLabel"], ParsedLabel) + assert returned_asset["latestLabel"].jobs["JOB_0"].text == "some text abc" assert isinstance(returned_asset["labels"][0], ParsedLabel) + assert returned_asset["labels"][0].jobs["JOB_0"].text == "some text abc" def test_given_query_parameters_I_can_query_assets_and_download_their_media( diff --git a/tests/unit/services/asset_import/test_import_common.py b/tests/unit/services/asset_import/test_import_common.py index 221f8cfb4..62fe55735 100644 --- a/tests/unit/services/asset_import/test_import_common.py +++ b/tests/unit/services/asset_import/test_import_common.py @@ -2,10 +2,9 @@ from uuid import UUID from kili.core.graphql.operations.asset.mutations import GQL_APPEND_MANY_ASSETS -from kili.core.graphql.operations.asset.queries import AssetQuery from kili.core.graphql.operations.organization.queries import OrganizationQuery from kili.core.graphql.operations.project.queries import ProjectQuery -from kili.orm import Asset +from kili.gateways.kili_api_gateway.asset import AssetOperationMixin from kili.services.asset_import import import_assets from kili.services.asset_import.exceptions import MimeTypeError from tests.unit.services.asset_import.base import ImportTestCase @@ -21,11 +20,7 @@ @patch("kili.utils.bucket.generate_unique_id", mocked_unique_id) @patch("kili.utils.bucket.request_signed_urls", mocked_request_signed_urls) @patch("kili.utils.bucket.upload_data_via_rest", mocked_upload_data_via_rest) -@patch.object( - AssetQuery, - "__call__", - return_value=[], -) +@patch.object(AssetOperationMixin, "list_assets", return_value=[]) @patch.object( OrganizationQuery, "__call__", @@ -33,7 +28,7 @@ ) class TestContentType(ImportTestCase): @patch.object(ProjectQuery, "__call__", side_effect=mocked_project_input_type("VIDEO_LEGACY")) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_cannot_upload_an_image_to_video_project(self, *_): url = "https://storage.googleapis.com/label-public-staging/car/car_1.jpg" path_image = self.downloader(url) @@ -42,7 +37,7 @@ def test_cannot_upload_an_image_to_video_project(self, *_): import_assets(self.kili, self.project_id, assets, disable_tqdm=True) @patch.object(ProjectQuery, "__call__", side_effect=mocked_project_input_type("IMAGE")) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_cannot_import_files_not_found_to_an_image_project(self, *_): path = "./doesnotexist.png" assets = [{"content": path, "external_id": "image"}] @@ -50,7 +45,7 @@ def test_cannot_import_files_not_found_to_an_image_project(self, *_): import_assets(self.kili, self.project_id, assets, disable_tqdm=True) @patch.object(ProjectQuery, "__call__", side_effect=mocked_project_input_type("PDF")) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_cannot_upload_raw_text_to_pdf_project(self, *_): path = "Hello world" assets = [{"content": path, "external_id": "image"}] @@ -58,7 +53,7 @@ def test_cannot_upload_raw_text_to_pdf_project(self, *_): import_assets(self.kili, self.project_id, assets, disable_tqdm=True) @patch.object(ProjectQuery, "__call__", side_effect=mocked_project_input_type("TEXT")) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_return_the_ids_of_created_assets(self, *_): assets = [{"content": "One"}, {"content": "Two"}, {"content": "Three"}] @@ -73,7 +68,7 @@ def graphql_execute_side_effect(*args, **kwargs): assert set(created_assets) == {"id1", "id2", "id3"} @patch.object(ProjectQuery, "__call__", side_effect=mocked_project_input_type("TEXT")) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_generate_different_uuid4_external_ids_if_not_given(self, *_): assets = [{"content": "One"}, {"content": "Two"}, {"content": "Three"}] self.kili.graphql_client.execute.reset_mock() @@ -92,7 +87,7 @@ def test_generate_different_uuid4_external_ids_if_not_given(self, *_): assert external_ids_are_uniques @patch.object(ProjectQuery, "__call__", side_effect=mocked_project_input_type("IMAGE")) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) @patch("kili.services.asset_import.base.BaseBatchImporter.verify_batch_imported") def test_import_assets_verify(self, mocked_verify_batch_imported, *_): assets = [{"content": "https://hosted-data", "external_id": "externalid"}] diff --git a/tests/unit/services/asset_import/test_import_image.py b/tests/unit/services/asset_import/test_import_image.py index e5547a9d5..ed7f08a65 100644 --- a/tests/unit/services/asset_import/test_import_image.py +++ b/tests/unit/services/asset_import/test_import_image.py @@ -2,10 +2,9 @@ import pytest -from kili.core.graphql.operations.asset.queries import AssetQuery from kili.core.graphql.operations.organization.queries import OrganizationQuery from kili.core.graphql.operations.project.queries import ProjectQuery -from kili.entrypoints.queries.asset import QueriesAsset +from kili.gateways.kili_api_gateway.asset import AssetOperationMixin from kili.services.asset_import import import_assets from kili.services.asset_import.exceptions import UploadFromLocalDataForbiddenError from tests.unit.services.asset_import.base import ImportTestCase @@ -22,18 +21,14 @@ @patch("kili.utils.bucket.upload_data_via_rest", mocked_upload_data_via_rest) @patch("kili.utils.bucket.generate_unique_id", mocked_unique_id) @patch.object(ProjectQuery, "__call__", side_effect=mocked_project_input_type("IMAGE")) -@patch.object( - QueriesAsset, - "assets", - MagicMock(return_value=[]), -) +@patch.object(AssetOperationMixin, "list_assets", MagicMock(return_value=[])) @patch.object( OrganizationQuery, "__call__", side_effect=mocked_organization_with_upload_from_local(upload_local_data=True), ) class ImageTestCase(ImportTestCase): - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_one_local_image(self, *_): url = "https://storage.googleapis.com/label-public-staging/car/car_1.jpg" path_image = self.downloader(url) @@ -49,7 +44,7 @@ def test_upload_from_one_local_image(self, *_): ) self.kili.graphql_client.execute.assert_called_with(*expected_parameters) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_one_hosted_image(self, *_): assets = [ {"content": "https://hosted-data", "external_id": "hosted file", "id": "unique_id"} @@ -60,7 +55,7 @@ def test_upload_from_one_hosted_image(self, *_): ) self.kili.graphql_client.execute.assert_called_with(*expected_parameters) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_one_local_tiff_image(self, *_): url = "https://storage.googleapis.com/label-public-staging/geotiffs/bogota.tif" path_image = self.downloader(url) @@ -75,7 +70,9 @@ def test_upload_from_one_local_tiff_image(self, *_): ) self.kili.graphql_client.execute.assert_called_with(*expected_parameters) - @patch.object(AssetQuery, "count", return_value=1) # 2 images are uploaded in different batches + @patch.object( + AssetOperationMixin, "count_assets", return_value=1 + ) # 2 images are uploaded in different batches def test_upload_with_one_tiff_and_one_basic_image(self, *_): url_tiff = "https://storage.googleapis.com/label-public-staging/geotiffs/bogota.tif" url_basic = "https://storage.googleapis.com/label-public-staging/car/car_1.jpg" @@ -104,11 +101,11 @@ def test_upload_with_one_tiff_and_one_basic_image(self, *_): calls = [call(*expected_parameters_sync), call(*expected_parameters_async)] self.kili.graphql_client.execute.assert_has_calls(calls, any_order=True) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_several_batches(self, *_): self.assert_upload_several_batches() - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_one_hosted_image_authorized_while_local_forbidden(self, *_): OrganizationQuery.__call__.side_effect = mocked_organization_with_upload_from_local( upload_local_data=False diff --git a/tests/unit/services/asset_import/test_import_pdf.py b/tests/unit/services/asset_import/test_import_pdf.py index 188c79982..0be8e7b62 100644 --- a/tests/unit/services/asset_import/test_import_pdf.py +++ b/tests/unit/services/asset_import/test_import_pdf.py @@ -2,10 +2,9 @@ import pytest -from kili.core.graphql.operations.asset.queries import AssetQuery from kili.core.graphql.operations.organization.queries import OrganizationQuery from kili.core.graphql.operations.project.queries import ProjectQuery -from kili.entrypoints.queries.asset import QueriesAsset +from kili.gateways.kili_api_gateway.asset import AssetOperationMixin from kili.services.asset_import import import_assets from kili.services.asset_import.exceptions import UploadFromLocalDataForbiddenError from tests.unit.services.asset_import.base import ImportTestCase @@ -22,18 +21,14 @@ @patch("kili.utils.bucket.upload_data_via_rest", mocked_upload_data_via_rest) @patch("kili.utils.bucket.generate_unique_id", mocked_unique_id) @patch.object(ProjectQuery, "__call__", side_effect=mocked_project_input_type("PDF")) -@patch.object( - QueriesAsset, - "assets", - MagicMock(return_value=[]), -) +@patch.object(AssetOperationMixin, "list_assets", MagicMock(return_value=[])) @patch.object( OrganizationQuery, "__call__", side_effect=mocked_organization_with_upload_from_local(upload_local_data=True), ) class PDFTestCase(ImportTestCase): - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_one_local_pdf(self, *_): url = ( "https://storage.googleapis.com/label-public-staging/asset-test-sample/pdfs/sample.pdf" @@ -51,7 +46,7 @@ def test_upload_from_one_local_pdf(self, *_): ) self.kili.graphql_client.execute.assert_called_with(*expected_parameters) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_one_hosted_pdf(self, *_): assets = [ {"content": "https://hosted-data", "external_id": "hosted file", "id": "unique_id"} @@ -62,11 +57,11 @@ def test_upload_from_one_hosted_pdf(self, *_): ) self.kili.graphql_client.execute.assert_called_with(*expected_parameters) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_several_batches(self, *_): self.assert_upload_several_batches() - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_one_hosted_pdf_authorized_while_local_forbidden(self, *_): OrganizationQuery.__call__.side_effect = mocked_organization_with_upload_from_local( upload_local_data=False diff --git a/tests/unit/services/asset_import/test_import_text.py b/tests/unit/services/asset_import/test_import_text.py index d9c98e6d2..38c85f5fb 100644 --- a/tests/unit/services/asset_import/test_import_text.py +++ b/tests/unit/services/asset_import/test_import_text.py @@ -2,10 +2,9 @@ import pytest -from kili.core.graphql.operations.asset.queries import AssetQuery from kili.core.graphql.operations.organization.queries import OrganizationQuery from kili.core.graphql.operations.project.queries import ProjectQuery -from kili.entrypoints.queries.asset import QueriesAsset +from kili.gateways.kili_api_gateway.asset import AssetOperationMixin from kili.services.asset_import import import_assets from kili.services.asset_import.exceptions import UploadFromLocalDataForbiddenError from tests.unit.services.asset_import.base import ImportTestCase @@ -21,18 +20,14 @@ @patch("kili.utils.bucket.upload_data_via_rest", mocked_upload_data_via_rest) @patch("kili.utils.bucket.generate_unique_id", mocked_unique_id) @patch.object(ProjectQuery, "__call__", side_effect=lambda *args: [{"inputType": "TEXT"}]) -@patch.object( - QueriesAsset, - "assets", - MagicMock(return_value=[]), -) +@patch.object(AssetOperationMixin, "list_assets", MagicMock(return_value=[])) @patch.object( OrganizationQuery, "__call__", side_effect=mocked_organization_with_upload_from_local(upload_local_data=True), ) class TextTestCase(ImportTestCase): - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_one_local_text_file(self, *_): url = "https://storage.googleapis.com/label-public-staging/asset-test-sample/texts/test_text_file.txt" path = self.downloader(url) @@ -48,7 +43,7 @@ def test_upload_from_one_local_text_file(self, *_): ) self.kili.graphql_client.execute.assert_called_with(*expected_parameters) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_one_hosted_text_file(self, *_): assets = [ {"content": "https://hosted-data", "external_id": "hosted file", "id": "unique_id"} @@ -59,7 +54,7 @@ def test_upload_from_one_hosted_text_file(self, *_): ) self.kili.graphql_client.execute.assert_called_with(*expected_parameters) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_raw_text(self, *_): assets = [{"content": "this is raw text", "external_id": "raw text"}] import_assets(self.kili, self.project_id, assets) @@ -68,7 +63,7 @@ def test_upload_from_raw_text(self, *_): ) self.kili.graphql_client.execute.assert_called_with(*expected_parameters) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_one_rich_text(self, *_): json_content = [ { @@ -93,11 +88,11 @@ def test_upload_from_one_rich_text(self, *_): ) self.kili.graphql_client.execute.assert_called_with(*expected_parameters) - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_several_batches(self, *_): self.assert_upload_several_batches() - @patch.object(AssetQuery, "count", return_value=1) + @patch.object(AssetOperationMixin, "count_assets", return_value=1) def test_upload_from_one_hosted_text_authorized_while_local_forbidden(self, *_): OrganizationQuery.__call__.side_effect = mocked_organization_with_upload_from_local( upload_local_data=False diff --git a/tests/unit/services/asset_import/test_import_video.py b/tests/unit/services/asset_import/test_import_video.py index 54331f7ef..9dfc0f7ae 100644 --- a/tests/unit/services/asset_import/test_import_video.py +++ b/tests/unit/services/asset_import/test_import_video.py @@ -3,10 +3,9 @@ import pytest -from kili.core.graphql.operations.asset.queries import AssetQuery from kili.core.graphql.operations.organization.queries import OrganizationQuery from kili.core.graphql.operations.project.queries import ProjectQuery -from kili.entrypoints.queries.asset import QueriesAsset +from kili.gateways.kili_api_gateway.asset import AssetOperationMixin from kili.services.asset_import import import_assets from kili.services.asset_import.exceptions import UploadFromLocalDataForbiddenError from tests.unit.services.asset_import.base import ImportTestCase @@ -23,17 +22,13 @@ @patch("kili.utils.bucket.upload_data_via_rest", mocked_upload_data_via_rest) @patch("kili.utils.bucket.generate_unique_id", mocked_unique_id) @patch.object(ProjectQuery, "__call__", side_effect=mocked_project_input_type("VIDEO")) -@patch.object( - QueriesAsset, - "assets", - MagicMock(return_value=[]), -) +@patch.object(AssetOperationMixin, "list_assets", MagicMock(return_value=[])) @patch.object( OrganizationQuery, "__call__", side_effect=mocked_organization_with_upload_from_local(upload_local_data=True), ) -@patch.object(AssetQuery, "count", return_value=1) +@patch.object(AssetOperationMixin, "count_assets", return_value=1) class VideoTestCase(ImportTestCase): def test_upload_from_one_local_video_file_to_native(self, *_): url = "https://storage.googleapis.com/label-public-staging/asset-test-sample/video/short_video.mp4" @@ -321,7 +316,7 @@ def test_import_one_video_with_metadata(self, *_): "assets", MagicMock(return_value=[]), ) -@patch.object(AssetQuery, "count", return_value=1) +@patch.object(AssetOperationMixin, "count_assets", return_value=1) class VideoLegacyTestCase(ImportTestCase): def test_upload_from_one_hosted_video_file_to_video_legacy_project(self, *_): assets = [ diff --git a/tests/unit/services/export/test_export.py b/tests/unit/services/export/test_export.py index 00205c875..d24fd1a80 100644 --- a/tests/unit/services/export/test_export.py +++ b/tests/unit/services/export/test_export.py @@ -9,9 +9,9 @@ import pytest import pytest_mock -from kili.core.graphql.operations.asset.queries import AssetQuery from kili.core.graphql.operations.project.queries import ProjectQuery from kili.entrypoints.queries.label import QueriesLabel +from kili.gateways.kili_api_gateway.asset import AssetOperationMixin from kili.orm import Asset from kili.services import export_labels from kili.services.export.exceptions import ( @@ -620,8 +620,8 @@ def get_file_tree(folder: str): ], ) def test_export_service_layout(mocker: pytest_mock.MockerFixture, name, test_case): - mocker.patch.object(AssetQuery, "count", side_effect=mocked_AssetQuery_count) - mocker.patch.object(AssetQuery, "__call__", side_effect=mocked_AssetQuery) + mocker.patch.object(AssetOperationMixin, "count_assets", side_effect=mocked_AssetQuery_count) + mocker.patch.object(AssetOperationMixin, "list_assets", side_effect=mocked_AssetQuery) mocker.patch.object(ProjectQuery, "__call__", side_effect=mocked_ProjectQuery) mocker_ffmpeg = mocker.patch("kili.services.export.media.video.ffmpeg") mocker.patch( @@ -754,8 +754,8 @@ def test_export_service_layout(mocker: pytest_mock.MockerFixture, name, test_cas ], ) @patch.object(ProjectQuery, "__call__", side_effect=mocked_ProjectQuery) -@patch.object(AssetQuery, "__call__", side_effect=mocked_AssetQuery) -@patch.object(AssetQuery, "count", side_effect=mocked_AssetQuery_count) +@patch.object(AssetOperationMixin, "list_assets", side_effect=mocked_AssetQuery) +@patch.object(AssetOperationMixin, "count_assets", side_effect=mocked_AssetQuery_count) def test_export_service_errors( mocker_asset_count, mocker_asset, mocker_project, name, test_case, error ): diff --git a/tests/unit/services/import_labels/test_import_from_dict.py b/tests/unit/services/import_labels/test_import_from_dict.py index cba241d28..8ba399485 100644 --- a/tests/unit/services/import_labels/test_import_from_dict.py +++ b/tests/unit/services/import_labels/test_import_from_dict.py @@ -5,8 +5,8 @@ import pytest from kili import services -from kili.core.graphql.operations.asset.queries import AssetQuery from kili.core.graphql.operations.label.mutations import GQL_APPEND_MANY_LABELS +from kili.gateways.kili_api_gateway.asset import AssetOperationMixin def mocked_AssetQuery(*_): @@ -68,7 +68,7 @@ def test_import_default_labels_with_asset_id(self): GQL_APPEND_MANY_LABELS, call, timeout=60 ) - @patch.object(AssetQuery, "__call__", side_effect=mocked_AssetQuery) + @patch.object(AssetOperationMixin, "list_assets", side_effect=mocked_AssetQuery) def test_import_default_labels_with_external_id(self, mocker): project_id = "project_id" label_type = "DEFAULT" @@ -181,8 +181,8 @@ def test_return_error_when_give_wrong_field_type(self): ) @patch.object( - AssetQuery, - "__call__", + AssetOperationMixin, + "list_assets", side_effect=mocked_AssetQuery, ) def test_import_predictions(self, mocker): @@ -227,8 +227,8 @@ def test_import_predictions(self, mocker): ) @patch.object( - AssetQuery, - "__call__", + AssetOperationMixin, + "list_assets", side_effect=mocked_AssetQuery, ) def test_import_predictions_with_overwritting(self, mocker): diff --git a/tests/unit/services/label_data_parsing/test_label_utils.py b/tests/unit/services/label_data_parsing/test_label_utils.py index 40a4cad41..8bf932c8c 100644 --- a/tests/unit/services/label_data_parsing/test_label_utils.py +++ b/tests/unit/services/label_data_parsing/test_label_utils.py @@ -4,7 +4,6 @@ from typing_extensions import assert_type from kili.client import Kili -from kili.entrypoints.queries.asset import QueriesAsset from kili.entrypoints.queries.label import QueriesLabel from kili.services.label_data_parsing.annotation import Annotation, AnnotationList from kili.services.label_data_parsing.category import Category, CategoryList @@ -261,19 +260,6 @@ def test_integration_of_label_parsing_in_kili_labels_assert_types(mocker): ) -def test_integration_of_label_parsing_in_kili_asset_assert_types(mocker): - """This test does not check types at runtime, but rather during pyright type checking.""" - _ = mocker.patch.object(Kili, "__init__", return_value=None) - _ = mocker.patch.object(QueriesAsset, "assets") - assert_type(Kili().assets("project_id"), List[Dict]) - assert_type(Kili().assets("project_id", as_generator=True), Generator[Dict, None, None]) - assert_type(Kili().assets("project_id", label_output_format="parsed_label"), List[Dict]) - assert_type( - Kili().assets("project_id", label_output_format="parsed_label", as_generator=True), - Generator[Dict, None, None], - ) - - def test_integration_of_label_parsing_in_kili_labels(mocker): mocker_project_query = mocker.patch( "kili.core.graphql.operations.project.queries.ProjectQuery.__call__", @@ -313,113 +299,3 @@ def test_integration_of_label_parsing_in_kili_labels(mocker): assert all(isinstance(labl, ParsedLabel) for labl in labels) # pylint: disable=not-an-iterable assert len(labels) == 1 assert labels[0].jobs["JOB_0"].text == "some text abc" # pylint: disable=unsubscriptable-object - - -def test_integration_of_label_parsing_in_kili_assets(mocker): - mocker_project_query = mocker.patch( - "kili.core.graphql.operations.project.queries.ProjectQuery.__call__", - side_effect=lambda *args, **kwargs: ( - x - for x in [ - { - "jsonInterface": { - "jobs": { - "JOB_0": {"mlTask": "TRANSCRIPTION", "required": 1, "isChild": False} - } - }, - "inputType": "TEXT", - } - ] - ), - ) - - mocker_label_query = mocker.patch( - "kili.core.graphql.operations.asset.queries.AssetQuery.get_number_of_elements_to_query", - return_value=1, - ) - - label = { - "author": { - "id": "cldbnzmmq00go0jwc20fq1jkl", - "email": "john.doe@kili-technology.com", - }, - "createdAt": "2023-05-11T16:01:48.093Z", - "id": "clhjbhrul015m0k7hct21drz4", - "jsonResponse": {"JOB_0": {"text": "some text abc"}}, - } - mocked_execute = mocker.MagicMock( - return_value={ - "data": [ - { - "labels": [label], - "latestLabel": label, - "content": "https://storage.googleapis.com/label-backend-staging/", - "createdAt": "2023-05-11T15:55:01.134Z", - "externalId": ( - "4bad2303e43bfefa0169d890c68f5c9d--cherry-blossom-tree-blossom-trees.jpg" - ), - "id": "clhjb919i00002a6a2kwtp5qr", - "isHoneypot": False, - "jsonMetadata": {}, - "skipped": False, - "status": "LABELED", - } - ] - } - ) - kili = QueriesAsset() - kili.graphql_client = mocker.MagicMock() - kili.graphql_client.execute = mocked_execute - kili.http_client = mocker.MagicMock() - - fields = [ - "content", - "createdAt", - "externalId", - "id", - "isHoneypot", - "jsonMetadata", - "labels.author.id", - "labels.author.email", - "labels.createdAt", - "labels.id", - "labels.jsonResponse", - "skipped", - "status", - "latestLabel.jsonResponse", - ] - - for label_output_format in ("dict", "parsed_label"): - assets = kili.assets( - project_id="project_id", label_output_format=label_output_format, fields=fields - ) - - assert_type(assets, List[Dict]) # static test with pyright - assert isinstance(assets, List) - assert len(assets) == 1 - assert len(assets[0]["labels"]) == 1 # pylint: disable=unsubscriptable-object - # pylint: disable=unsubscriptable-object - for label in (assets[0]["labels"][0], assets[0]["latestLabel"]): - if label_output_format == "dict": - assert isinstance(label, Dict) - assert label["jsonResponse"]["JOB_0"]["text"] == "some text abc" - elif label_output_format == "parsed_label": - assert isinstance(label, ParsedLabel) - assert label.jobs["JOB_0"].text == "some text abc" - - # test return a generator of assets with parsed labels - assets = kili.assets( - project_id="project_id", - label_output_format="parsed_label", - fields=fields, - as_generator=True, - ) - assert_type(assets, Generator[Dict, None, None]) # static test with pyright - assert isinstance(assets, Generator) - for i, asset in enumerate(assets): - assert len(asset["labels"]) == 1 - for label in (asset["labels"][0], asset["latestLabel"]): - assert isinstance(label, ParsedLabel) - assert label.jobs["JOB_0"].text == "some text abc" - - assert i == 0 # type: ignore diff --git a/tests/unit/services/test_helpers.py b/tests/unit/services/test_helpers.py index ab2e05fae..59ab57f12 100644 --- a/tests/unit/services/test_helpers.py +++ b/tests/unit/services/test_helpers.py @@ -1,7 +1,7 @@ import pytest -from kili.core.graphql.operations.asset.queries import AssetQuery from kili.core.graphql.operations.project.queries import ProjectQuery +from kili.gateways.kili_api_gateway.asset import AssetOperationMixin from kili.services.helpers import _build_id_map from tests.fakes.fake_kili import ( FakeKili, @@ -22,8 +22,8 @@ def test__build_id_map_when_project_has_less_than_2000_assets(mocker, name, asse _ = name kili = FakeKili() mocker.patch.object(ProjectQuery, "__call__", side_effect=mocked_ProjectQuery) - mocker.patch.object(AssetQuery, "__call__", side_effect=mocked_AssetQuery) - mocker.patch.object(AssetQuery, "count", side_effect=mocked_AssetQuery_count) + mocker.patch.object(AssetOperationMixin, "list_assets", side_effect=mocked_AssetQuery) + mocker.patch.object(AssetOperationMixin, "count_assets", side_effect=mocked_AssetQuery_count) asset_external_ids = [f"ext-{i}" for i in range(asset_count)] project_id = "object_detection_2500_assets"