From be721d066f1b96a834dc79c46fd50df757d1724c Mon Sep 17 00:00:00 2001 From: theodu Date: Mon, 28 Aug 2023 12:04:39 +0200 Subject: [PATCH] feat: move QueryOptions --- src/kili/client.py | 2 +- src/kili/core/graphql/__init__.py | 11 +- src/kili/entrypoints/cli/project/list_.py | 2 +- .../entrypoints/cli/project/member/add.py | 2 +- .../entrypoints/cli/project/member/helpers.py | 2 +- .../entrypoints/cli/project/member/list_.py | 2 +- .../entrypoints/cli/project/member/remove.py | 2 +- .../entrypoints/cli/project/member/update.py | 2 +- .../entrypoints/mutations/asset/__init__.py | 24 +- .../mutations/data_connection/__init__.py | 2 +- .../entrypoints/mutations/issue/__init__.py | 2 +- .../entrypoints/mutations/issue/helpers.py | 2 +- .../entrypoints/queries/api_key/__init__.py | 2 +- .../entrypoints/queries/asset/__init__.py | 216 ------------------ .../queries/data_connection/__init__.py | 2 +- .../queries/data_integration/__init__.py | 2 +- .../entrypoints/queries/issue/__init__.py | 2 +- .../entrypoints/queries/label/__init__.py | 12 +- .../queries/notification/__init__.py | 2 +- .../queries/organization/__init__.py | 2 +- .../entrypoints/queries/plugins/__init__.py | 2 +- .../entrypoints/queries/project/__init__.py | 2 +- .../queries/project_user/__init__.py | 2 +- .../queries/project_version/__init__.py | 2 +- src/kili/entrypoints/queries/user/__init__.py | 2 +- src/kili/services/asset_import/base.py | 20 +- src/kili/services/copy_project/__init__.py | 16 +- src/kili/services/data_connection/__init__.py | 11 +- src/kili/services/export/format/base.py | 2 +- src/kili/services/export/tools.py | 2 +- src/kili/services/helpers.py | 12 +- src/kili/services/project/__init__.py | 2 +- src/kili/use_cases/asset/media_downloader.py | 2 +- 33 files changed, 83 insertions(+), 289 deletions(-) delete mode 100644 src/kili/entrypoints/queries/asset/__init__.py diff --git a/src/kili/client.py b/src/kili/client.py index 802bffc74..e83acc443 100644 --- a/src/kili/client.py +++ b/src/kili/client.py @@ -10,7 +10,6 @@ import requests from kili import __version__ -from kili.core.graphql import QueryOptions from kili.core.graphql.graphql_client import GraphQLClient, GraphQLClientName from kili.core.graphql.operations.api_key.queries import APIKeyQuery, APIKeyWhere from kili.core.graphql.operations.user.queries import GQL_ME @@ -38,6 +37,7 @@ from kili.entrypoints.subscriptions.label import SubscriptionsLabel from kili.exceptions import AuthenticationFailed, UserNotFoundError from kili.gateways.kili_api_gateway import KiliAPIGateway +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.presentation.client.asset import AssetClientMethods from kili.presentation.client.internal import InternalClientMethods from kili.presentation.client.issue import IssueClientMethods diff --git a/src/kili/core/graphql/__init__.py b/src/kili/core/graphql/__init__.py index 18d81d1ff..56bb79c28 100644 --- a/src/kili/core/graphql/__init__.py +++ b/src/kili/core/graphql/__init__.py @@ -1,13 +1,14 @@ """GraphQL module.""" from abc import ABC, abstractmethod -from typing import Callable, Dict, Generator, List, NamedTuple, Optional, Type, TypeVar +from typing import Callable, Dict, Generator, List, Optional, Type, TypeVar import requests from typeguard import typechecked from kili.core.constants import QUERY_BATCH_SIZE from kili.core.helpers import format_result +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.utils.tqdm import tqdm from .graphql_client import GraphQLClient @@ -15,14 +16,6 @@ T = TypeVar("T") -class QueryOptions(NamedTuple): - """Options when calling GraphQLQuery from the SDK.""" - - disable_tqdm: Optional[bool] - first: Optional[int] = None - skip: int = 0 - - class BaseQueryWhere(ABC): """Abtsract class for defining the where payload to send in a graphQL query.""" diff --git a/src/kili/entrypoints/cli/project/list_.py b/src/kili/entrypoints/cli/project/list_.py index 14aa7bea2..652c30104 100644 --- a/src/kili/entrypoints/cli/project/list_.py +++ b/src/kili/entrypoints/cli/project/list_.py @@ -7,10 +7,10 @@ import pandas as pd from tabulate import tabulate -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.project.queries import ProjectQuery, ProjectWhere from kili.entrypoints.cli.common_args import Options from kili.entrypoints.cli.helpers import get_kili_client +from kili.gateways.kili_api_gateway.queries import QueryOptions @click.command(name="list") diff --git a/src/kili/entrypoints/cli/project/member/add.py b/src/kili/entrypoints/cli/project/member/add.py index d93a5951e..6acb7d546 100644 --- a/src/kili/entrypoints/cli/project/member/add.py +++ b/src/kili/entrypoints/cli/project/member/add.py @@ -5,7 +5,6 @@ import click -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.project_user.queries import ( ProjectUserQuery, ProjectUserWhere, @@ -18,6 +17,7 @@ collect_members_from_emails, collect_members_from_project, ) +from kili.gateways.kili_api_gateway.queries import QueryOptions # pylint: disable=too-many-arguments diff --git a/src/kili/entrypoints/cli/project/member/helpers.py b/src/kili/entrypoints/cli/project/member/helpers.py index 66fa3bd36..b0f2446f0 100644 --- a/src/kili/entrypoints/cli/project/member/helpers.py +++ b/src/kili/entrypoints/cli/project/member/helpers.py @@ -7,13 +7,13 @@ import requests from kili.client import Kili -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.project_user.queries import ( ProjectUserQuery, ProjectUserWhere, ) from kili.entrypoints.cli.common_args import ROLES from kili.entrypoints.cli.helpers import collect_from_csv +from kili.gateways.kili_api_gateway.queries import QueryOptions REGEX_EMAIL = re.compile(r"([A-Za-z0-9]+[.-_])*[A-Za-z0-9]+@[A-Za-z0-9-]+(\.[A-Z|a-z]{2,})+") diff --git a/src/kili/entrypoints/cli/project/member/list_.py b/src/kili/entrypoints/cli/project/member/list_.py index 1303522cf..e78db040f 100644 --- a/src/kili/entrypoints/cli/project/member/list_.py +++ b/src/kili/entrypoints/cli/project/member/list_.py @@ -6,13 +6,13 @@ import pandas as pd from tabulate import tabulate -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.project_user.queries import ( ProjectUserQuery, ProjectUserWhere, ) from kili.entrypoints.cli.common_args import Arguments, Options from kili.entrypoints.cli.helpers import get_kili_client +from kili.gateways.kili_api_gateway.queries import QueryOptions ROLE_ORDER = {v: i for i, v in enumerate(["ADMIN", "TEAM_MANAGER", "REVIEWER", "LABELER"])} diff --git a/src/kili/entrypoints/cli/project/member/remove.py b/src/kili/entrypoints/cli/project/member/remove.py index 8cf4b6838..5dc49f0bb 100644 --- a/src/kili/entrypoints/cli/project/member/remove.py +++ b/src/kili/entrypoints/cli/project/member/remove.py @@ -5,7 +5,6 @@ import click -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.project_user.queries import ( ProjectUserQuery, ProjectUserWhere, @@ -18,6 +17,7 @@ collect_members_from_emails, collect_members_from_project, ) +from kili.gateways.kili_api_gateway.queries import QueryOptions @click.command(name="rm") diff --git a/src/kili/entrypoints/cli/project/member/update.py b/src/kili/entrypoints/cli/project/member/update.py index e3bfd2dd8..d8e27cafc 100644 --- a/src/kili/entrypoints/cli/project/member/update.py +++ b/src/kili/entrypoints/cli/project/member/update.py @@ -5,7 +5,6 @@ import click -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.project_user.queries import ( ProjectUserQuery, ProjectUserWhere, @@ -18,6 +17,7 @@ collect_members_from_emails, collect_members_from_project, ) +from kili.gateways.kili_api_gateway.queries import QueryOptions @click.command(name="update") diff --git a/src/kili/entrypoints/mutations/asset/__init__.py b/src/kili/entrypoints/mutations/asset/__init__.py index 75b9e359a..6dee82ad3 100644 --- a/src/kili/entrypoints/mutations/asset/__init__.py +++ b/src/kili/entrypoints/mutations/asset/__init__.py @@ -7,8 +7,6 @@ from tenacity.wait import wait_exponential from typeguard import typechecked -from kili.core.graphql import QueryOptions -from kili.core.graphql.operations.asset.queries import AssetQuery, AssetWhere from kili.core.helpers import is_empty_list_with_warning from kili.core.utils.pagination import mutate_from_paginated_call from kili.entrypoints.base import BaseOperationEntrypointMixin @@ -22,6 +20,9 @@ GQL_UPDATE_PROPERTIES_IN_ASSETS, ) from kili.exceptions import MissingArgumentError +from kili.gateways.kili_api_gateway import KiliAPIGateway +from kili.gateways.kili_api_gateway.asset.types import AssetWhere +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.orm import Asset from kili.services.asset_import import import_assets from kili.services.asset_import_csv import get_text_assets_from_csv @@ -36,6 +37,8 @@ class MutationsAsset(BaseOperationEntrypointMixin): """Set of Asset mutations.""" + kili_api_gateway: KiliAPIGateway + # pylint: disable=too-many-arguments,too-many-locals @typechecked def append_many_to_dataset( @@ -414,7 +417,8 @@ def verify_last_batch(last_batch: Dict, results: List) -> None: return asset_ids = last_batch["asset_ids"][-1:] # check last asset of the batch only - nb_assets_in_kili = AssetQuery(self.graphql_client, self.http_client).count( + + nb_assets_in_kili = self.kili_api_gateway.count_assets( AssetWhere( project_id=project_id_, asset_id_in=asset_ids, @@ -490,7 +494,7 @@ def verify_last_batch(last_batch: Dict, results: List) -> None: return asset_ids = last_batch["asset_ids"][-1:] # check last asset of the batch only - nb_assets_in_review = AssetQuery(self.graphql_client, self.http_client).count( + nb_assets_in_review = self.kili_api_gateway.count_assets( AssetWhere( project_id=project_id_, asset_id_in=asset_ids, @@ -511,10 +515,11 @@ def verify_last_batch(last_batch: Dict, results: List) -> None: # unlike send_back_to_queue, the add_to_review mutation doesn't always return the project ID # it happens when no assets have been sent to review if isinstance(result, dict) and "id" in result: - assets_in_review = AssetQuery(self.graphql_client, self.http_client)( - AssetWhere(project_id=result["id"], asset_id_in=asset_ids, status_in=["TO_REVIEW"]), + assets_in_review = self.kili_api_gateway.list_assets( ["id"], + AssetWhere(project_id=result["id"], asset_id_in=asset_ids, status_in=["TO_REVIEW"]), QueryOptions(disable_tqdm=True), + None, ) result["asset_ids"] = [asset["id"] for asset in assets_in_review] return result @@ -573,7 +578,7 @@ def verify_last_batch(last_batch: Dict, results: List) -> None: return asset_ids = last_batch["asset_ids"][-1:] # check lastest asset of the batch only - nb_assets_in_queue = AssetQuery(self.graphql_client, self.http_client).count( + nb_assets_in_queue = self.kili_api_gateway.count_assets( AssetWhere( project_id=project_id_, asset_id_in=asset_ids, @@ -592,10 +597,11 @@ def verify_last_batch(last_batch: Dict, results: List) -> None: ) result = self.format_result("data", results[0]) if isinstance(result, dict) and "id" in result: - assets_in_queue = AssetQuery(self.graphql_client, self.http_client)( - AssetWhere(project_id=result["id"], asset_id_in=asset_ids, status_in=["ONGOING"]), + assets_in_queue = self.kili_api_gateway.list_assets( ["id"], + AssetWhere(project_id=result["id"], asset_id_in=asset_ids, status_in=["ONGOING"]), QueryOptions(disable_tqdm=True), + None, ) result["asset_ids"] = [asset["id"] for asset in assets_in_queue] return result diff --git a/src/kili/entrypoints/mutations/data_connection/__init__.py b/src/kili/entrypoints/mutations/data_connection/__init__.py index c324ec45a..b9f85cd7a 100644 --- a/src/kili/entrypoints/mutations/data_connection/__init__.py +++ b/src/kili/entrypoints/mutations/data_connection/__init__.py @@ -6,12 +6,12 @@ from typeguard import typechecked from kili import services -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.data_integration.queries import ( DataIntegrationsQuery, DataIntegrationWhere, ) from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.utils.logcontext import for_all_methods, log_call from .queries import GQL_ADD_PROJECT_DATA_CONNECTION diff --git a/src/kili/entrypoints/mutations/issue/__init__.py b/src/kili/entrypoints/mutations/issue/__init__.py index b2529c2dd..302b6a676 100644 --- a/src/kili/entrypoints/mutations/issue/__init__.py +++ b/src/kili/entrypoints/mutations/issue/__init__.py @@ -4,12 +4,12 @@ from typeguard import typechecked -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.label.queries import LabelQuery, LabelWhere from kili.core.helpers import deprecate from kili.entrypoints.base import BaseOperationEntrypointMixin from kili.entrypoints.mutations.asset.helpers import get_asset_ids_or_throw_error from kili.gateways.kili_api_gateway.issue.operations import GQL_CREATE_ISSUES +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.services.helpers import assert_all_arrays_have_same_size from kili.utils.logcontext import for_all_methods, log_call diff --git a/src/kili/entrypoints/mutations/issue/helpers.py b/src/kili/entrypoints/mutations/issue/helpers.py index cb83e0b91..56f21fa4a 100644 --- a/src/kili/entrypoints/mutations/issue/helpers.py +++ b/src/kili/entrypoints/mutations/issue/helpers.py @@ -3,10 +3,10 @@ from typing import List, Literal, cast -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.issue.queries import IssueQuery, IssueWhere from kili.core.graphql.operations.label.queries import LabelQuery, LabelWhere from kili.exceptions import NotFound +from kili.gateways.kili_api_gateway.queries import QueryOptions # pylint: disable=missing-type-doc diff --git a/src/kili/entrypoints/queries/api_key/__init__.py b/src/kili/entrypoints/queries/api_key/__init__.py index 25239d168..782776091 100644 --- a/src/kili/entrypoints/queries/api_key/__init__.py +++ b/src/kili/entrypoints/queries/api_key/__init__.py @@ -4,9 +4,9 @@ from typeguard import typechecked -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.api_key.queries import APIKeyQuery, APIKeyWhere from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.presentation.client.common_validators import disable_tqdm_if_as_generator from kili.utils.logcontext import for_all_methods, log_call diff --git a/src/kili/entrypoints/queries/asset/__init__.py b/src/kili/entrypoints/queries/asset/__init__.py deleted file mode 100644 index e074d4efd..000000000 --- a/src/kili/entrypoints/queries/asset/__init__.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Asset queries.""" - -import warnings -from typing import List, Literal, Optional - -from typeguard import typechecked - -from kili.core.graphql.operations.asset.queries import AssetQuery, AssetWhere -from kili.core.helpers import validate_category_search_query -from kili.entrypoints.base import BaseOperationEntrypointMixin -from kili.utils.logcontext import for_all_methods, log_call - - -@for_all_methods(log_call, exclude=["__init__"]) -class QueriesAsset(BaseOperationEntrypointMixin): - """Set of Asset queries.""" - - # pylint: disable=too-many-arguments,too-many-locals,dangerous-default-value,redefined-builtin - - @typechecked - def count_assets( - self, - project_id: str, - asset_id: Optional[str] = None, - asset_id_in: Optional[List[str]] = None, - asset_id_not_in: Optional[List[str]] = None, - external_id_contains: Optional[List[str]] = None, - metadata_where: Optional[dict] = None, - status_in: Optional[List[str]] = None, - consensus_mark_gt: Optional[float] = None, - consensus_mark_lt: Optional[float] = None, - honeypot_mark_gt: Optional[float] = None, - honeypot_mark_lt: Optional[float] = None, - label_type_in: Optional[List[str]] = None, - label_author_in: Optional[List[str]] = None, - label_consensus_mark_gt: Optional[float] = None, - label_consensus_mark_lt: Optional[float] = None, - label_created_at: Optional[str] = None, - label_created_at_gt: Optional[str] = None, - label_created_at_lt: Optional[str] = None, - label_honeypot_mark_gt: Optional[float] = None, - label_honeypot_mark_lt: Optional[float] = None, - skipped: Optional[bool] = None, - updated_at_gte: Optional[str] = None, - updated_at_lte: Optional[str] = None, - label_category_search: Optional[str] = None, - created_at_gte: Optional[str] = None, - created_at_lte: Optional[str] = None, - honeypot_mark_gte: Optional[float] = None, - honeypot_mark_lte: Optional[float] = None, - consensus_mark_gte: Optional[float] = None, - consensus_mark_lte: Optional[float] = None, - inference_mark_gte: Optional[float] = None, - inference_mark_lte: Optional[float] = None, - label_reviewer_in: Optional[List[str]] = None, - label_consensus_mark_gte: Optional[float] = None, - label_consensus_mark_lte: Optional[float] = None, - label_created_at_gte: Optional[str] = None, - label_created_at_lte: Optional[str] = None, - label_honeypot_mark_gte: Optional[float] = None, - label_honeypot_mark_lte: Optional[float] = None, - issue_type: Optional[Literal["QUESTION", "ISSUE"]] = None, - issue_status: Optional[Literal["OPEN", "SOLVED"]] = None, - external_id_strictly_in: Optional[List[str]] = None, - external_id_in: Optional[List[str]] = None, - ) -> int: - # pylint: disable=line-too-long - """Count and return the number of assets with the given constraints. - - Parameters beginning with 'label_' apply to labels, others apply to assets. - - Args: - project_id: Identifier of the project - asset_id: The unique id of the asset to retrieve. - asset_id_in: A list of the ids of the assets to retrieve. - asset_id_not_in: A list of the ids of the assets to exclude. - external_id_contains: Deprecated. Use `external_id_strictly_in` instead. - metadata_where: Filters by the values of the metadata of the asset. - status_in: Returned assets should have a status that belongs to that list, if given. Possible choices: `TODO`, `ONGOING`, `LABELED`, `TO_REVIEW` or `REVIEWED`. - consensus_mark_gt: Deprecated. Use `consensus_mark_gte` instead. - consensus_mark_lt: Deprecated. Use `consensus_mark_lte` instead. - honeypot_mark_gt: Deprecated. Use `honeypot_mark_gte` instead. - honeypot_mark_lt: Deprecated. Use `honeypot_mark_lte` instead. - label_type_in: Returned assets should have a label whose type belongs to that list, if given. - label_author_in: Returned assets should have a label whose author belongs to that list, if given. An author can be designated by the first name, the last name, or the first name + last name. - label_consensus_mark_gt: Deprecated. Use `label_consensus_mark_gte` instead. - label_consensus_mark_lt: Deprecated. Use `label_consensus_mark_lte` instead. - label_created_at: Returned assets should have a label whose creation date is equal to this date. - label_created_at_gt: Deprecated. Use `label_created_at_gte` instead. - label_created_at_lt: Deprecated. Use `label_created_at_lte` instead. - label_honeypot_mark_gt: Deprecated. Use `label_honeypot_mark_gte` instead. - label_honeypot_mark_lt: Deprecated. Use `label_honeypot_mark_lte` instead. - skipped: Returned assets should be skipped. - updated_at_gte: Returned assets should have a label whose update date is greated or equal to this date. - updated_at_lte: Returned assets should have a label whose update date is lower or equal to this date. - label_category_search: Returned assets should have a label that follows this category search query. - created_at_gte: Returned assets should have their import date greater or equal to this date. - created_at_lte: Returned assets should have their import date lower or equal to this date. - honeypot_mark_lte: Maximum amount of honeypot for the asset. - honeypot_mark_gte: Minimum amount of honeypot for the asset. - consensus_mark_lte: Maximum amount of consensus for the asset. - consensus_mark_gte: Minimum amount of consensus for the asset. - inference_mark_gte: Minimum amount of human/model IoU for the asset. - inference_mark_lte: Maximum amount of human/model IoU for the asset. - label_reviewer_in: Returned assets should have a label whose reviewer belongs to that list, if given. - label_consensus_mark_gte: Returned assets should have a label whose consensus is greater or equal to this number. - label_consensus_mark_lte: Returned assets should have a label whose consensus is lower or equal to this number. - label_created_at_lte: Returned assets should have a label whose creation date is lower or equal to this date. - label_created_at_gte: Returned assets should have a label whose creation date is greater or equal to this date. - label_honeypot_mark_gte: Returned assets should have a label whose honeypot is greater or equal to this number. - label_honeypot_mark_lte: Returned assets should have a label whose honeypot is lower or equal to this number. - issue_type: Returned assets should have issues of type `QUESTION` or `ISSUE`. - issue_status: Returned assets should have issues of status `OPEN` or `SOLVED`. - external_id_strictly_in: Returned assets should have external ids that match exactly the ones in the list. - external_id_in: Returned assets should have external ids that partially match the ones in the list. - For example, with `external_id_in=['abc']`, any asset with an external id containing `'abc'` will be returned. - - !!! info "Dates format" - Date strings should have format: "YYYY-MM-DD" - - Returns: - The number of assets that match the given constraints. - - Examples: - >>> kili.count_assets(project_id=project_id) - 250 - >>> kili.count_assets(asset_id=asset_id) - 1 - - !!! example "How to filter based on Metadata" - - `metadata_where = {key1: "value1"}` to filter on assets whose metadata - have key "key1" with value "value1" - - `metadata_where = {key1: ["value1", "value2"]}` to filter on assets whose metadata - have key "key1" with value "value1" or value "value2 - - `metadata_where = {key2: [2, 10]}` to filter on assets whose metadata - have key "key2" with a value between 2 and 10. - """ - if label_category_search: - validate_category_search_query(label_category_search) - - if external_id_contains is not None: - warnings.warn( - "external_id_contains is deprecated, use external_id_strictly_in instead", - DeprecationWarning, - stacklevel=1, - ) - - for arg_name, arg_value in zip( - ( - "consensus_mark_gt", - "consensus_mark_lt", - "honeypot_mark_gt", - "honeypot_mark_lt", - "label_consensus_mark_gt", - "label_consensus_mark_lt", - "label_created_at_gt", - "label_created_at_lt", - "label_honeypot_mark_gt", - "label_honeypot_mark_lt", - ), - ( - consensus_mark_gt, - consensus_mark_lt, - honeypot_mark_gt, - honeypot_mark_lt, - label_consensus_mark_gt, - label_consensus_mark_lt, - label_created_at_gt, - label_created_at_lt, - label_honeypot_mark_gt, - label_honeypot_mark_lt, - ), - ): - if arg_value: - warnings.warn( - f"'{arg_name}' is deprecated, please use" - f" '{arg_name.replace('_gt', '_gte').replace('_lt', '_lte')}' instead.", - DeprecationWarning, - stacklevel=1, - ) - - where = AssetWhere( - project_id=project_id, - asset_id=asset_id, - asset_id_in=asset_id_in, - asset_id_not_in=asset_id_not_in, - consensus_mark_gte=consensus_mark_gt or consensus_mark_gte, - consensus_mark_lte=consensus_mark_lt or consensus_mark_lte, - external_id_strictly_in=external_id_strictly_in or external_id_contains, - external_id_in=external_id_in, - honeypot_mark_gte=honeypot_mark_gt or honeypot_mark_gte, - honeypot_mark_lte=honeypot_mark_lt or honeypot_mark_lte, - inference_mark_gte=inference_mark_gte, - inference_mark_lte=inference_mark_lte, - label_author_in=label_author_in, - label_reviewer_in=label_reviewer_in, - label_consensus_mark_gte=label_consensus_mark_gt or label_consensus_mark_gte, - label_consensus_mark_lte=label_consensus_mark_lt or label_consensus_mark_lte, - label_created_at=label_created_at, - label_created_at_gte=label_created_at_gt or label_created_at_gte, - label_created_at_lte=label_created_at_lt or label_created_at_lte, - label_honeypot_mark_gte=label_honeypot_mark_gt or label_honeypot_mark_gte, - label_honeypot_mark_lte=label_honeypot_mark_lt or label_honeypot_mark_lte, - label_type_in=label_type_in, - metadata_where=metadata_where, - skipped=skipped, - status_in=status_in, - updated_at_gte=updated_at_gte, - updated_at_lte=updated_at_lte, - label_category_search=label_category_search, - created_at_gte=created_at_gte, - created_at_lte=created_at_lte, - issue_status=issue_status, - issue_type=issue_type, - ) - return AssetQuery(self.graphql_client, self.http_client).count(where) diff --git a/src/kili/entrypoints/queries/data_connection/__init__.py b/src/kili/entrypoints/queries/data_connection/__init__.py index 5c0675ba2..d913e94eb 100644 --- a/src/kili/entrypoints/queries/data_connection/__init__.py +++ b/src/kili/entrypoints/queries/data_connection/__init__.py @@ -5,12 +5,12 @@ from typeguard import typechecked from kili import services -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.data_connection.queries import ( DataConnectionsQuery, DataConnectionsWhere, ) from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.presentation.client.common_validators import disable_tqdm_if_as_generator from kili.utils.logcontext import for_all_methods, log_call diff --git a/src/kili/entrypoints/queries/data_integration/__init__.py b/src/kili/entrypoints/queries/data_integration/__init__.py index 641bf2033..8aa7f9304 100644 --- a/src/kili/entrypoints/queries/data_integration/__init__.py +++ b/src/kili/entrypoints/queries/data_integration/__init__.py @@ -4,12 +4,12 @@ from typeguard import typechecked -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.data_integration.queries import ( DataIntegrationsQuery, DataIntegrationWhere, ) from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.presentation.client.common_validators import disable_tqdm_if_as_generator from kili.utils.logcontext import for_all_methods, log_call diff --git a/src/kili/entrypoints/queries/issue/__init__.py b/src/kili/entrypoints/queries/issue/__init__.py index 59ee26c47..9fb57e095 100644 --- a/src/kili/entrypoints/queries/issue/__init__.py +++ b/src/kili/entrypoints/queries/issue/__init__.py @@ -3,9 +3,9 @@ from typeguard import typechecked -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.issue.queries import IssueQuery, IssueWhere from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.presentation.client.common_validators import disable_tqdm_if_as_generator from kili.utils.logcontext import for_all_methods, log_call diff --git a/src/kili/entrypoints/queries/label/__init__.py b/src/kili/entrypoints/queries/label/__init__.py index 4b8382d2e..9627408c2 100644 --- a/src/kili/entrypoints/queries/label/__init__.py +++ b/src/kili/entrypoints/queries/label/__init__.py @@ -17,11 +17,12 @@ from typeguard import typechecked from kili import services -from kili.core.graphql import QueryOptions -from kili.core.graphql.operations.asset.queries import AssetQuery, AssetWhere from kili.core.graphql.operations.label.queries import LabelQuery, LabelWhere from kili.core.helpers import validate_category_search_query from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway import KiliAPIGateway +from kili.gateways.kili_api_gateway.asset.types import AssetWhere +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.presentation.client.common_validators import disable_tqdm_if_as_generator from kili.services.export.exceptions import NoCompatibleJobError from kili.services.export.types import CocoAnnotationModifier, LabelFormat, SplitOption @@ -36,6 +37,8 @@ class QueriesLabel(BaseOperationEntrypointMixin): """Set of Label queries.""" + kili_api_gateway: KiliAPIGateway + # pylint: disable=too-many-arguments,too-many-locals,dangerous-default-value @overload @@ -636,10 +639,11 @@ def export_labels_as_df( A pandas DataFrame containing the labels. """ services.get_project(self, project_id, ["id"]) - assets_gen = AssetQuery(self.graphql_client, self.http_client)( - AssetWhere(project_id=project_id), + assets_gen = self.kili_api_gateway.list_assets( asset_fields + ["labels." + field for field in fields], + AssetWhere(project_id=project_id), QueryOptions(disable_tqdm=False), + None, ) labels = [ dict( diff --git a/src/kili/entrypoints/queries/notification/__init__.py b/src/kili/entrypoints/queries/notification/__init__.py index 31c3fee36..28f9001df 100644 --- a/src/kili/entrypoints/queries/notification/__init__.py +++ b/src/kili/entrypoints/queries/notification/__init__.py @@ -4,12 +4,12 @@ from typeguard import typechecked -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.notification.queries import ( NotificationQuery, NotificationWhere, ) from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.presentation.client.common_validators import disable_tqdm_if_as_generator from kili.utils.logcontext import for_all_methods, log_call diff --git a/src/kili/entrypoints/queries/organization/__init__.py b/src/kili/entrypoints/queries/organization/__init__.py index 1085232eb..4d95181b1 100644 --- a/src/kili/entrypoints/queries/organization/__init__.py +++ b/src/kili/entrypoints/queries/organization/__init__.py @@ -5,7 +5,6 @@ from typeguard import typechecked -from kili.core.graphql import QueryOptions from kili.core.graphql.graphql_client import GraphQLClient from kili.core.graphql.operations.organization.queries import ( OrganizationMetricsWhere, @@ -13,6 +12,7 @@ OrganizationWhere, ) from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.presentation.client.common_validators import disable_tqdm_if_as_generator from kili.utils.logcontext import for_all_methods, log_call diff --git a/src/kili/entrypoints/queries/plugins/__init__.py b/src/kili/entrypoints/queries/plugins/__init__.py index 23d00570f..8a160d257 100644 --- a/src/kili/entrypoints/queries/plugins/__init__.py +++ b/src/kili/entrypoints/queries/plugins/__init__.py @@ -6,13 +6,13 @@ from typeguard import typechecked -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.plugin.queries import ( PluginBuildErrorsWhere, PluginLogsWhere, PluginQuery, ) from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.services.plugins import PluginUploader from kili.utils.logcontext import for_all_methods, log_call diff --git a/src/kili/entrypoints/queries/project/__init__.py b/src/kili/entrypoints/queries/project/__init__.py index 1fbdedddb..85adc1723 100644 --- a/src/kili/entrypoints/queries/project/__init__.py +++ b/src/kili/entrypoints/queries/project/__init__.py @@ -4,9 +4,9 @@ from typeguard import typechecked -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.project.queries import ProjectQuery, ProjectWhere from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.presentation.client.common_validators import disable_tqdm_if_as_generator from kili.utils.logcontext import for_all_methods, log_call diff --git a/src/kili/entrypoints/queries/project_user/__init__.py b/src/kili/entrypoints/queries/project_user/__init__.py index 5c7dd232a..1801854a1 100644 --- a/src/kili/entrypoints/queries/project_user/__init__.py +++ b/src/kili/entrypoints/queries/project_user/__init__.py @@ -4,12 +4,12 @@ from typeguard import typechecked -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.project_user.queries import ( ProjectUserQuery, ProjectUserWhere, ) from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.presentation.client.common_validators import disable_tqdm_if_as_generator from kili.utils.logcontext import for_all_methods, log_call diff --git a/src/kili/entrypoints/queries/project_version/__init__.py b/src/kili/entrypoints/queries/project_version/__init__.py index 2c47bbafc..d8e075644 100644 --- a/src/kili/entrypoints/queries/project_version/__init__.py +++ b/src/kili/entrypoints/queries/project_version/__init__.py @@ -4,12 +4,12 @@ from typeguard import typechecked -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.project_version.queries import ( ProjectVersionQuery, ProjectVersionWhere, ) from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.presentation.client.common_validators import disable_tqdm_if_as_generator from kili.utils.logcontext import for_all_methods, log_call diff --git a/src/kili/entrypoints/queries/user/__init__.py b/src/kili/entrypoints/queries/user/__init__.py index 465922a98..740281ecf 100644 --- a/src/kili/entrypoints/queries/user/__init__.py +++ b/src/kili/entrypoints/queries/user/__init__.py @@ -4,9 +4,9 @@ from typeguard import typechecked -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.user.queries import UserQuery, UserWhere from kili.entrypoints.base import BaseOperationEntrypointMixin +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.presentation.client.common_validators import disable_tqdm_if_as_generator from kili.utils.logcontext import for_all_methods, log_call diff --git a/src/kili/services/asset_import/base.py b/src/kili/services/asset_import/base.py index 319bcfe78..72507f3f6 100644 --- a/src/kili/services/asset_import/base.py +++ b/src/kili/services/asset_import/base.py @@ -15,18 +15,19 @@ from tenacity.retry import retry_if_exception_type from tenacity.wait import wait_exponential -from kili.core.graphql import QueryOptions +from kili.client import Kili from kili.core.graphql.operations.asset.mutations import ( GQL_APPEND_MANY_ASSETS, GQL_APPEND_MANY_FRAMES_TO_DATASET, ) -from kili.core.graphql.operations.asset.queries import AssetQuery, AssetWhere from kili.core.graphql.operations.organization.queries import ( OrganizationQuery, OrganizationWhere, ) from kili.core.helpers import RetryLongWaitWarner, T, format_result, is_url from kili.core.utils import pagination +from kili.gateways.kili_api_gateway.asset.types import AssetWhere +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.orm import Asset from kili.services.asset_import.constants import ( IMPORT_BATCH_SIZE, @@ -73,7 +74,9 @@ class LoggerParams(NamedTuple): class BaseBatchImporter: # pylint: disable=too-many-instance-attributes """Base class for BatchImporters.""" - def __init__(self, kili, project_params: ProjectParams, batch_params: BatchParams, pbar: tqdm): + def __init__( + self, kili: Kili, project_params: ProjectParams, batch_params: BatchParams, pbar: tqdm + ): self.kili = kili self.project_id = project_params.project_id self.input_type = project_params.input_type @@ -124,9 +127,7 @@ def verify_batch_imported(self, assets: List): with attempt: assets_ids = [assets[-1]["id"]] # check last asset of the batch only where = AssetWhere(project_id=self.project_id, asset_id_in=assets_ids) - nb_assets_in_kili = AssetQuery( - self.kili.graphql_client, self.kili.http_client - ).count(where) + nb_assets_in_kili = self.kili.kili_api_gateway.count_assets(where) if len(assets_ids) != nb_assets_in_kili: raise BatchImportError( "Number of assets to upload is not equal to number of assets uploaded in" @@ -341,7 +342,7 @@ class BaseAbstractAssetImporter(abc.ABC): def __init__( self, - kili, + kili: Kili, project_params: ProjectParams, processing_params: ProcessingParams, logger_params: LoggerParams, @@ -443,10 +444,11 @@ def filter_duplicate_external_ids(self, assets): """Filter out assets whose external_id is already in the project.""" if len(assets) == 0: raise ImportValidationError("No assets to import") - assets_in_project = AssetQuery(self.kili.graphql_client, self.kili.http_client)( - AssetWhere(project_id=self.project_params.project_id), + assets_in_project = self.kili.kili_api_gateway.list_assets( ["externalId"], + AssetWhere(project_id=self.project_params.project_id), QueryOptions(disable_tqdm=True), + None, ) external_ids_in_project = [asset["externalId"] for asset in assets_in_project] filtered_assets = [ diff --git a/src/kili/services/copy_project/__init__.py b/src/kili/services/copy_project/__init__.py index 7bf614aa8..a8cd235ed 100644 --- a/src/kili/services/copy_project/__init__.py +++ b/src/kili/services/copy_project/__init__.py @@ -4,9 +4,10 @@ from typing import Dict, Optional from kili import services -from kili.core.graphql import QueryOptions -from kili.core.graphql.operations.asset.queries import AssetQuery, AssetWhere +from kili.client import Kili from kili.core.graphql.operations.label.queries import LabelQuery, LabelWhere +from kili.gateways.kili_api_gateway.asset.types import AssetWhere +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.use_cases.asset.media_downloader import get_download_assets_function from kili.utils.tempfile import TemporaryDirectory from kili.utils.tqdm import tqdm @@ -30,7 +31,7 @@ class ProjectCopier: # pylint: disable=too-few-public-methods "reviewCoverage", ] - def __init__(self, kili) -> None: + def __init__(self, kili: Kili) -> None: self.disable_tqdm = False self.kili = kili @@ -174,8 +175,8 @@ def download_and_upload_assets(assets): downloaded_assets = self._download_assets(from_project_id, fields, tmp_dir, assets) return self._upload_assets(new_project_id, downloaded_assets) - asset_gen = AssetQuery(self.kili.graphql_client, self.kili.http_client)( - where, fields, options, download_and_upload_assets + asset_gen = self.kili.kili_api_gateway.list_assets( + fields, where, options, download_and_upload_assets ) # Generator needs to be iterated over to actually fetch assets for _ in asset_gen: @@ -235,10 +236,11 @@ def _upload_assets(self, new_project_id, assets): # pylint: disable=too-many-locals def _copy_labels(self, from_project_id: str, new_project_id: str) -> None: - assets_new_project = AssetQuery(self.kili.graphql_client, self.kili.http_client)( - AssetWhere(project_id=new_project_id), + assets_new_project = self.kili.kili_api_gateway.list_assets( ["id", "externalId"], + AssetWhere(project_id=new_project_id), QueryOptions(disable_tqdm=True), + None, ) assets_new_project_map = {asset["externalId"]: asset["id"] for asset in assets_new_project} diff --git a/src/kili/services/data_connection/__init__.py b/src/kili/services/data_connection/__init__.py index 2221e3bfc..2d7d694f1 100644 --- a/src/kili/services/data_connection/__init__.py +++ b/src/kili/services/data_connection/__init__.py @@ -8,8 +8,7 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_exponential -from kili.core.graphql import QueryOptions -from kili.core.graphql.operations.asset.queries import AssetQuery, AssetWhere +from kili.client import Kili from kili.core.graphql.operations.data_connection.queries import ( DataConnectionIdWhere, DataConnectionQuery, @@ -19,6 +18,8 @@ GQL_COMPUTE_DATA_CONNECTION_DIFFERENCES, GQL_VALIDATE_DATA_DIFFERENCES, ) +from kili.gateways.kili_api_gateway.asset.types import AssetWhere +from kili.gateways.kili_api_gateway.queries import QueryOptions LOGGER = None @@ -61,13 +62,13 @@ def trigger_validate_data_differences( def validate_data_differences( - kili, diff_type: Literal["ADD", "REMOVE"], data_connection: Dict + kili: Kili, diff_type: Literal["ADD", "REMOVE"], data_connection: Dict ) -> None: """Call the validateDataDifferences resolver and wait until the validation is done.""" diff = data_connection["dataDifferencesSummary"]["added" if diff_type == "ADD" else "removed"] where = AssetWhere(project_id=data_connection["projectId"]) - nb_assets_before = AssetQuery(kili.graphql_client, kili.http_client).count(where) + nb_assets_before = kili.kili_api_gateway.count_assets(where) trigger_validate_data_differences(kili, diff_type, data_connection["id"]) @@ -78,7 +79,7 @@ def validate_data_differences( reraise=True, ): with attempt: - nb_assets_after = AssetQuery(kili.graphql_client, kili.http_client).count(where) + nb_assets_after = kili.kili_api_gateway.count_assets(where) if abs(nb_assets_after - nb_assets_before) != diff: raise ValueError( "Number of assets in project after validation is not correct: before" diff --git a/src/kili/services/export/format/base.py b/src/kili/services/export/format/base.py index dcb8ef138..35015990a 100644 --- a/src/kili/services/export/format/base.py +++ b/src/kili/services/export/format/base.py @@ -10,11 +10,11 @@ from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Tuple, cast -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.data_connection.queries import ( DataConnectionsQuery, DataConnectionsWhere, ) +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.orm import Asset, Label from kili.services.export.repository import AbstractContentRepository from kili.services.export.tools import fetch_assets diff --git a/src/kili/services/export/tools.py b/src/kili/services/export/tools.py index e5443f9e0..0a5467991 100644 --- a/src/kili/services/export/tools.py +++ b/src/kili/services/export/tools.py @@ -4,9 +4,9 @@ import requests -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.asset.queries import AssetQuery, AssetWhere from kili.core.helpers import validate_category_search_query +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.services.export.types import ExportType from kili.use_cases.asset.media_downloader import get_download_assets_function diff --git a/src/kili/services/helpers.py b/src/kili/services/helpers.py index f3954dcd0..d30321435 100644 --- a/src/kili/services/helpers.py +++ b/src/kili/services/helpers.py @@ -3,10 +3,11 @@ from pathlib import Path from typing import Any, Dict, Generator, Iterable, List, Optional, TypeVar -from kili.core.graphql import QueryOptions -from kili.core.graphql.operations.asset.queries import AssetQuery, AssetWhere +from kili.client import Kili from kili.core.utils import pagination from kili.exceptions import NotFound +from kili.gateways.kili_api_gateway.asset.types import AssetWhere +from kili.gateways.kili_api_gateway.queries import QueryOptions from kili.services.exceptions import ( NotEnoughArgumentsSpecifiedError, TooManyArgumentsSpecifiedError, @@ -86,15 +87,16 @@ def infer_ids_from_external_ids( return id_map -def _build_id_map(kili, asset_external_ids, project_id): +def _build_id_map(kili: Kili, asset_external_ids, project_id): assets_generators: List[Generator[Dict, None, None]] = [] # query all assets by external ids batches when there are too many for external_ids_batch in pagination.BatchIteratorBuilder(asset_external_ids, 1000): assets_generators.append( - AssetQuery(kili.graphql_client, kili.http_client)( - AssetWhere(project_id, external_id_strictly_in=external_ids_batch), + kili.kili_api_gateway.list_assets( ["id", "externalId"], + AssetWhere(project_id, external_id_strictly_in=external_ids_batch), QueryOptions(disable_tqdm=True), + None, ) ) assets = chain(*assets_generators) diff --git a/src/kili/services/project/__init__.py b/src/kili/services/project/__init__.py index e69ce24ef..cead128d8 100644 --- a/src/kili/services/project/__init__.py +++ b/src/kili/services/project/__init__.py @@ -2,9 +2,9 @@ from typing import Dict, List -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.project.queries import ProjectQuery, ProjectWhere from kili.exceptions import NotFound +from kili.gateways.kili_api_gateway.queries import QueryOptions def get_project(kili, project_id: str, fields: List[str]) -> Dict: diff --git a/src/kili/use_cases/asset/media_downloader.py b/src/kili/use_cases/asset/media_downloader.py index 45f51dc3a..e265d6500 100644 --- a/src/kili/use_cases/asset/media_downloader.py +++ b/src/kili/use_cases/asset/media_downloader.py @@ -12,12 +12,12 @@ from tenacity.stop import stop_after_attempt from tenacity.wait import wait_random -from kili.core.graphql import QueryOptions from kili.core.graphql.operations.data_connection.queries import ( DataConnectionsQuery, DataConnectionsWhere, ) from kili.gateways.kili_api_gateway import KiliAPIGateway +from kili.gateways.kili_api_gateway.queries import QueryOptions from .exceptions import DownloadNotAllowedError, MissingPropertyError