From 2b31db7c7c2b5ad1c4a25156682669d58c884ae2 Mon Sep 17 00:00:00 2001 From: Joan Antoni RE Date: Wed, 27 Sep 2023 15:28:14 +0200 Subject: [PATCH] Upgrade fastapi requirements to <=0.95.2 (#1364) * Relax and upgrade fastapi requirements to <=0.95.2 * Removing all hack * Upgrade and pin fastapi to latest version: 0.103.1 * Use lifespan instead of deprecated on_startup and on_shutdown * Upgrade min version of FastAPI for security reasons * Update examples as openapi_examples * Ignore mypy issues with grpc.aio channels * Revert "Use lifespan instead of deprecated on_startup and on_shutdown" This reverts commit 6de588f826c9776d1ff05daf7f74bbd966e47631. --- .../nucliadb/common/cluster/discovery/base.py | 4 +- .../nucliadb/common/cluster/index_node.py | 6 +-- .../common/cluster/standalone/index_node.py | 2 +- nucliadb/nucliadb/health.py | 2 +- nucliadb/nucliadb/ingest/service/__init__.py | 2 +- .../test_clean_and_upgrade_index_servicer.py | 2 +- .../servicer/test_entities_servicer.py | 2 +- .../tests/integration/servicer/test_export.py | 6 +-- .../servicer/test_knowledgebox_servicer.py | 11 ++--- .../integration/servicer/test_list_members.py | 2 +- nucliadb/nucliadb/ingest/utils.py | 3 +- nucliadb/nucliadb/search/api/v1/chat.py | 13 ++--- nucliadb/nucliadb/search/api/v1/find.py | 13 ++--- .../nucliadb/search/api/v1/resource/ask.py | 13 ++--- nucliadb/nucliadb/search/api/v1/search.py | 23 ++++----- nucliadb/nucliadb/search/requesters/utils.py | 7 +-- nucliadb/nucliadb/tests/fixtures.py | 4 +- .../train/tests/test_image_classification.py | 2 +- nucliadb/nucliadb/train/utils.py | 2 +- nucliadb/requirements.lock.txt | 47 +++++++++---------- nucliadb/requirements.txt | 4 +- nucliadb_node/nucliadb_node/reader.py | 2 +- nucliadb_node/nucliadb_node/service.py | 2 +- nucliadb_node/nucliadb_node/writer.py | 2 +- nucliadb_telemetry/nucliadb_telemetry/grpc.py | 22 +++++++-- .../nucliadb_telemetry/grpc_metrics.py | 20 ++++++-- .../nucliadb_telemetry/grpc_sentry.py | 2 +- nucliadb_utils/requirements-fastapi.txt | 4 +- 28 files changed, 127 insertions(+), 97 deletions(-) diff --git a/nucliadb/nucliadb/common/cluster/discovery/base.py b/nucliadb/nucliadb/common/cluster/discovery/base.py index 70274f7bab..8d55c1ed43 100644 --- a/nucliadb/nucliadb/common/cluster/discovery/base.py +++ b/nucliadb/nucliadb/common/cluster/discovery/base.py @@ -105,7 +105,7 @@ async def _get_index_node_metadata( else: grpc_address = f"{address}:{settings.node_writer_port}" channel = get_traced_grpc_channel(grpc_address, "discovery", variant="_writer") - stub = nodewriter_pb2_grpc.NodeWriterStub(channel) + stub = nodewriter_pb2_grpc.NodeWriterStub(channel) # type: ignore metadata: nodewriter_pb2.NodeMetadata = await stub.GetMetadata(noderesources_pb2.EmptyQuery()) # type: ignore return IndexNodeMetadata( node_id=metadata.node_id, @@ -124,7 +124,7 @@ async def _get_standalone_index_node_metadata( else: grpc_address = address channel = get_traced_grpc_channel(grpc_address, "standalone_proxy") - stub = standalone_pb2_grpc.StandaloneClusterServiceStub(channel) + stub = standalone_pb2_grpc.StandaloneClusterServiceStub(channel) # type: ignore resp: standalone_pb2.NodeInfoResponse = await stub.NodeInfo(standalone_pb2.NodeInfoRequest()) # type: ignore return IndexNodeMetadata( node_id=resp.id, diff --git a/nucliadb/nucliadb/common/cluster/index_node.py b/nucliadb/nucliadb/common/cluster/index_node.py index 5fe476cc94..0461717a9a 100644 --- a/nucliadb/nucliadb/common/cluster/index_node.py +++ b/nucliadb/nucliadb/common/cluster/index_node.py @@ -67,7 +67,7 @@ def sidecar(self) -> NodeSidecarStub: channel = get_traced_grpc_channel( grpc_address, SERVICE_NAME, variant="_sidecar" ) - SIDECAR_CONNECTIONS[self.address] = NodeSidecarStub(channel) + SIDECAR_CONNECTIONS[self.address] = NodeSidecarStub(channel) # type: ignore else: SIDECAR_CONNECTIONS[self.address] = DummySidecarStub() self._sidecar = SIDECAR_CONNECTIONS[self.address] @@ -83,7 +83,7 @@ def writer(self) -> NodeWriterStub: channel = get_traced_grpc_channel( grpc_address, SERVICE_NAME, variant="_writer" ) - WRITE_CONNECTIONS[self.address] = NodeWriterStub(channel) + WRITE_CONNECTIONS[self.address] = NodeWriterStub(channel) # type: ignore else: WRITE_CONNECTIONS[self.address] = DummyWriterStub() self._writer = WRITE_CONNECTIONS[self.address] @@ -99,7 +99,7 @@ def reader(self) -> NodeReaderStub: channel = get_traced_grpc_channel( grpc_address, SERVICE_NAME, variant="_reader" ) - READ_CONNECTIONS[self.address] = NodeReaderStub(channel) + READ_CONNECTIONS[self.address] = NodeReaderStub(channel) # type: ignore else: READ_CONNECTIONS[self.address] = DummyReaderStub() self._reader = READ_CONNECTIONS[self.address] diff --git a/nucliadb/nucliadb/common/cluster/standalone/index_node.py b/nucliadb/nucliadb/common/cluster/standalone/index_node.py index c20aa3c03e..301666bf01 100644 --- a/nucliadb/nucliadb/common/cluster/standalone/index_node.py +++ b/nucliadb/nucliadb/common/cluster/standalone/index_node.py @@ -103,7 +103,7 @@ def __init__(self, address: str, type: str, original_type: Any): else: grpc_address = address self._channel = get_traced_grpc_channel(grpc_address, "standalone_proxy") - self._stub = standalone_pb2_grpc.StandaloneClusterServiceStub(self._channel) + self._stub = standalone_pb2_grpc.StandaloneClusterServiceStub(self._channel) # type: ignore def __getattr__(self, name): async def call(request): diff --git a/nucliadb/nucliadb/health.py b/nucliadb/nucliadb/health.py index cba30723e0..89d7828983 100644 --- a/nucliadb/nucliadb/health.py +++ b/nucliadb/nucliadb/health.py @@ -108,7 +108,7 @@ async def finalizer(): async def start_grpc_health_service(port: int) -> Callable[[], Awaitable[None]]: - aio.init_grpc_aio() + aio.init_grpc_aio() # type: ignore server = aio.server() server.add_insecure_port(f"0.0.0.0:{port}") diff --git a/nucliadb/nucliadb/ingest/service/__init__.py b/nucliadb/nucliadb/ingest/service/__init__.py index 41c4d3c864..a699dd4638 100644 --- a/nucliadb/nucliadb/ingest/service/__init__.py +++ b/nucliadb/nucliadb/ingest/service/__init__.py @@ -31,7 +31,7 @@ async def start_grpc(service_name: Optional[str] = None): - aio.init_grpc_aio() + aio.init_grpc_aio() # type: ignore await setup_telemetry(service_name or "ingest") server = get_traced_grpc_server( diff --git a/nucliadb/nucliadb/ingest/tests/integration/servicer/test_clean_and_upgrade_index_servicer.py b/nucliadb/nucliadb/ingest/tests/integration/servicer/test_clean_and_upgrade_index_servicer.py index 0969885381..e2ff3c253c 100644 --- a/nucliadb/nucliadb/ingest/tests/integration/servicer/test_clean_and_upgrade_index_servicer.py +++ b/nucliadb/nucliadb/ingest/tests/integration/servicer/test_clean_and_upgrade_index_servicer.py @@ -28,7 +28,7 @@ @pytest.mark.asyncio async def test_clean_and_upgrade_kb_index(grpc_servicer: IngestFixture): - stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) + stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) # type: ignore kb_id = str(uuid4()) pb = knowledgebox_pb2.KnowledgeBoxNew(slug="test", forceuuid=kb_id) diff --git a/nucliadb/nucliadb/ingest/tests/integration/servicer/test_entities_servicer.py b/nucliadb/nucliadb/ingest/tests/integration/servicer/test_entities_servicer.py index af92268e31..9f35b3841e 100644 --- a/nucliadb/nucliadb/ingest/tests/integration/servicer/test_entities_servicer.py +++ b/nucliadb/nucliadb/ingest/tests/integration/servicer/test_entities_servicer.py @@ -30,7 +30,7 @@ async def test_create_entities_group( grpc_servicer: IngestFixture, entities_manager_mock ): - stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) + stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) # type: ignore kb_id = str(uuid4()) pb = knowledgebox_pb2.KnowledgeBoxNew(slug="test", forceuuid=kb_id) diff --git a/nucliadb/nucliadb/ingest/tests/integration/servicer/test_export.py b/nucliadb/nucliadb/ingest/tests/integration/servicer/test_export.py index 258438000c..5b3e365b58 100644 --- a/nucliadb/nucliadb/ingest/tests/integration/servicer/test_export.py +++ b/nucliadb/nucliadb/ingest/tests/integration/servicer/test_export.py @@ -53,7 +53,7 @@ @pytest.mark.asyncio async def test_export_resources(grpc_servicer: IngestFixture): - stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) + stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) # type: ignore pb = knowledgebox_pb2.KnowledgeBoxNew(slug=f"test-{uuid4()}") pb.config.title = "My Title" @@ -143,7 +143,7 @@ async def test_export_resources(grpc_servicer: IngestFixture): @pytest.mark.asyncio async def test_upload_download(grpc_servicer: IngestFixture): - stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) + stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) # type: ignore # Create a KB pb = knowledgebox_pb2.KnowledgeBoxNew(slug=f"test-{uuid4()}") @@ -183,7 +183,7 @@ async def upload_iterator(): @pytest.mark.asyncio async def test_export_file(grpc_servicer: IngestFixture): - stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) + stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) # type: ignore pb = knowledgebox_pb2.KnowledgeBoxNew(slug=f"test-{uuid4()}") pb.config.title = "My Title" diff --git a/nucliadb/nucliadb/ingest/tests/integration/servicer/test_knowledgebox_servicer.py b/nucliadb/nucliadb/ingest/tests/integration/servicer/test_knowledgebox_servicer.py index 645b92c715..5e02dc29a6 100644 --- a/nucliadb/nucliadb/ingest/tests/integration/servicer/test_knowledgebox_servicer.py +++ b/nucliadb/nucliadb/ingest/tests/integration/servicer/test_knowledgebox_servicer.py @@ -32,7 +32,7 @@ async def test_create_knowledgebox(grpc_servicer: IngestFixture, maindb_driver): if isinstance(maindb_driver, LocalDriver): pytest.skip("There is a bug in the local driver that needs to be fixed") - stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) + stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) # type: ignore pb_prefix = knowledgebox_pb2.KnowledgeBoxPrefix(prefix="") count = 0 @@ -79,8 +79,7 @@ async def get_kb_similarity(txn, kbid) -> utils_pb2.VectorSimilarity.ValueType: @pytest.mark.asyncio async def test_create_knowledgebox_with_similarity(grpc_servicer: IngestFixture): - stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) - + stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) # type: ignore pb = knowledgebox_pb2.KnowledgeBoxNew(slug="test-dot") pb.config.title = "My Title" pb.similarity = utils_pb2.VectorSimilarity.DOT @@ -98,7 +97,7 @@ async def test_create_knowledgebox_defaults_to_cosine_similarity( grpc_servicer: IngestFixture, txn, ): - stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) + stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) # type: ignore pb = knowledgebox_pb2.KnowledgeBoxNew(slug="test-default") pb.config.title = "My Title" result = await stub.NewKnowledgeBox(pb) # type: ignore @@ -113,7 +112,7 @@ async def test_create_knowledgebox_defaults_to_cosine_similarity( @pytest.mark.asyncio async def test_get_resource_id(grpc_servicer: IngestFixture) -> None: - stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) + stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) # type: ignore pb = knowledgebox_pb2.KnowledgeBoxNew(slug="test") pb.config.title = "My Title" @@ -128,7 +127,7 @@ async def test_get_resource_id(grpc_servicer: IngestFixture) -> None: async def test_delete_knowledgebox_handles_unexisting_kb( grpc_servicer: IngestFixture, ) -> None: - stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) + stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) # type: ignore pbid = knowledgebox_pb2.KnowledgeBoxID(slug="idonotexist") result = await stub.DeleteKnowledgeBox(pbid) # type: ignore diff --git a/nucliadb/nucliadb/ingest/tests/integration/servicer/test_list_members.py b/nucliadb/nucliadb/ingest/tests/integration/servicer/test_list_members.py index 5cb5aa8e54..5f582574ba 100644 --- a/nucliadb/nucliadb/ingest/tests/integration/servicer/test_list_members.py +++ b/nucliadb/nucliadb/ingest/tests/integration/servicer/test_list_members.py @@ -28,7 +28,7 @@ @pytest.mark.asyncio async def test_list_members(grpc_servicer: IngestFixture): - stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) + stub = writer_pb2_grpc.WriterStub(grpc_servicer.channel) # type: ignore response = await stub.ListMembers(ListMembersRequest()) # type: ignore diff --git a/nucliadb/nucliadb/ingest/utils.py b/nucliadb/nucliadb/ingest/utils.py index b8cd8cd797..b305ea3673 100644 --- a/nucliadb/nucliadb/ingest/utils.py +++ b/nucliadb/nucliadb/ingest/utils.py @@ -41,7 +41,8 @@ async def start_ingest(service_name: Optional[str] = None): nucliadb_settings.nucliadb_ingest, service_name or "ingest" ) set_utility(Utility.CHANNEL, channel) - set_utility(Utility.INGEST, WriterStub(channel)) + ingest = WriterStub(channel) # type: ignore + set_utility(Utility.INGEST, ingest) else: # Its not distributed create a ingest from nucliadb.ingest.service.writer import WriterServicer diff --git a/nucliadb/nucliadb/search/api/v1/chat.py b/nucliadb/nucliadb/search/api/v1/chat.py index a5953110b3..e3a4341da8 100644 --- a/nucliadb/nucliadb/search/api/v1/chat.py +++ b/nucliadb/nucliadb/search/api/v1/chat.py @@ -22,6 +22,7 @@ import pydantic from fastapi import Body, Header, Request, Response +from fastapi.openapi.models import Example from fastapi_versioning import version from starlette.responses import StreamingResponse @@ -53,13 +54,13 @@ class SyncChatResponse(pydantic.BaseModel): CHAT_EXAMPLES = { - "search_and_chat": { - "summary": "Ask who won the league final", - "description": "You can ask a question to your knowledge box", # noqa - "value": { + "search_and_chat": Example( + summary="Ask who won the league final", + description="You can ask a question to your knowledge box", # noqa + value={ "query": "Who won the league final?", }, - }, + ), } @@ -77,7 +78,7 @@ class SyncChatResponse(pydantic.BaseModel): async def chat_knowledgebox_endpoint( request: Request, kbid: str, - item: ChatRequest = Body(examples=CHAT_EXAMPLES), + item: ChatRequest = Body(openapi_examples=CHAT_EXAMPLES), x_ndb_client: NucliaDBClientType = Header(NucliaDBClientType.API), x_nucliadb_user: str = Header(""), x_forwarded_for: str = Header(""), diff --git a/nucliadb/nucliadb/search/api/v1/find.py b/nucliadb/nucliadb/search/api/v1/find.py index 5edb13a20d..a3dedd5019 100644 --- a/nucliadb/nucliadb/search/api/v1/find.py +++ b/nucliadb/nucliadb/search/api/v1/find.py @@ -22,6 +22,7 @@ from typing import List, Optional, Union from fastapi import Body, Header, Request, Response +from fastapi.openapi.models import Example from fastapi_versioning import version from pydantic.error_wrappers import ValidationError @@ -44,14 +45,14 @@ from nucliadb_utils.exceptions import LimitsExceededError FIND_EXAMPLES = { - "find_hybrid_search": { - "summary": "Do a hybrid search on a Knowledge Box", - "description": "Perform a hybrid search that will return text and semantic results matching the query", - "value": { + "find_hybrid_search": Example( + summary="Do a hybrid search on a Knowledge Box", + description="Perform a hybrid search that will return text and semantic results matching the query", + value={ "query": "How can I be an effective product manager?", "features": [SearchOptions.PARAGRAPH, SearchOptions.VECTOR], }, - } + ) } @@ -166,7 +167,7 @@ async def find_post_knowledgebox( request: Request, response: Response, kbid: str, - item: FindRequest = Body(examples=FIND_EXAMPLES), + item: FindRequest = Body(openapi_examples=FIND_EXAMPLES), x_ndb_client: NucliaDBClientType = Header(NucliaDBClientType.API), x_nucliadb_user: str = Header(""), x_forwarded_for: str = Header(""), diff --git a/nucliadb/nucliadb/search/api/v1/resource/ask.py b/nucliadb/nucliadb/search/api/v1/resource/ask.py index 061ca51edb..546c884347 100644 --- a/nucliadb/nucliadb/search/api/v1/resource/ask.py +++ b/nucliadb/nucliadb/search/api/v1/resource/ask.py @@ -20,6 +20,7 @@ from typing import Union from fastapi import Body, Header, Request, Response +from fastapi.openapi.models import Example from fastapi_versioning import version from nucliadb_protos.resources_pb2 import FieldComputedMetadata from nucliadb_protos.utils_pb2 import ExtractedText @@ -40,13 +41,13 @@ from nucliadb_utils.utilities import get_storage, has_feature ASK_EXAMPLES = { - "Ask a Resource": { - "summary": "Ask a question to the document", - "description": "Ask a question to the document. The whole document is sent as context to the generative AI", - "value": { + "Ask a Resource": Example( + summary="Ask a question to the document", + description="Ask a question to the document. The whole document is sent as context to the generative AI", + value={ "question": "Does this document contain personal information?", }, - } + ) } @@ -69,7 +70,7 @@ async def resource_ask_endpoint( kbid: str, rid: str, item: AskRequest = Body( - examples=ASK_EXAMPLES, description="Ask a question payload" + openapi_examples=ASK_EXAMPLES, description="Ask a question payload" ), x_nucliadb_user: str = Header("", description="User Id", include_in_schema=False), ) -> Union[AskResponse, HTTPClientError]: diff --git a/nucliadb/nucliadb/search/api/v1/search.py b/nucliadb/nucliadb/search/api/v1/search.py index 323d6406ae..c504f324a4 100644 --- a/nucliadb/nucliadb/search/api/v1/search.py +++ b/nucliadb/nucliadb/search/api/v1/search.py @@ -23,6 +23,7 @@ from typing import List, Optional, Tuple, Union from fastapi import Body, Header, Request, Response +from fastapi.openapi.models import Example from fastapi_versioning import version from pydantic.error_wrappers import ValidationError @@ -57,24 +58,24 @@ from nucliadb_utils.utilities import get_audit SEARCH_EXAMPLES = { - "filtering_by_icon": { - "summary": "Search for pdf documents where the text 'Noam Chomsky' appears", - "description": "For a complete list of filters, visit: https://github.com/nuclia/nucliadb/blob/main/docs/internal/SEARCH.md#filters-and-facets", # noqa - "value": { + "filtering_by_icon": Example( + summary="Search for pdf documents where the text 'Noam Chomsky' appears", + description="For a complete list of filters, visit: https://github.com/nuclia/nucliadb/blob/main/docs/internal/SEARCH.md#filters-and-facets", # noqa + value={ "query": "Noam Chomsky", "filters": ["/n/i/application/pdf"], "features": [SearchOptions.DOCUMENT], }, - }, - "get_language_counts": { - "summary": "Get the number of documents for each language", - "description": "For a complete list of facets, visit: https://github.com/nuclia/nucliadb/blob/main/docs/internal/SEARCH.md#filters-and-facets", # noqa - "value": { + ), + "get_language_counts": Example( + summary="Get the number of documents for each language", + description="For a complete list of facets, visit: https://github.com/nuclia/nucliadb/blob/main/docs/internal/SEARCH.md#filters-and-facets", # noqa + value={ "page_size": 0, "faceted": ["/s/p"], "features": [SearchOptions.DOCUMENT], }, - }, + ), } @@ -254,7 +255,7 @@ async def search_post_knowledgebox( request: Request, response: Response, kbid: str, - item: SearchRequest = Body(examples=SEARCH_EXAMPLES), + item: SearchRequest = Body(openapi_examples=SEARCH_EXAMPLES), x_ndb_client: NucliaDBClientType = Header(NucliaDBClientType.API), x_nucliadb_user: str = Header(""), x_forwarded_for: str = Header(""), diff --git a/nucliadb/nucliadb/search/requesters/utils.py b/nucliadb/nucliadb/search/requesters/utils.py index c9b1809f85..49c06b9259 100644 --- a/nucliadb/nucliadb/search/requesters/utils.py +++ b/nucliadb/nucliadb/search/requesters/utils.py @@ -214,11 +214,12 @@ def validate_node_query_results(results: list[Any]) -> Optional[HTTPException]: if isinstance(result, AioRpcError): if result.code() is GrpcStatusCode.INTERNAL: # handle node response errors - if "AllButQueryForbidden" in result.details(): + details = result.details() or "gRPC error without details" + if "AllButQueryForbidden" in details: status_code = 412 - reason = result.details().split(":")[-1].strip().strip("'") + reason = details.split(":")[-1].strip().strip("'") else: - reason = result.details() + reason = details logger.exception(f"Unhandled node error", exc_info=result) else: logger.error( diff --git a/nucliadb/nucliadb/tests/fixtures.py b/nucliadb/nucliadb/tests/fixtures.py index 580ada3889..92479d3e0a 100644 --- a/nucliadb/nucliadb/tests/fixtures.py +++ b/nucliadb/nucliadb/tests/fixtures.py @@ -191,13 +191,13 @@ async def knowledgebox(nucliadb_manager: AsyncClient): @pytest.fixture(scope="function") async def nucliadb_grpc(nucliadb: Settings): - stub = WriterStub(aio.insecure_channel(f"localhost:{nucliadb.ingest_grpc_port}")) + stub = WriterStub(aio.insecure_channel(f"localhost:{nucliadb.ingest_grpc_port}")) # type: ignore return stub @pytest.fixture(scope="function") async def nucliadb_train(nucliadb: Settings): - stub = TrainStub(aio.insecure_channel(f"localhost:{nucliadb.train_grpc_port}")) + stub = TrainStub(aio.insecure_channel(f"localhost:{nucliadb.train_grpc_port}")) # type: ignore return stub diff --git a/nucliadb/nucliadb/train/tests/test_image_classification.py b/nucliadb/nucliadb/train/tests/test_image_classification.py index cbc3cfd1f0..9105d19715 100644 --- a/nucliadb/nucliadb/train/tests/test_image_classification.py +++ b/nucliadb/nucliadb/train/tests/test_image_classification.py @@ -142,7 +142,7 @@ async def image_classification_resource( ) as _, ): resp = await nucliadb_grpc.ProcessMessage( # type: ignore - [broker_message], timeout=10, wait_for_ready=True + iter([broker_message]), timeout=10, wait_for_ready=True ) assert resp.status == OpStatusWriter.Status.OK yield diff --git a/nucliadb/nucliadb/train/utils.py b/nucliadb/nucliadb/train/utils.py index c061540c19..d960d253e5 100644 --- a/nucliadb/nucliadb/train/utils.py +++ b/nucliadb/nucliadb/train/utils.py @@ -42,7 +42,7 @@ async def start_train_grpc(service_name: Optional[str] = None): if actual_service is not None: return - aio.init_grpc_aio() + aio.init_grpc_aio() # type: ignore await setup_telemetry(service_name or "train") server = get_traced_grpc_server(service_name or "train") diff --git a/nucliadb/requirements.lock.txt b/nucliadb/requirements.lock.txt index 005ea37f43..cd3b8344b3 100644 --- a/nucliadb/requirements.lock.txt +++ b/nucliadb/requirements.lock.txt @@ -17,39 +17,39 @@ certifi==2023.7.22 cffi==1.15.1 charset-normalizer==3.2.0 click==8.1.7 -cryptography==41.0.3 +cryptography==41.0.4 Deprecated==1.2.14 ed25519==1.5 exceptiongroup==1.1.3 -fastapi==0.95.2 +fastapi==0.103.1 fastapi-versioning==0.10.0 frozenlist==1.4.0 gcloud==0.18.3 google-api-core==2.11.1 -google-api-python-client==2.97.0 -google-auth==2.22.0 -google-auth-httplib2==0.1.0 +google-api-python-client==2.100.0 +google-auth==2.23.0 +google-auth-httplib2==0.1.1 google-cloud-core==2.3.3 -google-cloud-storage==2.10.0 +google-cloud-storage==2.11.0 google-crc32c==1.5.0 -google-resumable-media==2.5.0 +google-resumable-media==2.6.0 googleapis-common-protos==1.60.0 grpc-stubs==1.53.0.2 -grpcio==1.57.0 -grpcio-channelz==1.57.0 -grpcio-health-checking==1.57.0 -grpcio-reflection==1.57.0 -grpcio-status==1.57.0 -grpcio-testing==1.57.0 -grpcio-tools==1.57.0 -h11==0.12.0 -httpcore==0.15.0 +grpcio==1.58.0 +grpcio-channelz==1.58.0 +grpcio-health-checking==1.58.0 +grpcio-reflection==1.58.0 +grpcio-status==1.58.0 +grpcio-testing==1.58.0 +grpcio-tools==1.58.0 +h11==0.14.0 +httpcore==0.18.0 httplib2==0.22.0 -httpx==0.23.0 +httpx==0.25.0 idna==3.4 jmespath==1.0.1 jwcrypto==1.5.0 -kubernetes-asyncio==24.2.3 +kubernetes-asyncio==25.11.0 lru-dict==1.2.0 memorylru==1.1.2 mmh3==4.0.1 @@ -72,7 +72,7 @@ opentelemetry-proto==1.11.1 opentelemetry-sdk==1.11.1 opentelemetry-semantic-conventions==0.30b1 opentelemetry-util-http==0.30b1 -orjson==3.9.5 +orjson==3.9.7 prometheus-client==0.17.1 protobuf==4.22.3 pyasn1==0.5.0 @@ -87,9 +87,8 @@ python-dateutil==2.8.2 PyYAML==6.0.1 redis==5.0.0 requests==2.31.0 -rfc3986==1.5.0 rsa==4.9 -sentry-sdk==1.29.2 +sentry-sdk==1.31.0 six==1.16.0 sniffio==1.3.0 starlette==0.27.0 @@ -99,10 +98,10 @@ types-aiofiles==23.2.0.0 types-orjson==3.6.2 types-pkg-resources==0.1.3 types-protobuf==3.20.4.6 -types-requests==2.31.0.2 -types-setuptools==68.1.0.0 +types-requests==2.31.0.3 +types-setuptools==68.2.0.0 types-urllib3==1.26.25.14 -typing_extensions==4.7.1 +typing_extensions==4.8.0 uritemplate==4.1.1 urllib3==1.26.16 uvicorn==0.18.3 diff --git a/nucliadb/requirements.txt b/nucliadb/requirements.txt index 97440f38f2..a1b0fd8867 100644 --- a/nucliadb/requirements.txt +++ b/nucliadb/requirements.txt @@ -18,6 +18,7 @@ grpcio-testing>=1.44.0 grpcio-reflection>=1.44.0 orjson>=3.6.7 types-setuptools +# pydantic 2 need a careful migration pydantic>=1.9.0,<2.0 aiobotocore>=2.5.2 botocore<1.30.0 @@ -27,7 +28,8 @@ oauth2client jwcrypto>=1.5.0 fastapi-versioning>=0.10.0 -fastapi>=0.75.0,<=0.95.2 +# At some point FastAPI will drop support for pydantic v1 +fastapi>=0.95.2,<=0.103.1 sentry-sdk>=1.5.12 pyjwt>=2.4.0 mmh3>=3.0.0 diff --git a/nucliadb_node/nucliadb_node/reader.py b/nucliadb_node/nucliadb_node/reader.py index 8a77788b05..517c6d6500 100644 --- a/nucliadb_node/nucliadb_node/reader.py +++ b/nucliadb_node/nucliadb_node/reader.py @@ -42,7 +42,7 @@ class Reader: def __init__(self, grpc_reader_address: str): self.lock = asyncio.Lock() self.channel = get_traced_grpc_channel(grpc_reader_address, SERVICE_NAME) - self.stub = NodeReaderStub(self.channel) + self.stub = NodeReaderStub(self.channel) # type: ignore async def get_shard(self, pb: ShardId) -> Optional[Shard]: if pb.id not in CACHE: diff --git a/nucliadb_node/nucliadb_node/service.py b/nucliadb_node/nucliadb_node/service.py index 58358447c0..ceab0c6ee1 100644 --- a/nucliadb_node/nucliadb_node/service.py +++ b/nucliadb_node/nucliadb_node/service.py @@ -31,7 +31,7 @@ async def start_grpc(writer: Writer, reader: Reader): - aio.init_grpc_aio() + aio.init_grpc_aio() # type: ignore server = get_traced_grpc_server(SERVICE_NAME) servicer = SidecarServicer(reader=reader, writer=writer) diff --git a/nucliadb_node/nucliadb_node/writer.py b/nucliadb_node/nucliadb_node/writer.py index 509962eccf..d91fe3eb5c 100644 --- a/nucliadb_node/nucliadb_node/writer.py +++ b/nucliadb_node/nucliadb_node/writer.py @@ -45,7 +45,7 @@ def __init__(self, grpc_writer_address: str): self.channel = get_traced_grpc_channel( grpc_writer_address, SERVICE_NAME, max_send_message=250 ) - self.stub = NodeWriterStub(self.channel) + self.stub = NodeWriterStub(self.channel) # type: ignore async def set_resource(self, pb: Resource) -> OpStatus: return await self.stub.SetResource(pb) # type: ignore diff --git a/nucliadb_telemetry/nucliadb_telemetry/grpc.py b/nucliadb_telemetry/nucliadb_telemetry/grpc.py index 809bdb4170..6834ea6d1f 100644 --- a/nucliadb_telemetry/nucliadb_telemetry/grpc.py +++ b/nucliadb_telemetry/nucliadb_telemetry/grpc.py @@ -220,7 +220,7 @@ async def wrapper(request: Any, context: aio.ServicerContext) -> Any: with self._set_remote_context(context): with self.start_span_server( handler_call_details, - context, + context, # type: ignore set_status_on_exception=False, ) as span: # And now we run the actual RPC. @@ -252,7 +252,10 @@ def __init__(self, tracer): self.tracer = tracer async def intercept_unary_unary( - self, continuation, client_call_details: ClientCallDetails, request + self, + continuation, + client_call_details: ClientCallDetails, # type: ignore + request, ): span = start_span_client(self.tracer, client_call_details) try: @@ -273,7 +276,10 @@ def __init__(self, tracer): self.tracer = tracer async def intercept_unary_stream( - self, continuation, client_call_details: ClientCallDetails, request + self, + continuation, + client_call_details: ClientCallDetails, # type: ignore + request, ): span = start_span_client(self.tracer, client_call_details) @@ -296,7 +302,10 @@ def __init__(self, tracer): self.tracer = tracer async def intercept_stream_stream( - self, continuation, client_call_details: ClientCallDetails, request_iterator + self, + continuation, + client_call_details: ClientCallDetails, # type: ignore + request_iterator, ): span = start_span_client(self.tracer, client_call_details) try: @@ -318,7 +327,10 @@ def __init__(self, tracer): self.tracer = tracer async def intercept_stream_unary( - self, continuation, client_call_details: ClientCallDetails, request_iterator + self, + continuation, + client_call_details: ClientCallDetails, # type: ignore + request_iterator, ): span = start_span_client(self.tracer, client_call_details) try: diff --git a/nucliadb_telemetry/nucliadb_telemetry/grpc_metrics.py b/nucliadb_telemetry/nucliadb_telemetry/grpc_metrics.py index 3888bb3a65..760cda4e56 100644 --- a/nucliadb_telemetry/nucliadb_telemetry/grpc_metrics.py +++ b/nucliadb_telemetry/nucliadb_telemetry/grpc_metrics.py @@ -99,7 +99,10 @@ def _to_str(v: Union[str, bytes]) -> str: class UnaryUnaryClientInterceptor(aio.UnaryUnaryClientInterceptor): async def intercept_unary_unary( - self, continuation, client_call_details: ClientCallDetails, request + self, + continuation, + client_call_details: ClientCallDetails, # type: ignore + request, ): metric = grpc_client_observer( labels={"method": _to_str(client_call_details.method)} @@ -114,7 +117,10 @@ async def intercept_unary_unary( class UnaryStreamClientInterceptor(aio.UnaryStreamClientInterceptor): async def intercept_unary_stream( - self, continuation, client_call_details: ClientCallDetails, request + self, + continuation, + client_call_details: ClientCallDetails, # type: ignore + request, ): metric = grpc_client_observer( labels={"method": _to_str(client_call_details.method)} @@ -129,7 +135,10 @@ async def intercept_unary_stream( class StreamStreamClientInterceptor(aio.StreamStreamClientInterceptor): async def intercept_stream_stream( - self, continuation, client_call_details: ClientCallDetails, request_iterator + self, + continuation, + client_call_details: ClientCallDetails, # type: ignore + request_iterator, ): metric = grpc_client_observer( labels={"method": _to_str(client_call_details.method)} @@ -144,7 +153,10 @@ async def intercept_stream_stream( class StreamUnaryClientInterceptor(aio.StreamUnaryClientInterceptor): async def intercept_stream_unary( - self, continuation, client_call_details: ClientCallDetails, request_iterator + self, + continuation, + client_call_details: ClientCallDetails, # type: ignore + request_iterator, ): metric = grpc_client_observer( labels={"method": _to_str(client_call_details.method)} diff --git a/nucliadb_telemetry/nucliadb_telemetry/grpc_sentry.py b/nucliadb_telemetry/nucliadb_telemetry/grpc_sentry.py index d0037db895..759f440021 100644 --- a/nucliadb_telemetry/nucliadb_telemetry/grpc_sentry.py +++ b/nucliadb_telemetry/nucliadb_telemetry/grpc_sentry.py @@ -21,8 +21,8 @@ from typing import Any, Awaitable, Callable from grpc import HandlerCallDetails, RpcMethodHandler +from grpc.experimental import aio # type: ignore from grpc.experimental import wrap_server_method_handler # type: ignore -from grpc.experimental import aio from nucliadb_telemetry.errors import capture_exception diff --git a/nucliadb_utils/requirements-fastapi.txt b/nucliadb_utils/requirements-fastapi.txt index ab038e15ae..557aec3678 100644 --- a/nucliadb_utils/requirements-fastapi.txt +++ b/nucliadb_utils/requirements-fastapi.txt @@ -1,3 +1,3 @@ -fastapi>=0.75.0,<=0.95.2 +fastapi>=0.95.2 uvicorn>=0.16.0,<0.19.0 -starlette>=0.21.0 \ No newline at end of file +starlette>=0.21.0