Skip to content

Commit

Permalink
fix: hack for LegacyDocument
Browse files Browse the repository at this point in the history
  • Loading branch information
JoanFM committed Mar 7, 2024
1 parent 9000f65 commit 4cc9e41
Show file tree
Hide file tree
Showing 6 changed files with 67 additions and 16 deletions.
2 changes: 2 additions & 0 deletions jina/_docarray.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

docarray_v2 = True

from jina._docarray_legacy import LegacyDocumentJina

Check warning on line 7 in jina/_docarray.py

View check run for this annotation

Codecov / codecov/patch

jina/_docarray.py#L7

Added line #L7 was not covered by tests

except ImportError:
from docarray import Document, DocumentArray

Expand Down
49 changes: 49 additions & 0 deletions jina/_docarray_legacy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
from __future__ import annotations

Check warning on line 1 in jina/_docarray_legacy.py

View check run for this annotation

Codecov / codecov/patch

jina/_docarray_legacy.py#L1

Added line #L1 was not covered by tests

from docarray import BaseDoc
from docarray import DocList

Check warning on line 4 in jina/_docarray_legacy.py

View check run for this annotation

Codecov / codecov/patch

jina/_docarray_legacy.py#L3-L4

Added lines #L3 - L4 were not covered by tests

docarray_v2 = True

Check warning on line 6 in jina/_docarray_legacy.py

View check run for this annotation

Codecov / codecov/patch

jina/_docarray_legacy.py#L6

Added line #L6 was not covered by tests

from typing import Any, Dict, Optional, List, Union

Check warning on line 8 in jina/_docarray_legacy.py

View check run for this annotation

Codecov / codecov/patch

jina/_docarray_legacy.py#L8

Added line #L8 was not covered by tests

from docarray.typing import AnyEmbedding, AnyTensor

Check warning on line 10 in jina/_docarray_legacy.py

View check run for this annotation

Codecov / codecov/patch

jina/_docarray_legacy.py#L10

Added line #L10 was not covered by tests


class LegacyDocumentJina(BaseDoc):

Check warning on line 13 in jina/_docarray_legacy.py

View check run for this annotation

Codecov / codecov/patch

jina/_docarray_legacy.py#L13

Added line #L13 was not covered by tests
"""
This Document is the LegacyDocumentJina. It follows the same schema as in DocArray <=0.21.
It can be useful to start migrating a codebase from v1 to v2.
Nevertheless, the API is not totally compatible with DocArray <=0.21 `Document`.
Indeed, none of the method associated with `Document` are present. Only the schema
of the data is similar.
```python
from docarray import DocList
from docarray.documents.legacy import LegacyDocument
import numpy as np
doc = LegacyDocument(text='hello')
doc.url = 'http://myimg.png'
doc.tensor = np.zeros((3, 224, 224))
doc.embedding = np.zeros((100, 1))
doc.tags['price'] = 10
doc.chunks = DocList[Document]([Document() for _ in range(10)])
doc.chunks = DocList[Document]([Document() for _ in range(10)])
```
"""

tensor: Optional[AnyTensor] = None
chunks: Optional[Union[DocList[LegacyDocumentJina], List[LegacyDocumentJina]]] = None
matches: Optional[Union[DocList[LegacyDocumentJina], List[LegacyDocumentJina]]] = None
blob: Optional[bytes] = None
text: Optional[str] = None
url: Optional[str] = None
embedding: Optional[AnyEmbedding] = None
tags: Dict[str, Any] = dict()
scores: Optional[Dict[str, Any]] = None

Check warning on line 49 in jina/_docarray_legacy.py

View check run for this annotation

Codecov / codecov/patch

jina/_docarray_legacy.py#L41-L49

Added lines #L41 - L49 were not covered by tests
4 changes: 2 additions & 2 deletions jina/serve/executors/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
from jina.serve.instrumentation import MetricsTimer

if docarray_v2:
from docarray.documents.legacy import LegacyDocument
from jina._docarray import LegacyDocumentJina

Check warning on line 55 in jina/serve/executors/__init__.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/executors/__init__.py#L55

Added line #L55 was not covered by tests

if TYPE_CHECKING: # pragma: no cover
from opentelemetry.context.context import Context
Expand Down Expand Up @@ -257,7 +257,7 @@ def get_function_with_schema(fn: Callable) -> T:
from docarray import BaseDoc, DocList

default_annotations = (
DocList[LegacyDocument] if is_batch_docs else LegacyDocument
DocList[LegacyDocumentJina] if is_batch_docs else LegacyDocumentJina
)
else:
from jina import Document, DocumentArray
Expand Down
10 changes: 4 additions & 6 deletions jina/serve/runtimes/gateway/graph/topology_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@

if docarray_v2:
from docarray import DocList
from docarray.documents.legacy import LegacyDocument
from jina._docarray import LegacyDocumentJina

Check warning on line 21 in jina/serve/runtimes/gateway/graph/topology_graph.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/gateway/graph/topology_graph.py#L21

Added line #L21 was not covered by tests

if not is_pydantic_v2:
from jina.serve.runtimes.helper import _create_pydantic_model_from_schema as create_base_doc_from_schema

Check warning on line 24 in jina/serve/runtimes/gateway/graph/topology_graph.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/gateway/graph/topology_graph.py#L23-L24

Added lines #L23 - L24 were not covered by tests
else:
from docarray.utils.create_dynamic_doc_class import create_base_doc_from_schema

Check warning on line 26 in jina/serve/runtimes/gateway/graph/topology_graph.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/gateway/graph/topology_graph.py#L26

Added line #L26 was not covered by tests


legacy_doc_schema = LegacyDocument.schema()
legacy_doc_schema = LegacyDocumentJina.schema()

Check warning on line 29 in jina/serve/runtimes/gateway/graph/topology_graph.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/gateway/graph/topology_graph.py#L29

Added line #L29 was not covered by tests


class TopologyGraph:
Expand Down Expand Up @@ -222,8 +222,6 @@ async def task():
endp, _ = endpoints_proto
self.endpoints = endp.endpoints
if docarray_v2:
from docarray.documents.legacy import LegacyDocument

schemas = json_format.MessageToDict(endp.schemas)
self._pydantic_models_by_endpoint = {}
models_created_by_name = {}
Expand All @@ -240,7 +238,7 @@ async def task():
else:
if input_model_name not in models_created_by_name:
if input_model_schema == legacy_doc_schema:
input_model = LegacyDocument
input_model = LegacyDocumentJina

Check warning on line 241 in jina/serve/runtimes/gateway/graph/topology_graph.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/gateway/graph/topology_graph.py#L241

Added line #L241 was not covered by tests
else:
input_model = (
create_base_doc_from_schema(
Expand Down Expand Up @@ -270,7 +268,7 @@ async def task():
else:
if output_model_name not in models_created_by_name:
if output_model_name == legacy_doc_schema:
output_model = LegacyDocument
output_model = LegacyDocumentJina

Check warning on line 271 in jina/serve/runtimes/gateway/graph/topology_graph.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/gateway/graph/topology_graph.py#L271

Added line #L271 was not covered by tests
else:
output_model = (
create_base_doc_from_schema(
Expand Down
11 changes: 6 additions & 5 deletions jina/serve/runtimes/head/request_handling.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@
from docarray import DocList
from docarray.base_doc.any_doc import AnyDoc

from jina._docarray import LegacyDocumentJina

Check warning on line 29 in jina/serve/runtimes/head/request_handling.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/head/request_handling.py#L29

Added line #L29 was not covered by tests

legacy_doc_schema = LegacyDocumentJina.schema()

Check warning on line 31 in jina/serve/runtimes/head/request_handling.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/head/request_handling.py#L31

Added line #L31 was not covered by tests


if TYPE_CHECKING: # pragma: no cover
from prometheus_client import CollectorRegistry
Expand Down Expand Up @@ -333,9 +337,6 @@ def _get_endpoints_from_workers(
self, connection_pool: GrpcConnectionPool, name: str, retries: int, stop_event
):
from google.protobuf import json_format
from docarray.documents.legacy import LegacyDocument

legacy_doc_schema = LegacyDocument.schema()

async def task():
self.logger.debug(
Expand All @@ -359,7 +360,7 @@ async def task():

if input_model_schema == legacy_doc_schema:
models_created_by_name[input_model_name] = (
LegacyDocument
LegacyDocumentJina
)
elif input_model_name not in models_created_by_name:
input_model = create_base_doc_from_schema(

Check warning on line 366 in jina/serve/runtimes/head/request_handling.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/head/request_handling.py#L366

Added line #L366 was not covered by tests
Expand All @@ -369,7 +370,7 @@ async def task():

if output_model_name == legacy_doc_schema:
models_created_by_name[output_model_name] = (
LegacyDocument
LegacyDocumentJina
)
elif output_model_name not in models_created_by_name:
output_model = create_base_doc_from_schema(

Check warning on line 376 in jina/serve/runtimes/head/request_handling.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/head/request_handling.py#L376

Added line #L376 was not covered by tests
Expand Down
7 changes: 4 additions & 3 deletions jina/serve/runtimes/worker/request_handling.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@

if docarray_v2:
from docarray import DocList
from jina._docarray import LegacyDocumentJina
legacy_doc_schema = LegacyDocumentJina.schema()

Check warning on line 37 in jina/serve/runtimes/worker/request_handling.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/worker/request_handling.py#L36-L37

Added lines #L36 - L37 were not covered by tests


if TYPE_CHECKING: # pragma: no cover
import grpc
Expand Down Expand Up @@ -1011,14 +1014,12 @@ async def endpoint_discovery(self, empty, context) -> jina_pb2.EndpointsProto:
endpoints_proto.write_endpoints.extend(list(self._executor.write_endpoints))
schemas = self._executor._get_endpoint_models_dict()
if docarray_v2:
from docarray.documents.legacy import LegacyDocument

if not is_pydantic_v2:
from jina.serve.runtimes.helper import _create_aux_model_doc_list_to_list as create_pure_python_type_model

Check warning on line 1018 in jina/serve/runtimes/worker/request_handling.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/worker/request_handling.py#L1017-L1018

Added lines #L1017 - L1018 were not covered by tests
else:
from docarray.utils.create_dynamic_doc_class import create_pure_python_type_model

Check warning on line 1020 in jina/serve/runtimes/worker/request_handling.py

View check run for this annotation

Codecov / codecov/patch

jina/serve/runtimes/worker/request_handling.py#L1020

Added line #L1020 was not covered by tests

legacy_doc_schema = LegacyDocument.schema()

for endpoint_name, inner_dict in schemas.items():
if inner_dict['input']['model'].schema() == legacy_doc_schema:
inner_dict['input']['model'] = legacy_doc_schema
Expand Down

0 comments on commit 4cc9e41

Please sign in to comment.