Skip to content

Commit

Permalink
Merge branch 'main' into event-loop/fix-hanging-due-to-multiprocessing
Browse files Browse the repository at this point in the history
  • Loading branch information
@dirkkul
dirkkul authored Nov 1, 2024
2 parents d44a585 + 6ecd809 commit 759847d
Show file tree
Hide file tree
Showing 9 changed files with 131 additions and 77 deletions.
3 changes: 2 additions & 1 deletion ci/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ services:
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
DEFAULT_VECTORIZER_MODULE: 'text2vec-contextionary'
ENABLE_MODULES: text2vec-contextionary,backup-filesystem,img2vec-neural
ENABLE_MODULES: text2vec-contextionary,backup-filesystem,img2vec-neural,generative-dummy,generative-anyscale,reranker-dummy,reranker-cohere
BACKUP_FILESYSTEM_PATH: "/tmp/backups"
CLUSTER_GOSSIP_BIND_PORT: "7100"
CLUSTER_DATA_BIND_PORT: "7101"
Expand All @@ -30,6 +30,7 @@ services:
IMAGE_INFERENCE_API: "http://i2v-neural:8080"
DISABLE_TELEMETRY: 'true'
DISABLE_LAZY_LOAD_SHARDS: 'true'
GRPC_MAX_MESSAGE_SIZE: 100000000 # 100mb
contextionary:
environment:
OCCURRENCE_WEIGHT_LINEAR_FACTOR: 0.75
Expand Down
20 changes: 10 additions & 10 deletions integration/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@
_InvertedIndexConfigCreate,
_ReferencePropertyBase,
Configure,
_GenerativeConfigCreate,
_GenerativeProvider,
_ReplicationConfigCreate,
DataType,
_MultiTenancyConfigCreate,
_VectorIndexConfigCreate,
_RerankerConfigCreate,
_RerankerProvider,
)
from weaviate.collections.classes.config_named_vectors import _NamedVectorConfigCreate
from weaviate.collections.classes.types import Properties
Expand All @@ -49,15 +49,15 @@ def __call__(
] = None,
inverted_index_config: Optional[_InvertedIndexConfigCreate] = None,
multi_tenancy_config: Optional[_MultiTenancyConfigCreate] = None,
generative_config: Optional[_GenerativeConfigCreate] = None,
generative_config: Optional[_GenerativeProvider] = None,
headers: Optional[Dict[str, str]] = None,
ports: Tuple[int, int] = (8080, 50051),
data_model_properties: Optional[Type[Properties]] = None,
data_model_refs: Optional[Type[Properties]] = None,
replication_config: Optional[_ReplicationConfigCreate] = None,
vector_index_config: Optional[_VectorIndexConfigCreate] = None,
description: Optional[str] = None,
reranker_config: Optional[_RerankerConfigCreate] = None,
reranker_config: Optional[_RerankerProvider] = None,
) -> Collection[Any, Any]:
"""Typing for fixture."""
...
Expand Down Expand Up @@ -117,15 +117,15 @@ def _factory(
] = None,
inverted_index_config: Optional[_InvertedIndexConfigCreate] = None,
multi_tenancy_config: Optional[_MultiTenancyConfigCreate] = None,
generative_config: Optional[_GenerativeConfigCreate] = None,
generative_config: Optional[_GenerativeProvider] = None,
headers: Optional[Dict[str, str]] = None,
ports: Tuple[int, int] = (8080, 50051),
data_model_properties: Optional[Type[Properties]] = None,
data_model_refs: Optional[Type[Properties]] = None,
replication_config: Optional[_ReplicationConfigCreate] = None,
vector_index_config: Optional[_VectorIndexConfigCreate] = None,
description: Optional[str] = None,
reranker_config: Optional[_RerankerConfigCreate] = None,
reranker_config: Optional[_RerankerProvider] = None,
) -> Collection[Any, Any]:
try:
nonlocal client_fixture, name_fixtures, call_counter
Expand Down Expand Up @@ -186,15 +186,15 @@ async def __call__(
] = None,
inverted_index_config: Optional[_InvertedIndexConfigCreate] = None,
multi_tenancy_config: Optional[_MultiTenancyConfigCreate] = None,
generative_config: Optional[_GenerativeConfigCreate] = None,
generative_config: Optional[_GenerativeProvider] = None,
headers: Optional[Dict[str, str]] = None,
ports: Tuple[int, int] = (8080, 50051),
data_model_properties: Optional[Type[Properties]] = None,
data_model_refs: Optional[Type[Properties]] = None,
replication_config: Optional[_ReplicationConfigCreate] = None,
vector_index_config: Optional[_VectorIndexConfigCreate] = None,
description: Optional[str] = None,
reranker_config: Optional[_RerankerConfigCreate] = None,
reranker_config: Optional[_RerankerProvider] = None,
) -> CollectionAsync[Any, Any]:
"""Typing for fixture."""
...
Expand Down Expand Up @@ -254,15 +254,15 @@ async def _factory(
] = None,
inverted_index_config: Optional[_InvertedIndexConfigCreate] = None,
multi_tenancy_config: Optional[_MultiTenancyConfigCreate] = None,
generative_config: Optional[_GenerativeConfigCreate] = None,
generative_config: Optional[_GenerativeProvider] = None,
headers: Optional[Dict[str, str]] = None,
ports: Tuple[int, int] = (8080, 50051),
data_model_properties: Optional[Type[Properties]] = None,
data_model_refs: Optional[Type[Properties]] = None,
replication_config: Optional[_ReplicationConfigCreate] = None,
vector_index_config: Optional[_VectorIndexConfigCreate] = None,
description: Optional[str] = None,
reranker_config: Optional[_RerankerConfigCreate] = None,
reranker_config: Optional[_RerankerProvider] = None,
) -> CollectionAsync[Any, Any]:
try:
nonlocal client_fixture, name_fixtures
Expand Down
22 changes: 19 additions & 3 deletions integration/test_collection_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
Vectorizers,
GenerativeSearches,
Rerankers,
_RerankerConfigCreate,
_RerankerProvider,
Tokenization,
)
from weaviate.collections.classes.tenants import Tenant
Expand Down Expand Up @@ -793,7 +793,7 @@ def test_config_vector_index_hnsw_and_quantizer_pq(collection_factory: Collectio
)
def test_config_reranker_module(
collection_factory: CollectionFactory,
reranker_config: _RerankerConfigCreate,
reranker_config: _RerankerProvider,
expected_reranker: Rerankers,
expected_model: dict,
) -> None:
Expand Down Expand Up @@ -1121,7 +1121,7 @@ def test_config_unknown_module(request: SubRequest) -> None:
client.collections.delete(name=collection_name)


def test_create_custom_module(collection_factory: CollectionFactory) -> None:
def test_create_custom_generative(collection_factory: CollectionFactory) -> None:
collection = collection_factory(
generative_config=Configure.Generative.custom(
"generative-anyscale", module_config={"temperature": 0.5}
Expand All @@ -1139,6 +1139,14 @@ def test_create_custom_module(collection_factory: CollectionFactory) -> None:
assert config.generative_config.generative == "generative-anyscale"
assert config.generative_config.model == {"temperature": 0.5}

if collection._connection._weaviate_version.is_at_least(1, 25, 24):
collection.config.update(
generative_config=Reconfigure.Generative.custom("generative-dummy"),
)
config = collection.config.get()
assert isinstance(config.generative_config.generative, str)
assert config.generative_config.generative == "generative-dummy"


def test_create_custom_reranker(collection_factory: CollectionFactory) -> None:
collection = collection_factory(
Expand All @@ -1158,6 +1166,14 @@ def test_create_custom_reranker(collection_factory: CollectionFactory) -> None:
assert config.reranker_config.reranker == "reranker-cohere"
assert config.reranker_config.model == {"model": "rerank-english-v2.0"}

if collection._connection._weaviate_version.is_at_least(1, 25, 24):
collection.config.update(
reranker_config=Reconfigure.Reranker.custom("reranker-dummy"),
)
config = collection.config.get()
assert isinstance(config.reranker_config.reranker, str)
assert config.reranker_config.reranker == "reranker-dummy"


def test_create_custom_vectorizer(collection_factory: CollectionFactory) -> None:
collection = collection_factory(
Expand Down
4 changes: 2 additions & 2 deletions requirements-devel.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,14 @@ mypy==1.12.0
mypy-extensions==1.0.0
tomli==2.0.2
types-protobuf==4.24.0.1
types-requests==2.32.0.20240914
types-requests==2.32.0.20241016
types-urllib3==1.26.25.14
typing_extensions==4.12.2

pre-commit

flake8
flake8-bugbear==24.8.19
flake8-comprehensions==3.15.0
flake8-comprehensions==3.16.0
flake8-builtins==2.5.0
black==24.10.0
8 changes: 4 additions & 4 deletions test/collection/test_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
from weaviate.collections.classes.config import (
_CollectionConfigCreate,
DataType,
_GenerativeConfigCreate,
_RerankerConfigCreate,
_GenerativeProvider,
_RerankerProvider,
_VectorizerConfigCreate,
Configure,
Property,
Expand Down Expand Up @@ -893,7 +893,7 @@ def test_config_with_vectorizer_and_properties(
TEST_CONFIG_WITH_GENERATIVE,
)
def test_config_with_generative(
generative_config: _GenerativeConfigCreate,
generative_config: _GenerativeProvider,
expected_mc: dict,
) -> None:
config = _CollectionConfigCreate(name="test", generative_config=generative_config)
Expand Down Expand Up @@ -931,7 +931,7 @@ def test_config_with_generative(

@pytest.mark.parametrize("reranker_config,expected_mc", TEST_CONFIG_WITH_RERANKER)
def test_config_with_reranker(
reranker_config: _RerankerConfigCreate,
reranker_config: _RerankerProvider,
expected_mc: dict,
) -> None:
config = _CollectionConfigCreate(name="test", reranker_config=reranker_config)
Expand Down
Loading

0 comments on commit 759847d

Please sign in to comment.