Bump odd-collector-sdk dependencies. Migrate pydantic v1 -> v2

odd-collector-sdk/odd_collector_sdk/domain/collector_config.py

@@ -2,15 +2,15 @@
 from pathlib import Path
 from typing import Dict, List, Optional, Type, Union
 
-import pydantic
+from pydantic_settings import BaseSettings
 
 from ..errors import LoadConfigError
 from ..logger import logger
 from ..utils.yaml_parser import parse_yaml as parse_config
 from .plugin import Plugin
 
 
-class CollectorConfig(pydantic.BaseSettings):
+class CollectorConfig(BaseSettings):
     default_pulling_interval: Optional[int] = None  # minutes
     connection_timeout_seconds: int = 300
     token: str
@@ -19,7 +19,7 @@ class CollectorConfig(pydantic.BaseSettings):
     chunk_size: int = 250
     misfire_grace_time: Optional[
         int
-    ]  # seconds after the designated runtime that the job is still allowed to be run
+    ] = None  # seconds after the designated runtime that the job is still allowed to be run
     max_instances: Optional[
         int
     ] = 1  # maximum number of concurrently running instances allowed
@@ -39,10 +39,10 @@ def load_config(
         with open(config_path) as f:
             parsed = parse_config(f)
             parsed["plugins"] = [
-                plugin_factory[plugin["type"]].parse_obj(plugin)
+                plugin_factory[plugin["type"]].model_validate(plugin)
                 for plugin in parsed["plugins"]
             ]
 
-            return CollectorConfig.parse_obj(parsed)
+            return CollectorConfig.model_validate(parsed)
     except Exception as e:
         raise LoadConfigError(e)

odd-collector-sdk/odd_collector_sdk/domain/collector_config_loader.py

@@ -48,10 +48,10 @@ def _build_collector_config(self):
             plugins = self._merge_plugins(sb_provider.get_plugins(), plugins)
 
         plugins = [
-            self.plugin_factory[plugin["type"]].parse_obj(plugin) for plugin in plugins
+            self.plugin_factory[plugin["type"]].model_validate(plugin) for plugin in plugins
         ]
 
-        return CollectorConfig.parse_obj({**collector_settings, "plugins": plugins})
+        return CollectorConfig.model_validate({**collector_settings, "plugins": plugins})
 
     def _parse_config(self) -> dict:
         """

odd-collector-sdk/odd_collector_sdk/domain/filter.py

@@ -1,15 +1,15 @@
 import re
 
 from funcy import partial
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, Field, field_validator
 
 
 class Filter(BaseModel):
-    include: list[str] = []
+    include: list[str] = Field(default=[], validate_default=True)
     exclude: list[str] = []
     ignore_case: bool = False
 
-    @validator("include", always=True)
+    @field_validator("include")
     def check_include(cls, include):
         if not include:
             return [".*"]

odd-collector-sdk/odd_collector_sdk/domain/plugin.py

@@ -1,9 +1,9 @@
 from typing import Optional
 
-from pydantic import BaseSettings, Extra
+from pydantic_settings import BaseSettings
 
 
-class Plugin(BaseSettings, extra=Extra.allow):
+class Plugin(BaseSettings, extra="allow"):
     type: str
     name: str
     description: Optional[str] = None

odd-collector-sdk/odd_collector_sdk/secrets/secrets_backend_factory.py

@@ -4,14 +4,14 @@
     AWSSystemsManagerParameterStoreBackend,
 )
 from odd_collector_sdk.secrets.base_secrets import BaseSecretsBackend
-from pydantic import BaseSettings, Extra
+from pydantic_settings import BaseSettings
 
 PROVIDERS = {
     "AWSSystemsManagerParameterStore": AWSSystemsManagerParameterStoreBackend,
 }
 
 
-class SecretsBackendSettings(BaseSettings, extra=Extra.allow):
+class SecretsBackendSettings(BaseSettings, extra="allow"):
     provider: Literal["AWSSystemsManagerParameterStore"]
 
 

odd-collector-sdk/pyproject.toml

@@ -13,22 +13,24 @@ authors = ["Open Data Discovery <[email protected]>", "Pavel Makarichev
 [tool.poetry.dependencies]
 python = "^3.9"
 APScheduler = "^3.8.1"
-pydantic = "^1.10.14"
+pydantic = "^2.7.0"
 aiohttp = "^3.8.1"
 pyaml-env = "^1.1.5"
-odd-models = "^2.0.44"
-importlib-metadata = "^5.1.0"
+odd-models = "^2.0.47"
+importlib-metadata = "^7.1.0"
 tqdm = "^4.64.1"
 funcy = "^2.0"
-loguru = "^0.6.0"
-oddrn-generator = "^0.1.99"
+loguru = "^0.7.2"
+oddrn-generator = "^0.1.101"
 flatdict = "^4.0.1"
 prettytable = "^3.8.0"
 lark-parser = "^0.12.0"
-boto3="1.28.70"
+boto3="^1.34.88"
+botocore = "^1.34.88"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.4.0"
+pytest = "^8.1.1"
+setuptools = "^69.5.1"
 
 
 [build-system]

odd-collector-sdk/tests/generator.py

@@ -1,18 +1,24 @@
 from typing import Optional
 
 from oddrn_generator import Generator
-from oddrn_generator.path_models import BasePathsModel
+from oddrn_generator.path_models import BasePathsModel, DependenciesMap
 from oddrn_generator.server_models import HostnameModel
+from pydantic import Field
 
 
 class TestPathModel(BasePathsModel):
-    databases: Optional[str]
+    databases: Optional[str] = None
 
-    class Config:
-        dependencies_map = {
+    @classmethod
+    def _dependencies_map_factory(cls):
+        return {
             "databases": ("databases",),
         }
 
+    dependencies_map: DependenciesMap = Field(
+        default_factory=lambda: TestPathModel._dependencies_map_factory()
+    )
+
 
 class TestGenerator(Generator):
     source = "test"

odd-collector-sdk/tests/utils/test_metadata.py

@@ -92,7 +92,7 @@ def test_extract_metadata_from_class_without_metadata_field(partial_extract_meta
     assert isinstance(metadata, MetadataExtension)
     assert metadata.metadata == {}
 
-    assert metadata.json()
+    assert metadata.model_dump_json()
 
 
 def test_extract_metadata_from_dataclass_without_metadata_field(
@@ -104,7 +104,7 @@ def test_extract_metadata_from_dataclass_without_metadata_field(
     assert isinstance(metadata, MetadataExtension)
     assert metadata.metadata == {}
 
-    assert metadata.json()
+    assert metadata.model_dump_json()
 
 
 def test_extract_metadata_from_pydantic_without_metadata_field(
@@ -116,7 +116,7 @@ def test_extract_metadata_from_pydantic_without_metadata_field(
     assert isinstance(metadata, MetadataExtension)
     assert metadata.metadata == {}
 
-    assert metadata.json()
+    assert metadata.model_dump_json()
 
 
 def test_extract_metadata_from_class_with_metadata_field(partial_extract_metadata):
@@ -126,7 +126,7 @@ def test_extract_metadata_from_class_with_metadata_field(partial_extract_metadat
     assert isinstance(metadata, MetadataExtension)
     assert metadata.metadata == entity.odd_metadata
 
-    assert metadata.json()
+    assert metadata.model_dump_json()
 
 
 def test_extract_metadata_from_class_with_metadata_added_at_runtime(
@@ -139,7 +139,7 @@ def test_extract_metadata_from_class_with_metadata_added_at_runtime(
     assert isinstance(metadata, MetadataExtension)
     assert metadata.metadata == entity.odd_metadata
 
-    assert metadata.json()
+    assert metadata.model_dump_json()
 
 
 def test_extract_metadata_from_dataclass_with_metadata_field(partial_extract_metadata):
@@ -151,7 +151,7 @@ def test_extract_metadata_from_dataclass_with_metadata_field(partial_extract_met
     assert isinstance(metadata, MetadataExtension)
     assert metadata.metadata == entity.odd_metadata
 
-    assert metadata.json()
+    assert metadata.model_dump_json()
 
 
 def test_extract_metadata_from_dataclass_with_metadata_field_added_at_runtime(
@@ -164,7 +164,7 @@ def test_extract_metadata_from_dataclass_with_metadata_field_added_at_runtime(
     assert isinstance(metadata, MetadataExtension)
     assert metadata.metadata == entity.odd_metadata
 
-    assert metadata.json()
+    assert metadata.model_dump_json()
 
 
 def test_extract_metadata_from_pydantic_class_with_metadata_field(
@@ -178,7 +178,7 @@ def test_extract_metadata_from_pydantic_class_with_metadata_field(
     assert isinstance(metadata, MetadataExtension)
     assert metadata.metadata == entity.odd_metadata
 
-    assert metadata.json()
+    assert metadata.model_dump_json()
 
 
 def test_extract_metadata_from_class_with_an_empty_with_complex(
@@ -188,7 +188,7 @@ def test_extract_metadata_from_class_with_an_empty_with_complex(
     metadata = partial_extract_metadata(entity=entity)
 
     assert metadata.metadata == COMPLEX_METADATA
-    assert metadata.json()
+    assert metadata.model_dump_json()
 
 
 def test_extract_flatten_metadata_from_class_with_an_empty_with_complex(
@@ -198,7 +198,7 @@ def test_extract_flatten_metadata_from_class_with_an_empty_with_complex(
     metadata = partial_extract_metadata(entity=entity, flatten=True)
 
     assert metadata.metadata == FLATTEN
-    assert metadata.json()
+    assert metadata.model_dump_json()
 
 
 def test_extract_jsonfy_metadata_from_class_with_an_empty_with_complex(
@@ -208,7 +208,7 @@ def test_extract_jsonfy_metadata_from_class_with_an_empty_with_complex(
     metadata = partial_extract_metadata(entity=entity, jsonify=True)
 
     assert metadata.metadata == ENCODED_METADATA
-    assert metadata.json()
+    assert metadata.model_dump_json()
 
 
 def test_extract_flatten_jsonfy_metadata_from_class_with_an_empty_with_complex(
@@ -218,7 +218,7 @@ def test_extract_flatten_jsonfy_metadata_from_class_with_an_empty_with_complex(
     metadata = partial_extract_metadata(entity=entity, flatten=True, jsonify=True)
 
     assert metadata.metadata == FLATTEN_ENCODED_METADATA
-    assert metadata.json()
+    assert metadata.model_dump_json()
 
 
 def test_filter_none_values(partial_extract_metadata):