Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

renames diffsync.DiffSync to diffsync.Adapter #242

Merged
merged 3 commits into from
Aug 31, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -130,16 +130,18 @@ jobs:
fail-fast: true
matrix:
python-version: ["3.7", "3.8", "3.9", "3.10"]
poetry-version: ["1.5.1"]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do you expect to parametrize test by poetry? why not pin it?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the act of pinning it, without this its latest

runs-on: "ubuntu-20.04"
env:
PYTHON_VER: "${{ matrix.python-version }}"
steps:
- name: "Check out repository code"
uses: "actions/checkout@v2"
- name: "Setup environment"
uses: "networktocode/gh-action-setup-poetry-environment@v5"
uses: "networktocode/gh-action-setup-poetry-environment@3ea5d3ecf382cdcb0c74d4c0ff0629d95fce63c7"
with:
python-version: "${{ matrix.python-version }}"
poetry-version: "${{ matrix.poetry-version }}"
- name: "Install redis"
run: "sudo apt-get install -y redis"
- name: "Run poetry Install"
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/*

RUN pip install --upgrade pip \
&& pip install poetry
&& pip install poetry==1.5.1


WORKDIR /local
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ DiffSync is at its most useful when you have multiple sources or sets of data to

# Overview of DiffSync

DiffSync acts as an intermediate translation layer between all of the data sets you are diffing and/or syncing. In practical terms, this means that to use DiffSync, you will define a set of data models as well as the “adapters” needed to translate between each base data source and the data model. In Python terms, the adapters will be subclasses of the `DiffSync` class, and each data model class will be a subclass of the `DiffSyncModel` class.
DiffSync acts as an intermediate translation layer between all of the data sets you are diffing and/or syncing. In practical terms, this means that to use DiffSync, you will define a set of data models as well as the “adapters” needed to translate between each base data source and the data model. In Python terms, the adapters will be subclasses of the `Adapter` class, and each data model class will be a subclass of the `DiffSyncModel` class.

![Diffsync Components](https://raw.githubusercontent.com/networktocode/diffsync/develop/docs/images/diffsync_components.png "Diffsync Components")

Expand Down
21 changes: 12 additions & 9 deletions diffsync/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ class DiffSyncModel(BaseModel):
Can be set as a class attribute or an instance attribute as needed.
"""

diffsync: Optional["DiffSync"] = None
diffsync: Optional["Adapter"] = None
"""Optional: the DiffSync instance that owns this model instance."""

_status: DiffSyncStatus = PrivateAttr(DiffSyncStatus.SUCCESS)
Expand Down Expand Up @@ -183,7 +183,7 @@ def set_status(self, status: DiffSyncStatus, message: StrType = "") -> None:
self._status_message = message

@classmethod
def create_base(cls, diffsync: "DiffSync", ids: Dict, attrs: Dict) -> Optional[Self]:
def create_base(cls, diffsync: "Adapter", ids: Dict, attrs: Dict) -> Optional[Self]:
"""Instantiate this class, along with any platform-specific data creation.

This method is not meant to be subclassed, users should redefine create() instead.
Expand All @@ -201,7 +201,7 @@ def create_base(cls, diffsync: "DiffSync", ids: Dict, attrs: Dict) -> Optional[S
return model

@classmethod
def create(cls, diffsync: "DiffSync", ids: Dict, attrs: Dict) -> Optional[Self]:
def create(cls, diffsync: "Adapter", ids: Dict, attrs: Dict) -> Optional[Self]:
"""Instantiate this class, along with any platform-specific data creation.

Subclasses must call `super().create()` or `self.create_base()`; they may wish to then override the default status information
Expand Down Expand Up @@ -402,7 +402,7 @@ def remove_child(self, child: "DiffSyncModel") -> None:
childs.remove(child.get_unique_id())


class DiffSync: # pylint: disable=too-many-public-methods
class Adapter: # pylint: disable=too-many-public-methods
"""Class for storing a group of DiffSyncModel instances and diffing/synchronizing to another DiffSync instance."""

# In any subclass, you would add mapping of names to specific model classes here:
Expand Down Expand Up @@ -535,7 +535,7 @@ def load_from_dict(self, data: Dict) -> None:

def sync_from( # pylint: disable=too-many-arguments
self,
source: "DiffSync",
source: "Adapter",
diff_class: Type[Diff] = Diff,
flags: DiffSyncFlags = DiffSyncFlags.NONE,
callback: Optional[Callable[[StrType, int, int], None]] = None,
Expand Down Expand Up @@ -573,7 +573,7 @@ def sync_from( # pylint: disable=too-many-arguments

def sync_to( # pylint: disable=too-many-arguments
self,
target: "DiffSync",
target: "Adapter",
diff_class: Type[Diff] = Diff,
flags: DiffSyncFlags = DiffSyncFlags.NONE,
callback: Optional[Callable[[StrType, int, int], None]] = None,
Expand All @@ -597,7 +597,7 @@ def sync_to( # pylint: disable=too-many-arguments

def sync_complete(
self,
source: "DiffSync",
source: "Adapter",
diff: Diff,
flags: DiffSyncFlags = DiffSyncFlags.NONE,
logger: Optional[structlog.BoundLogger] = None,
Expand All @@ -623,7 +623,7 @@ def sync_complete(

def diff_from(
self,
source: "DiffSync",
source: "Adapter",
diff_class: Type[Diff] = Diff,
flags: DiffSyncFlags = DiffSyncFlags.NONE,
callback: Optional[Callable[[StrType, int, int], None]] = None,
Expand All @@ -644,7 +644,7 @@ def diff_from(

def diff_to(
self,
target: "DiffSync",
target: "Adapter",
diff_class: Type[Diff] = Diff,
flags: DiffSyncFlags = DiffSyncFlags.NONE,
callback: Optional[Callable[[StrType, int, int], None]] = None,
Expand Down Expand Up @@ -854,5 +854,8 @@ def count(self, model: Union[StrType, "DiffSyncModel", Type["DiffSyncModel"], No
return self.store.count(model=model)


# For backwards-compatibility, keep around the old name
DiffSync = Adapter

# DiffSyncModel references DiffSync and DiffSync references DiffSyncModel. Break the typing loop:
DiffSyncModel.update_forward_refs()
10 changes: 5 additions & 5 deletions diffsync/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@

if TYPE_CHECKING: # pragma: no cover
# For type annotation purposes, we have a circular import loop between __init__.py and this file.
from . import DiffSync, DiffSyncModel # pylint: disable=cyclic-import
from . import Adapter, DiffSyncModel # pylint: disable=cyclic-import


class DiffSyncDiffer: # pylint: disable=too-many-instance-attributes
Expand All @@ -37,8 +37,8 @@ class DiffSyncDiffer: # pylint: disable=too-many-instance-attributes

def __init__( # pylint: disable=too-many-arguments
self,
src_diffsync: "DiffSync",
dst_diffsync: "DiffSync",
src_diffsync: "Adapter",
dst_diffsync: "Adapter",
flags: DiffSyncFlags,
diff_class: Type[Diff] = Diff,
callback: Optional[Callable[[str, int, int], None]] = None,
Expand Down Expand Up @@ -288,8 +288,8 @@ class DiffSyncSyncer: # pylint: disable=too-many-instance-attributes
def __init__( # pylint: disable=too-many-arguments
self,
diff: Diff,
src_diffsync: "DiffSync",
dst_diffsync: "DiffSync",
src_diffsync: "Adapter",
dst_diffsync: "Adapter",
flags: DiffSyncFlags,
callback: Optional[Callable[[str, int, int], None]] = None,
):
Expand Down
4 changes: 2 additions & 2 deletions diffsync/store/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

if TYPE_CHECKING:
from diffsync import DiffSyncModel
from diffsync import DiffSync
from diffsync import Adapter


class BaseStore:
Expand All @@ -15,7 +15,7 @@ class BaseStore:
def __init__(
self, # pylint: disable=unused-argument
*args: Any, # pylint: disable=unused-argument
diffsync: Optional["DiffSync"] = None,
diffsync: Optional["Adapter"] = None,
name: str = "",
**kwargs: Any, # pylint: disable=unused-argument
) -> None:
Expand Down
4 changes: 2 additions & 2 deletions docs/source/core_engine/01-flags.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,12 @@ diff = nautobot.diff_from(local, flags=flags)
Model flags are stored in the attribute `model_flags` of each model and are usually set when the data is being loaded into the adapter.

```python
from diffsync import DiffSync
from diffsync import Adapter
from diffsync.enum import DiffSyncModelFlags
from model import MyDeviceModel

class MyAdapter(DiffSync):

class MyAdapter(Adapter):
device = MyDeviceModel

def load(self, data):
Expand Down
28 changes: 16 additions & 12 deletions docs/source/core_engine/03-store.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,32 +2,36 @@

By default, `Diffsync` supports a local memory storage. All the loaded models from the adapters will be stored in memory, and become available for the diff calculation and sync process. This default behavior works well when executing all the steps in the same process, having access to the same memory space. However, if you want to scale out the execution of the tasks, running it in different processes or in totally different workers, a more distributed memory support is necessary.

The `store` is a class attribute in the `DiffSync` class, but all the store operations in that class are abstracted in the following methods: `get_all_model_names`, `get`, `get_by_uids`, `add`, `update`, `remove`, `get_or_instantiate`, `update_or_instantiate` and `count`.
The `store` is a class attribute in the `Adapter` class, but all the store operations in that class are abstracted in the following methods: `get_all_model_names`, `get`, `get_by_uids`, `add`, `update`, `remove`, `get_or_instantiate`, `update_or_instantiate` and `count`.

## Use the `LocalStore` Backend

When you initialize the `Diffsync` Adapter class, there is an optional keyed-argument, `internal_storage_engine`, defaulting to the `LocalStore` class.

```python
>>> from diffsync import DiffSync
>>> adapter = DiffSync()
>>> type(adapter.store)
<class 'diffsync.store.local.LocalStore'>
>> > from diffsync import Adapter
>> > adapter = Adapter()
>> > type(adapter.store)
<

class 'diffsync.store.local.LocalStore'>
```

## Use the `RedisStore` Backend

To get it, you have to install diffsync package with the "redis" extra option: `pip install diffsync[redis]`

The `RedisStore` backend, as the name suggests, connects to an external Redis service, to store data loaded by the `DiffSync` tasks. The biggest change is that it requires to initialize the Redis store class, before using it in the `DiffSync` adapter class.
The `RedisStore` backend, as the name suggests, connects to an external Redis service, to store data loaded by the `Adapter` tasks. The biggest change is that it requires to initialize the Redis store class, before using it in the `Adapter` adapter class.

```python
>>> from diffsync import DiffSync
>>> from diffsync.store.redis import RedisStore
>>> store = RedisStore(host="redis host")
>>> adapter = DiffSync(internal_storage_engine=store)
>>> type(adapter.store)
<class 'diffsync.store.local.RedisStore'>
>> > from diffsync import Adapter
>> > from diffsync.store.redis import RedisStore
>> > store = RedisStore(host="redis host")
>> > adapter = Adapter(internal_storage_engine=store)
>> > type(adapter.store)
<

class 'diffsync.store.local.RedisStore'>
```

Notice that the `RedisStore` will validate, when initialized, that there is a reachability to the Redis host, and if not, will raise an exception:
Expand Down
16 changes: 8 additions & 8 deletions docs/source/getting_started/01-getting-started.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
To be able to properly compare different datasets, DiffSync relies on a shared data model that both systems must use.
Specifically, each system or dataset must provide a `DiffSync` "adapter" subclass, which in turn represents its dataset as instances of one or more `DiffSyncModel` data model classes.
Specifically, each system or dataset must provide a `Adapter` "adapter" subclass, which in turn represents its dataset as instances of one or more `DiffSyncModel` data model classes.

When comparing two systems, DiffSync detects the intersection between the two systems (which data models they have in common, and which attributes are shared between each pair of data models) and uses this intersection to compare and/or synchronize the data.

Expand Down Expand Up @@ -41,24 +41,24 @@ Currently the relationships between models are very loose by design. Instead of

# Define your system adapter with DiffSync

A `DiffSync` "adapter" subclass must reference each model available at the top of the object by its modelname and must have a `top_level` attribute defined to indicate how the diff and the synchronization should be done. In the example below, `"site"` is the only top level object so the synchronization engine will only check all known `Site` instances and all children of each Site. In this case, as shown in the code above, `Device`s are children of `Site`s, so this is exactly the intended logic.
A `Adapter` "adapter" subclass must reference each model available at the top of the object by its modelname and must have a `top_level` attribute defined to indicate how the diff and the synchronization should be done. In the example below, `"site"` is the only top level object so the synchronization engine will only check all known `Site` instances and all children of each Site. In this case, as shown in the code above, `Device`s are children of `Site`s, so this is exactly the intended logic.

```python
from diffsync import DiffSync
from diffsync import Adapter

class BackendA(DiffSync):

class BackendA(Adapter):
site = Site
device = Device

top_level = ["site"]
```

It's up to the implementer to populate the `DiffSync`'s internal cache with the appropriate data. In the example below we are using the `load()` method to populate the cache but it's not mandatory, it could be done differently.
It's up to the implementer to populate the `Adapter`'s internal cache with the appropriate data. In the example below we are using the `load()` method to populate the cache but it's not mandatory, it could be done differently.

## Model Processing Ordering Logic

The models will be processed in a specfic order as defined by `top_level` atttribute on the `DiffSync` object and then the `_children` attribute on the `DiffSyncModel`. The processing algorithm is technically a "Preorder Tree Traversal", which means that "a parent node is processed before any of its child nodes is done." This can be described as:
The models will be processed in a specfic order as defined by `top_level` atttribute on the `Adapter` object and then the `_children` attribute on the `DiffSyncModel`. The processing algorithm is technically a "Preorder Tree Traversal", which means that "a parent node is processed before any of its child nodes is done." This can be described as:

- Start with the first element of the first model in `top_level` and process it.
- If that model has `_children` set on it, for each child of each child model, in order:
Expand Down Expand Up @@ -145,7 +145,7 @@ NetworkImporterAdapter
>>>
```

# Store data in a `DiffSync` object
# Store data in a `Adapter` object

To add a site to the local cache/store, you need to pass a valid `DiffSyncModel` object to the `add()` function.

Expand Down Expand Up @@ -174,7 +174,7 @@ convenient to manage individual records (as in a database) or modify the entire
## Manage individual records

To update individual records in a remote system, you need to extend your `DiffSyncModel` class(es) to define your own `create`, `update` and/or `delete` methods for each model.
A `DiffSyncModel` instance stores a reference to its parent `DiffSync` adapter instance in case you need to use it to look up other model instances from the `DiffSync`'s cache.
A `DiffSyncModel` instance stores a reference to its parent `Adapter` adapter instance in case you need to use it to look up other model instances from the `Adapter`'s cache.

```python
class Device(DiffSyncModel):
Expand Down
4 changes: 2 additions & 2 deletions examples/01-multiple-data-sources/backend_a.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"""

# pylint: disable=wrong-import-order
from diffsync import DiffSync
from diffsync import Adapter
from models import Site, Device, Interface # pylint: disable=no-name-in-module

DATA = {
Expand All @@ -31,7 +31,7 @@
}


class BackendA(DiffSync):
class BackendA(Adapter):
"""Example of a DiffSync adapter implementation."""

site = Site
Expand Down
4 changes: 2 additions & 2 deletions examples/01-multiple-data-sources/backend_b.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"""

# pylint: disable=wrong-import-order
from diffsync import DiffSync
from diffsync import Adapter
from models import Site, Device, Interface # pylint: disable=no-name-in-module

DATA = {
Expand All @@ -35,7 +35,7 @@
}


class BackendB(DiffSync):
class BackendB(Adapter):
"""Example of a DiffSync adapter implementation."""

site = Site
Expand Down
4 changes: 2 additions & 2 deletions examples/01-multiple-data-sources/backend_c.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"""

# pylint: disable=wrong-import-order
from diffsync import DiffSync
from diffsync import Adapter
from models import Site, Device, Interface # pylint: disable=no-name-in-module

DATA = {
Expand All @@ -31,7 +31,7 @@
}


class BackendC(DiffSync):
class BackendC(Adapter):
"""Example of a DiffSync adapter implementation."""

site = Site
Expand Down
10 changes: 5 additions & 5 deletions examples/02-callback-function/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"""
import random

from diffsync import DiffSync, DiffSyncModel
from diffsync import Adapter, DiffSyncModel
from diffsync.logging import enable_console_logging


Expand All @@ -30,7 +30,7 @@ class Number(DiffSyncModel):
number: int


class DiffSync1(DiffSync):
class Adapter1(Adapter):
"""DiffSync adapter that contains a number of Numbers constructed in order."""

number = Number
Expand All @@ -43,7 +43,7 @@ def load(self, count): # pylint: disable=arguments-differ
self.add(Number(number=(i + 1)))


class DiffSync2(DiffSync):
class Adapter2(Adapter):
"""DiffSync adapter that contains a number of Numbers spread randomly across a range."""

number = Number
Expand All @@ -69,11 +69,11 @@ def main():
enable_console_logging(verbosity=0) # Show WARNING and ERROR logs only

# Create a DiffSync1 instance and load it with records numbered 1-100
ds1 = DiffSync1()
ds1 = Adapter1()
ds1.load(count=100)

# Create a DiffSync2 instance and load it with 100 random records in the range 1-200
ds2 = DiffSync2()
ds2 = Adapter2()
ds2.load(count=100)

# Identify and attempt to resolve the differences between the two,
Expand Down
Loading
Loading