From 218082a3e356d5d3f11693053e5e1da1dbbb07e5 Mon Sep 17 00:00:00 2001 From: jld3103 Date: Wed, 26 Oct 2022 08:07:33 +0200 Subject: [PATCH] feat(synchronize): Init Signed-off-by: jld3103 --- commitlint.yaml | 1 + packages/synchronize/.gitignore | 30 + packages/synchronize/.metadata | 10 + packages/synchronize/LICENSE | 1 + packages/synchronize/README.md | 3 + packages/synchronize/analysis_options.yaml | 1 + packages/synchronize/lib/src/action.dart | 40 ++ packages/synchronize/lib/src/conflict.dart | 57 ++ packages/synchronize/lib/src/journal.dart | 31 ++ packages/synchronize/lib/src/journal.g.dart | 15 + .../synchronize/lib/src/journal_entry.dart | 49 ++ .../synchronize/lib/src/journal_entry.g.dart | 19 + packages/synchronize/lib/src/object.dart | 19 + packages/synchronize/lib/src/sources.dart | 39 ++ packages/synchronize/lib/src/sync.dart | 246 +++++++++ packages/synchronize/lib/synchronize.dart | 6 + packages/synchronize/pubspec.yaml | 20 + packages/synchronize/pubspec_overrides.yaml | 4 + packages/synchronize/test/sync_test.dart | 516 ++++++++++++++++++ tool/build-app.sh | 2 +- 20 files changed, 1108 insertions(+), 1 deletion(-) create mode 100644 packages/synchronize/.gitignore create mode 100644 packages/synchronize/.metadata create mode 120000 packages/synchronize/LICENSE create mode 100644 packages/synchronize/README.md create mode 100644 packages/synchronize/analysis_options.yaml create mode 100644 packages/synchronize/lib/src/action.dart create mode 100644 packages/synchronize/lib/src/conflict.dart create mode 100644 packages/synchronize/lib/src/journal.dart create mode 100644 packages/synchronize/lib/src/journal.g.dart create mode 100644 packages/synchronize/lib/src/journal_entry.dart create mode 100644 packages/synchronize/lib/src/journal_entry.g.dart create mode 100644 packages/synchronize/lib/src/object.dart create mode 100644 packages/synchronize/lib/src/sources.dart create mode 100644 packages/synchronize/lib/src/sync.dart create mode 100644 packages/synchronize/lib/synchronize.dart create mode 100644 packages/synchronize/pubspec.yaml create mode 100644 packages/synchronize/pubspec_overrides.yaml create mode 100644 packages/synchronize/test/sync_test.dart diff --git a/commitlint.yaml b/commitlint.yaml index 8aea72a85e0..04f5210415c 100644 --- a/commitlint.yaml +++ b/commitlint.yaml @@ -23,3 +23,4 @@ rules: - neon_lints - nextcloud - sort_box + - synchronize diff --git a/packages/synchronize/.gitignore b/packages/synchronize/.gitignore new file mode 100644 index 00000000000..96486fd9302 --- /dev/null +++ b/packages/synchronize/.gitignore @@ -0,0 +1,30 @@ +# Miscellaneous +*.class +*.log +*.pyc +*.swp +.DS_Store +.atom/ +.buildlog/ +.history +.svn/ +migrate_working_dir/ + +# IntelliJ related +*.iml +*.ipr +*.iws +.idea/ + +# The .vscode folder contains launch configuration and tasks you configure in +# VS Code which you may wish to be included in version control, so this line +# is commented out by default. +#.vscode/ + +# Flutter/Dart/Pub related +# Libraries should not include pubspec.lock, per https://dart.dev/guides/libraries/private-files#pubspeclock. +/pubspec.lock +**/doc/api/ +.dart_tool/ +.packages +build/ diff --git a/packages/synchronize/.metadata b/packages/synchronize/.metadata new file mode 100644 index 00000000000..e093d99e29a --- /dev/null +++ b/packages/synchronize/.metadata @@ -0,0 +1,10 @@ +# This file tracks properties of this Flutter project. +# Used by Flutter tool to assess capabilities and perform upgrades etc. +# +# This file should be version controlled and should not be manually edited. + +version: + revision: 2aa348b9407e96ffe4eca8e8f213c7984afad3f7 + channel: master + +project_type: package diff --git a/packages/synchronize/LICENSE b/packages/synchronize/LICENSE new file mode 120000 index 00000000000..30cff7403da --- /dev/null +++ b/packages/synchronize/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/packages/synchronize/README.md b/packages/synchronize/README.md new file mode 100644 index 00000000000..df0e57b1db0 --- /dev/null +++ b/packages/synchronize/README.md @@ -0,0 +1,3 @@ +# synchronize + +A simple generic implementation of https://unterwaditzer.net/2016/sync-algorithm.html \ No newline at end of file diff --git a/packages/synchronize/analysis_options.yaml b/packages/synchronize/analysis_options.yaml new file mode 100644 index 00000000000..4db3c296b81 --- /dev/null +++ b/packages/synchronize/analysis_options.yaml @@ -0,0 +1 @@ +include: package:neon_lints/dart.yaml diff --git a/packages/synchronize/lib/src/action.dart b/packages/synchronize/lib/src/action.dart new file mode 100644 index 00000000000..3d5b8a61764 --- /dev/null +++ b/packages/synchronize/lib/src/action.dart @@ -0,0 +1,40 @@ +import 'package:meta/meta.dart'; +import 'package:synchronize/src/object.dart'; + +/// Action to be executed in the sync process +@internal +sealed class SyncAction { + // ignore: public_member_api_docs + SyncAction(this.object); + + // ignore: public_member_api_docs + final SyncObject object; +} + +/// Action to delete object from A +@internal +interface class SyncActionDeleteFromA extends SyncAction { + // ignore: public_member_api_docs + SyncActionDeleteFromA(super.object); +} + +/// Action to delete object from B +@internal +interface class SyncActionDeleteFromB extends SyncAction { + // ignore: public_member_api_docs + SyncActionDeleteFromB(super.object); +} + +/// Action to write object to A +@internal +interface class SyncActionWriteToA extends SyncAction { + // ignore: public_member_api_docs + SyncActionWriteToA(super.object); +} + +/// Action to write object to B +@internal +interface class SyncActionWriteToB extends SyncAction { + // ignore: public_member_api_docs + SyncActionWriteToB(super.object); +} diff --git a/packages/synchronize/lib/src/conflict.dart b/packages/synchronize/lib/src/conflict.dart new file mode 100644 index 00000000000..9a1e87da088 --- /dev/null +++ b/packages/synchronize/lib/src/conflict.dart @@ -0,0 +1,57 @@ +import 'package:meta/meta.dart'; +import 'package:synchronize/src/object.dart'; + +/// Contains information about a conflict that appeared during sync. +@immutable +class SyncConflict { + // ignore: public_member_api_docs + const SyncConflict({ + required this.id, + required this.type, + required this.objectA, + required this.objectB, + this.skipped = false, + }); + + /// Id of the objects involved in the conflict. + final String id; + + /// Type of the conflict that appeared. See [SyncConflictType] for more info. + final SyncConflictType type; + + /// Object A involved in the conflict. + final SyncObject objectA; + + /// Object B involved in the conflict. + final SyncObject objectB; + + /// Whether the conflict was skipped by the user, useful for ignoring it later on + final bool skipped; + + @override + bool operator ==(final dynamic other) => other is SyncConflict && other.id == id; + + @override + int get hashCode => id.hashCode; +} + +/// Types of conflicts that can appear during sync. +enum SyncConflictType { + /// New objects with the same id exist on both sides. + bothNew, + + /// Both objects with the same id have changed. + bothChanged, +} + +/// Ways to resolve [SyncConflict]s. +enum SyncConflictSolution { + /// Overwrite the content of object A with the content of object B. + overwriteA, + + /// Overwrite the content of object B with the content of object A. + overwriteB, + + /// Skip the conflict and just do nothing. + skip, +} diff --git a/packages/synchronize/lib/src/journal.dart b/packages/synchronize/lib/src/journal.dart new file mode 100644 index 00000000000..d3793383c17 --- /dev/null +++ b/packages/synchronize/lib/src/journal.dart @@ -0,0 +1,31 @@ +import 'package:json_annotation/json_annotation.dart'; +import 'package:synchronize/src/journal_entry.dart'; + +part 'journal.g.dart'; + +/// Contains the journal. +/// +/// Used for detecting changes and new or deleted files. +@JsonSerializable() +class SyncJournal { + // Note: This must not be const as otherwise the entries are not modifiable when a const set is used! + // ignore: public_member_api_docs + SyncJournal( + this.entries, + ); + + // ignore: public_member_api_docs + factory SyncJournal.fromJson(final Map json) => _$SyncJournalFromJson(json); + // ignore: public_member_api_docs + Map toJson() => _$SyncJournalToJson(this); + + /// All entries contained in the journal. + final Set entries; + + /// Update an [entry]. + void updateEntry(final SyncJournalEntry entry) { + entries + ..remove(entry) + ..add(entry); + } +} diff --git a/packages/synchronize/lib/src/journal.g.dart b/packages/synchronize/lib/src/journal.g.dart new file mode 100644 index 00000000000..87a172721ae --- /dev/null +++ b/packages/synchronize/lib/src/journal.g.dart @@ -0,0 +1,15 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'journal.dart'; + +// ************************************************************************** +// JsonSerializableGenerator +// ************************************************************************** + +SyncJournal _$SyncJournalFromJson(Map json) => SyncJournal( + (json['entries'] as List).map((e) => SyncJournalEntry.fromJson(e as Map)).toSet(), + ); + +Map _$SyncJournalToJson(SyncJournal instance) => { + 'entries': instance.entries.toList(), + }; diff --git a/packages/synchronize/lib/src/journal_entry.dart b/packages/synchronize/lib/src/journal_entry.dart new file mode 100644 index 00000000000..48fc0c6852e --- /dev/null +++ b/packages/synchronize/lib/src/journal_entry.dart @@ -0,0 +1,49 @@ +import 'package:collection/collection.dart'; +import 'package:json_annotation/json_annotation.dart'; +import 'package:meta/meta.dart'; +import 'package:synchronize/src/journal.dart'; + +part 'journal_entry.g.dart'; + +/// Stores a single entry in the [SyncJournal]. +/// +/// It contains an [id] and ETags for each object, [etagA] and [etagB] respectively. +@immutable +@JsonSerializable() +class SyncJournalEntry { + // ignore: public_member_api_docs + const SyncJournalEntry( + this.id, + this.etagA, + this.etagB, + ); + + // ignore: public_member_api_docs + factory SyncJournalEntry.fromJson(final Map json) => _$SyncJournalEntryFromJson(json); + // ignore: public_member_api_docs + Map toJson() => _$SyncJournalEntryToJson(this); + + // ignore: public_member_api_docs + final String id; + + /// ETag of the object A. + final String etagA; + + /// ETag of the object B. + final String etagB; + + @override + String toString() => 'SyncJournalEntry(id: $id, etagA: $etagA, etagB: $etagB)'; + + @override + bool operator ==(final Object other) => other is SyncJournalEntry && other.id == id; + + @override + int get hashCode => id.hashCode; +} + +// ignore: public_member_api_docs +extension SyncJournalEntriesFind on Iterable { + // ignore: public_member_api_docs + SyncJournalEntry? tryFind(final String id) => firstWhereOrNull((final entry) => entry.id == id); +} diff --git a/packages/synchronize/lib/src/journal_entry.g.dart b/packages/synchronize/lib/src/journal_entry.g.dart new file mode 100644 index 00000000000..67e59deceec --- /dev/null +++ b/packages/synchronize/lib/src/journal_entry.g.dart @@ -0,0 +1,19 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'journal_entry.dart'; + +// ************************************************************************** +// JsonSerializableGenerator +// ************************************************************************** + +SyncJournalEntry _$SyncJournalEntryFromJson(Map json) => SyncJournalEntry( + json['id'] as String, + json['etagA'] as String, + json['etagB'] as String, + ); + +Map _$SyncJournalEntryToJson(SyncJournalEntry instance) => { + 'id': instance.id, + 'etagA': instance.etagA, + 'etagB': instance.etagB, + }; diff --git a/packages/synchronize/lib/src/object.dart b/packages/synchronize/lib/src/object.dart new file mode 100644 index 00000000000..023f3d95b71 --- /dev/null +++ b/packages/synchronize/lib/src/object.dart @@ -0,0 +1,19 @@ +import 'package:collection/collection.dart'; + +/// Wraps the actual data contained on each side. +typedef SyncObject = (String, T); + +// ignore: public_member_api_docs +extension SyncObjectFields on SyncObject { + /// Id of the object. + String get id => $1; + + /// Actual data of the object, can be anything. + T get data => $2; +} + +// ignore: public_member_api_docs +extension SyncObjectsFind on List> { + // ignore: public_member_api_docs + SyncObject? tryFind(final String id) => firstWhereOrNull((final object) => object.id == id); +} diff --git a/packages/synchronize/lib/src/sources.dart b/packages/synchronize/lib/src/sources.dart new file mode 100644 index 00000000000..42556e98bb2 --- /dev/null +++ b/packages/synchronize/lib/src/sources.dart @@ -0,0 +1,39 @@ +import 'dart:async'; + +import 'package:synchronize/src/conflict.dart'; +import 'package:synchronize/src/object.dart'; + +/// The source the sync uses to sync from and to. +abstract interface class SyncSource { + /// List all the objects. + FutureOr>> listObjects(); + + /// Calculates the ETag of a given [object]. + /// + /// Should be something easy to compute like the mtime of a file and preferably not the hash of the whole content in order to be fast + FutureOr getObjectETag(final SyncObject object); + + /// Writes the given [object]. + FutureOr> writeObject(final SyncObject object); + + /// Deletes the given [object]. + FutureOr deleteObject(final SyncObject object); +} + +/// The sources the sync uses to sync from and to. +abstract interface class SyncSources { + // ignore: public_member_api_docs + SyncSources( + this.sourceA, + this.sourceB, + ); + + // ignore: public_member_api_docs + final SyncSource sourceA; + + // ignore: public_member_api_docs + final SyncSource sourceB; + + /// Automatically find a solution for conflicts that don't matter. Useful e.g. for ignoring new directories. + SyncConflictSolution? findSolution(final SyncObject objectA, final SyncObject objectB); +} diff --git a/packages/synchronize/lib/src/sync.dart b/packages/synchronize/lib/src/sync.dart new file mode 100644 index 00000000000..93dd6fbea14 --- /dev/null +++ b/packages/synchronize/lib/src/sync.dart @@ -0,0 +1,246 @@ +import 'package:synchronize/src/action.dart'; +import 'package:synchronize/src/conflict.dart'; +import 'package:synchronize/src/journal.dart'; +import 'package:synchronize/src/journal_entry.dart'; +import 'package:synchronize/src/object.dart'; +import 'package:synchronize/src/sources.dart'; + +/// Sync between two [SyncSources]s. +/// +/// This implementation follows https://unterwaditzer.net/2016/sync-algorithm.html in a generic and abstract way +/// and should work for any two kinds of sources and objects. +Future>> sync( + final SyncSources sources, + final SyncJournal journal, + final Map conflictSolutions, { + final bool keepSkipsAsConflicts = false, +}) async { + final diff = await computeSyncDiff( + sources, + journal, + conflictSolutions, + keepSkipsAsConflicts: keepSkipsAsConflicts, + ); + await executeSyncDiff( + sources, + journal, + diff, + ); + return diff.conflicts; +} + +/// Differences between the two sources +class SyncDiff { + // ignore: public_member_api_docs + SyncDiff( + this.actions, + this.conflicts, + ); + + /// Actions required to solve the difference + final List actions; + + /// Conflicts without solutions that need to be solved + final List> conflicts; +} + +/// Executes the actions required to solve the difference +Future executeSyncDiff( + final SyncSources sources, + final SyncJournal journal, + final SyncDiff diff, +) async { + for (final action in diff.actions) { + switch (action) { + case SyncActionDeleteFromA(): + await sources.sourceA.deleteObject(action.object as SyncObject); + journal.entries.removeWhere((final entry) => entry.id == action.object.id); + case SyncActionDeleteFromB(): + await sources.sourceB.deleteObject(action.object as SyncObject); + journal.entries.removeWhere((final entry) => entry.id == action.object.id); + case SyncActionWriteToA(): + final objectA = await sources.sourceA.writeObject(action.object as SyncObject); + journal.updateEntry( + SyncJournalEntry( + action.object.id, + await sources.sourceA.getObjectETag(objectA), + await sources.sourceB.getObjectETag(action.object as SyncObject), + ), + ); + case SyncActionWriteToB(): + final objectB = await sources.sourceB.writeObject(action.object as SyncObject); + journal.updateEntry( + SyncJournalEntry( + action.object.id, + await sources.sourceA.getObjectETag(action.object as SyncObject), + await sources.sourceB.getObjectETag(objectB), + ), + ); + } + } +} + +/// Computes the difference, useful for displaying if a sync is up to date. +Future> computeSyncDiff( + final SyncSources sources, + final SyncJournal journal, + final Map conflictSolutions, { + final bool keepSkipsAsConflicts = false, +}) async { + final actions = []; + final conflicts = >{}; + var objectsA = await sources.sourceA.listObjects(); + var objectsB = await sources.sourceB.listObjects(); + + for (final objectA in objectsA) { + final objectB = objectsB.tryFind(objectA.id); + final journalEntry = journal.entries.tryFind(objectA.id); + + // If the ID exists on side A and the journal, but not on B, it has been deleted on B. Delete it from A and the journal. + if (journalEntry != null && objectB == null) { + actions.add(SyncActionDeleteFromA(objectA)); + continue; + } + + // If the ID exists on side A and side B, but not in journal, we can not just create it in journal, since the two items might contain different content each. + if (objectB != null && journalEntry == null) { + conflicts.add( + SyncConflict( + id: objectA.id, + type: SyncConflictType.bothNew, + objectA: objectA, + objectB: objectB, + ), + ); + continue; + } + + // If the ID exists on side A, but not on B or the journal, it must have been created on A. Copy the item from A to B and also insert it into journal. + if (objectB == null || journalEntry == null) { + actions.add(SyncActionWriteToB(objectA)); + continue; + } + } + + for (final objectB in objectsB) { + final objectA = objectsA.tryFind(objectB.id); + final journalEntry = journal.entries.tryFind(objectB.id); + + // If the ID exists on side B and the journal, but not on A, it has been deleted on A. Delete it from B and the journal. + if (journalEntry != null && objectA == null) { + actions.add(SyncActionDeleteFromB(objectB)); + continue; + } + + // If the ID exists on side B and side A, but not in journal, we can not just create it in journal, since the two items might contain different content each. + if (objectA != null && journalEntry == null) { + conflicts.add( + SyncConflict( + id: objectA.id, + type: SyncConflictType.bothNew, + objectA: objectA, + objectB: objectB, + ), + ); + continue; + } + + // If the ID exists on side B, but not on A or the journal, it must have been created on B. Copy the item from B to A and also insert it into journal. + if (objectA == null || journalEntry == null) { + actions.add(SyncActionWriteToA(objectB)); + continue; + } + } + + objectsA = await sources.sourceA.listObjects(); + objectsB = await sources.sourceB.listObjects(); + final entries = journal.entries.toList(); + for (final entry in entries) { + final objectA = objectsA.tryFind(entry.id); + final objectB = objectsB.tryFind(entry.id); + + // Remove all entries from journal that don't exist anymore + if (objectA == null && objectB == null) { + journal.entries.removeWhere((final e) => e.id == entry.id); + continue; + } + + if (objectA != null && objectB != null) { + final changedA = entry.etagA != await sources.sourceA.getObjectETag(objectA); + final changedB = entry.etagB != await sources.sourceB.getObjectETag(objectB); + + if (changedA && changedB) { + conflicts.add( + SyncConflict( + id: objectA.id, + type: SyncConflictType.bothChanged, + objectA: objectA, + objectB: objectB, + ), + ); + continue; + } + + if (changedA && !changedB) { + actions.add(SyncActionWriteToB(objectA)); + continue; + } + + if (changedB && !changedA) { + actions.add(SyncActionWriteToA(objectB)); + continue; + } + } + } + + final unsolvedConflicts = >[]; + for (final conflict in conflicts) { + final solution = conflictSolutions[conflict.id] ?? sources.findSolution(conflict.objectA, conflict.objectB); + switch (solution) { + case SyncConflictSolution.overwriteA: + actions.add(SyncActionWriteToA(conflict.objectB)); + case SyncConflictSolution.overwriteB: + actions.add(SyncActionWriteToB(conflict.objectA)); + case SyncConflictSolution.skip: + if (keepSkipsAsConflicts) { + unsolvedConflicts.add( + SyncConflict( + id: conflict.id, + type: conflict.type, + objectA: conflict.objectA, + objectB: conflict.objectB, + skipped: true, + ), + ); + } + case null: + unsolvedConflicts.add(conflict); + } + } + + return SyncDiff( + _sortActions(actions), + unsolvedConflicts, + ); +} + +List _sortActions(final List actions) { + final addActions = []; + final removeActions = []; + for (final action in actions) { + switch (action) { + case SyncActionWriteToA(): + addActions.add(action); + case SyncActionWriteToB(): + addActions.add(action); + case SyncActionDeleteFromA(): + removeActions.add(action); + case SyncActionDeleteFromB(): + removeActions.add(action); + } + } + return _innerSortActions(addActions)..addAll(_innerSortActions(removeActions).reversed); +} + +List _innerSortActions(final List actions) => + actions..sort((final a, final b) => a.object.id.compareTo(b.object.id)); diff --git a/packages/synchronize/lib/synchronize.dart b/packages/synchronize/lib/synchronize.dart new file mode 100644 index 00000000000..5b448e17676 --- /dev/null +++ b/packages/synchronize/lib/synchronize.dart @@ -0,0 +1,6 @@ +export 'package:synchronize/src/conflict.dart'; +export 'package:synchronize/src/journal.dart'; +export 'package:synchronize/src/journal_entry.dart'; +export 'package:synchronize/src/object.dart'; +export 'package:synchronize/src/sources.dart'; +export 'package:synchronize/src/sync.dart'; diff --git a/packages/synchronize/pubspec.yaml b/packages/synchronize/pubspec.yaml new file mode 100644 index 00000000000..3f4d204e7e6 --- /dev/null +++ b/packages/synchronize/pubspec.yaml @@ -0,0 +1,20 @@ +name: synchronize +version: 1.0.0 + +environment: + sdk: '>=3.0.0 <4.0.0' + +dependencies: + collection: ^1.17.1 + json_annotation: ^4.8.1 + meta: ^1.9.1 + +dev_dependencies: + build_runner: ^2.4.6 + crypto: ^3.0.3 + json_serializable: ^6.7.1 + neon_lints: + git: + url: https://github.com/nextcloud/neon + path: packages/neon_lints + test: ^1.24.5 diff --git a/packages/synchronize/pubspec_overrides.yaml b/packages/synchronize/pubspec_overrides.yaml new file mode 100644 index 00000000000..4abc9fdcbad --- /dev/null +++ b/packages/synchronize/pubspec_overrides.yaml @@ -0,0 +1,4 @@ +# melos_managed_dependency_overrides: neon_lints +dependency_overrides: + neon_lints: + path: ../neon_lints diff --git a/packages/synchronize/test/sync_test.dart b/packages/synchronize/test/sync_test.dart new file mode 100644 index 00000000000..cc97be075fd --- /dev/null +++ b/packages/synchronize/test/sync_test.dart @@ -0,0 +1,516 @@ +import 'dart:convert'; +import 'dart:math'; + +import 'package:crypto/crypto.dart'; +import 'package:synchronize/synchronize.dart'; +import 'package:test/test.dart'; + +abstract class Wrap { + Wrap(this.content); + + final String content; +} + +class WrapA extends Wrap { + WrapA(super.content); +} + +class WrapB extends Wrap { + WrapB(super.content); +} + +class TestSyncState { + TestSyncState( + this.stateA, + this.stateB, + ); + + final Map stateA; + final Map stateB; +} + +class TestSyncSourceA implements SyncSource { + TestSyncSourceA(this.state); + + final Map state; + + @override + Future>> listObjects() async => state.keys.map((final key) => (key, state[key]!)).toList(); + + @override + Future getObjectETag(final SyncObject object) async => etagA(object.data.content); + + @override + Future> writeObject(final SyncObject object) async { + final wrap = WrapA(object.data.content); + state[object.id] = wrap; + return (object.id, wrap); + } + + @override + Future deleteObject(final SyncObject object) async => state.remove(object.id); +} + +class TestSyncSourceB implements SyncSource { + TestSyncSourceB(this.state); + + final Map state; + + @override + Future>> listObjects() async => state.keys.map((final key) => (key, state[key]!)).toList(); + + @override + Future getObjectETag(final SyncObject object) async => etagB(object.data.content); + + @override + Future> writeObject(final SyncObject object) async { + final wrap = WrapB(object.data.content); + state[object.id] = wrap; + return (object.id, wrap); + } + + @override + Future deleteObject(final SyncObject object) async => state.remove(object.id); +} + +class TestSyncSources implements SyncSources { + TestSyncSources( + this.sourceA, + this.sourceB, + ); + + factory TestSyncSources.fromState(final TestSyncState state) => TestSyncSources( + TestSyncSourceA(state.stateA), + TestSyncSourceB(state.stateB), + ); + + @override + SyncSource sourceA; + + @override + SyncSource sourceB; + + @override + SyncConflictSolution? findSolution(final SyncObject objectA, final SyncObject objectB) => null; +} + +String etagA(final String content) => sha1.convert(utf8.encode('A$content')).toString(); + +String etagB(final String content) => sha1.convert(utf8.encode('B$content')).toString(); + +String randomEtag() => sha1.convert(utf8.encode(Random().nextDouble().toString())).toString(); + +Future main() async { + group('sync', () { + group('stub', () { + test('all empty', () async { + final state = TestSyncState({}, {}); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({}); + + final conflicts = await sync(sources, journal, {}); + expect(conflicts, isEmpty); + expect(state.stateA, isEmpty); + expect(state.stateB, isEmpty); + expect(journal.entries, isEmpty); + }); + + group('copy', () { + group('missing', () { + test('to A', () async { + const id = '123'; + const content = '456'; + final state = TestSyncState( + {}, + { + id: WrapB(content), + }, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({}); + + final conflicts = await sync(sources, journal, {}); + expect(conflicts, isEmpty); + expect(state.stateA, hasLength(1)); + expect(state.stateA[id]!.content, content); + expect(state.stateB, hasLength(1)); + expect(state.stateB[id]!.content, content); + expect(journal.entries, hasLength(1)); + expect(journal.entries.tryFind(id)!.etagA, etagA(content)); + expect(journal.entries.tryFind(id)!.etagB, etagB(content)); + }); + + test('to B', () async { + const id = '123'; + const content = '456'; + final state = TestSyncState( + { + id: WrapA(content), + }, + {}, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({}); + + final conflicts = await sync(sources, journal, {}); + expect(conflicts, isEmpty); + expect(state.stateA, hasLength(1)); + expect(state.stateA[id]!.content, content); + expect(state.stateB, hasLength(1)); + expect(state.stateB[id]!.content, content); + expect(journal.entries, hasLength(1)); + expect(journal.entries.tryFind(id)!.etagA, etagA(content)); + expect(journal.entries.tryFind(id)!.etagB, etagB(content)); + }); + }); + + group('changed', () { + test('to A', () async { + const id = '123'; + const contentA = '456'; + const contentB = '789'; + final state = TestSyncState( + { + id: WrapA(contentA), + }, + { + id: WrapB(contentB), + }, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({ + SyncJournalEntry(id, etagA(contentA), randomEtag()), + }); + + final conflicts = await sync(sources, journal, {}); + expect(conflicts, isEmpty); + expect(state.stateA, hasLength(1)); + expect(state.stateA[id]!.content, contentB); + expect(state.stateB, hasLength(1)); + expect(state.stateB[id]!.content, contentB); + expect(journal.entries, hasLength(1)); + expect(journal.entries.tryFind(id)!.etagA, etagA(contentB)); + expect(journal.entries.tryFind(id)!.etagB, etagB(contentB)); + }); + + test('to B', () async { + const id = '123'; + const contentA = '456'; + const contentB = '789'; + final state = TestSyncState( + { + id: WrapA(contentA), + }, + { + id: WrapB(contentB), + }, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({ + SyncJournalEntry(id, randomEtag(), etagB(contentB)), + }); + + final conflicts = await sync(sources, journal, {}); + expect(conflicts, isEmpty); + expect(state.stateA, hasLength(1)); + expect(state.stateA[id]!.content, contentA); + expect(state.stateB, hasLength(1)); + expect(state.stateB[id]!.content, contentA); + expect(journal.entries, hasLength(1)); + expect(journal.entries.tryFind(id)!.etagA, etagA(contentA)); + expect(journal.entries.tryFind(id)!.etagB, etagB(contentA)); + }); + }); + }); + + group('delete', () { + test('from A', () async { + const id = '123'; + const content = '456'; + final state = TestSyncState( + { + id: WrapA(content), + }, + {}, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({ + SyncJournalEntry(id, etagA(content), etagB(content)), + }); + + final conflicts = await sync(sources, journal, {}); + expect(conflicts, isEmpty); + expect(state.stateA, isEmpty); + expect(state.stateB, isEmpty); + expect(journal.entries, isEmpty); + }); + + test('from B', () async { + const id = '123'; + const content = '456'; + final state = TestSyncState( + {}, + { + id: WrapB(content), + }, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({ + SyncJournalEntry(id, etagA(content), etagB(content)), + }); + + final conflicts = await sync(sources, journal, {}); + expect(conflicts, isEmpty); + expect(state.stateA, isEmpty); + expect(state.stateB, isEmpty); + expect(journal.entries, isEmpty); + }); + + test('from journal', () async { + const id = '123'; + const content = '456'; + final state = TestSyncState({}, {}); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({ + SyncJournalEntry(id, etagA(content), etagB(content)), + }); + + final conflicts = await sync(sources, journal, {}); + expect(conflicts, isEmpty); + expect(state.stateA, isEmpty); + expect(state.stateB, isEmpty); + expect(journal.entries, isEmpty); + }); + }); + + group('conflict', () { + test('journal missing', () async { + const id = '123'; + const content = '456'; + final state = TestSyncState( + { + id: WrapA(content), + }, + { + id: WrapB(content), + }, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({}); + + final conflicts = await sync(sources, journal, {}); + expect(conflicts, hasLength(1)); + expect(conflicts[0].type, SyncConflictType.bothNew); + expect(state.stateA, hasLength(1)); + expect(state.stateA[id]!.content, content); + expect(state.stateB, hasLength(1)); + expect(state.stateB[id]!.content, content); + expect(journal.entries, isEmpty); + }); + + test('both changed', () async { + const id = '123'; + const contentA = '456'; + const contentB = '789'; + final state = TestSyncState( + { + id: WrapA(contentA), + }, + { + id: WrapB(contentB), + }, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({ + SyncJournalEntry(id, randomEtag(), randomEtag()), + }); + + final conflicts = await sync(sources, journal, {}); + expect(conflicts, hasLength(1)); + expect(conflicts[0].type, SyncConflictType.bothChanged); + expect(state.stateA, hasLength(1)); + expect(state.stateA[id]!.content, contentA); + expect(state.stateB, hasLength(1)); + expect(state.stateB[id]!.content, contentB); + expect(journal.entries, hasLength(1)); + }); + + group('solution', () { + group('journal missing', () { + test('skip', () async { + const id = '123'; + const contentA = '456'; + const contentB = '789'; + final state = TestSyncState( + { + id: WrapA(contentA), + }, + { + id: WrapB(contentB), + }, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({}); + + final conflicts = await sync(sources, journal, { + id: SyncConflictSolution.skip, + }); + expect(conflicts, isEmpty); + expect(state.stateA, hasLength(1)); + expect(state.stateA[id]!.content, contentA); + expect(state.stateB, hasLength(1)); + expect(state.stateB[id]!.content, contentB); + expect(journal.entries, isEmpty); + }); + + test('overwrite A', () async { + const id = '123'; + const contentA = '456'; + const contentB = '789'; + final state = TestSyncState( + { + id: WrapA(contentA), + }, + { + id: WrapB(contentB), + }, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({}); + + final conflicts = await sync(sources, journal, { + id: SyncConflictSolution.overwriteA, + }); + expect(conflicts, isEmpty); + expect(state.stateA, hasLength(1)); + expect(state.stateA[id]!.content, contentB); + expect(state.stateB, hasLength(1)); + expect(state.stateB[id]!.content, contentB); + expect(journal.entries, hasLength(1)); + expect(journal.entries.tryFind(id)!.etagA, etagA(contentB)); + expect(journal.entries.tryFind(id)!.etagB, etagB(contentB)); + }); + + test('overwrite B', () async { + const id = '123'; + const contentA = '456'; + const contentB = '789'; + final state = TestSyncState( + { + id: WrapA(contentA), + }, + { + id: WrapB(contentB), + }, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({}); + + final conflicts = await sync(sources, journal, { + id: SyncConflictSolution.overwriteB, + }); + expect(conflicts, isEmpty); + expect(state.stateA, hasLength(1)); + expect(state.stateA[id]!.content, contentA); + expect(state.stateB, hasLength(1)); + expect(state.stateB[id]!.content, contentA); + expect(journal.entries, hasLength(1)); + expect(journal.entries.tryFind(id)!.etagA, etagA(contentA)); + expect(journal.entries.tryFind(id)!.etagB, etagB(contentA)); + }); + }); + + group('both changed', () { + test('skip', () async { + const id = '123'; + const contentA = '456'; + const contentB = '789'; + final state = TestSyncState( + { + id: WrapA(contentA), + }, + { + id: WrapB(contentB), + }, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({ + SyncJournalEntry(id, randomEtag(), randomEtag()), + }); + + final conflicts = await sync(sources, journal, { + id: SyncConflictSolution.skip, + }); + expect(conflicts, isEmpty); + expect(state.stateA, hasLength(1)); + expect(state.stateA[id]!.content, contentA); + expect(state.stateB, hasLength(1)); + expect(state.stateB[id]!.content, contentB); + expect(journal.entries, hasLength(1)); + }); + + test('overwrite A', () async { + const id = '123'; + const contentA = '456'; + const contentB = '789'; + final state = TestSyncState( + { + id: WrapA(contentA), + }, + { + id: WrapB(contentB), + }, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({ + SyncJournalEntry(id, randomEtag(), randomEtag()), + }); + + final conflicts = await sync(sources, journal, { + id: SyncConflictSolution.overwriteA, + }); + expect(conflicts, isEmpty); + expect(state.stateA, hasLength(1)); + expect(state.stateA[id]!.content, contentB); + expect(state.stateB, hasLength(1)); + expect(state.stateB[id]!.content, contentB); + expect(journal.entries, hasLength(1)); + expect(journal.entries.tryFind(id)!.etagA, etagA(contentB)); + expect(journal.entries.tryFind(id)!.etagB, etagB(contentB)); + }); + + test('overwrite B', () async { + const id = '123'; + const contentA = '456'; + const contentB = '789'; + final state = TestSyncState( + { + id: WrapA(contentA), + }, + { + id: WrapB(contentB), + }, + ); + final sources = TestSyncSources.fromState(state); + final journal = SyncJournal({ + SyncJournalEntry(id, randomEtag(), randomEtag()), + }); + + final conflicts = await sync(sources, journal, { + id: SyncConflictSolution.overwriteB, + }); + expect(conflicts, isEmpty); + expect(state.stateA, hasLength(1)); + expect(state.stateA[id]!.content, contentA); + expect(state.stateB, hasLength(1)); + expect(state.stateB[id]!.content, contentA); + expect(journal.entries, hasLength(1)); + expect(journal.entries.tryFind(id)!.etagA, etagA(contentA)); + expect(journal.entries.tryFind(id)!.etagB, etagB(contentA)); + }); + }); + }); + }); + }); + }); +} diff --git a/tool/build-app.sh b/tool/build-app.sh index fd5fd9d8124..e4b383459d9 100755 --- a/tool/build-app.sh +++ b/tool/build-app.sh @@ -45,7 +45,7 @@ if [[ "$target" == "linux/arm64" ]] || [[ "$target" == "linux/amd64" ]]; then ./tool/build run_args=() - for path in packages/{app,dynamite/dynamite_runtime,file_icons,neon_lints,nextcloud,sort_box}/{lib,pubspec.yaml} packages/neon/*/{assets,lib,pubspec.yaml,pubspec_overrides.yaml} packages/file_icons/fonts packages/nextcloud/pubspec_overrides.yaml packages/app/{pubspec_overrides.yaml,assets,build,linux}; do + for path in packages/{app,dynamite/dynamite_runtime,file_icons,neon_lints,nextcloud,sort_box,synchronize}/{lib,pubspec.yaml} packages/neon/*/{assets,lib,pubspec.yaml,pubspec_overrides.yaml} packages/file_icons/fonts packages/nextcloud/pubspec_overrides.yaml packages/app/{pubspec_overrides.yaml,assets,build,linux}; do run_args+=(-v "$(pwd)/$path:/src/$path") done mkdir -p "packages/app/build"