From 533a59808ab3887f5d3aa3edcfd91c62e266af8b Mon Sep 17 00:00:00 2001 From: stack72 Date: Fri, 2 Aug 2024 21:27:44 +0100 Subject: [PATCH 1/3] fix(sdf): Don't overwrite sdf config file with empty args When using a config file, we didn't check that the args were empty before writing over the top --- bin/sdf/src/args.rs | 10 ++++++---- bin/sdf/src/main.rs | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/sdf/src/args.rs b/bin/sdf/src/args.rs index a5224ecb1a..6cb7085972 100644 --- a/bin/sdf/src/args.rs +++ b/bin/sdf/src/args.rs @@ -302,10 +302,12 @@ impl TryFrom for Config { config_map.set("create_workspace_permissions", create_workspace_permissions); } - config_map.set( - "create_workspace_allowlist", - args.create_workspace_allowlist, - ); + if !args.create_workspace_allowlist.is_empty() { + config_map.set( + "create_workspace_allowlist", + args.create_workspace_allowlist, + ); + } config_map.set("nats.connection_name", NAME); config_map.set("pg.application_name", NAME); diff --git a/bin/sdf/src/main.rs b/bin/sdf/src/main.rs index 5228fe9134..82f60ad20e 100644 --- a/bin/sdf/src/main.rs +++ b/bin/sdf/src/main.rs @@ -93,6 +93,8 @@ async fn async_main() -> Result<()> { let config = Config::try_from(args)?; + debug!(config =?config, "entire startup config"); + let encryption_key = Server::load_encryption_key(config.crypto().clone()).await?; let jwt_public_signing_key = Server::load_jwt_public_signing_key(config.jwt_signing_public_key().clone()).await?; From 910bce74e6b1590f9f4a42d91bdb84ce4fd9f89e Mon Sep 17 00:00:00 2001 From: Zachary Hamm Date: Mon, 5 Aug 2024 16:42:52 -0500 Subject: [PATCH 2/3] fix(dal): fix race in working_copy_mut --- lib/dal/src/workspace_snapshot.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/dal/src/workspace_snapshot.rs b/lib/dal/src/workspace_snapshot.rs index 0197158532..f56d125576 100644 --- a/lib/dal/src/workspace_snapshot.rs +++ b/lib/dal/src/workspace_snapshot.rs @@ -434,13 +434,13 @@ impl WorkspaceSnapshot { skip_all )] async fn working_copy_mut(&self) -> SnapshotWriteGuard<'_> { - if self.working_copy.read().await.is_none() { + let mut working_copy = self.working_copy.write().await; + if working_copy.is_none() { // Make a copy of the read only graph as our new working copy - *self.working_copy.write().await = Some(self.read_only_graph.inner().clone()); + *working_copy = Some(self.read_only_graph.inner().clone()); } - SnapshotWriteGuard { - working_copy_write_guard: self.working_copy.write().await, + working_copy_write_guard: working_copy, } } From b15536d72c19e64bd8405c999a214c5a64dafc9a Mon Sep 17 00:00:00 2001 From: Zachary Hamm Date: Sat, 3 Aug 2024 15:11:54 -0500 Subject: [PATCH 3/3] feat: switch rebaser to use operational transforms So we made a few changes here... Co-authored-by: Nick Gerace Co-authored-by: Zack Hamm Co-authored-by: Jacob Helwig Co-authored-by: Adam Jacob Signed-off-by: Fletcher Nichol --- lib/dal/examples/rebase/main.rs | 37 +- lib/dal/examples/snapshot-analyzer/main.rs | 108 +- .../attribute/value/dependent_value_graph.rs | 2 +- lib/dal/src/builtins/func.rs | 4 + lib/dal/src/change_set.rs | 33 +- lib/dal/src/component.rs | 8 +- lib/dal/src/component/frame.rs | 10 - lib/dal/src/context.rs | 131 +- lib/dal/src/diagram.rs | 31 +- .../src/job/definition/compute_validation.rs | 2 +- .../job/definition/dependent_values_update.rs | 1 + lib/dal/src/job/processor/nats_processor.rs | 2 - lib/dal/src/module.rs | 3 +- lib/dal/src/workspace_snapshot.rs | 222 +- lib/dal/src/workspace_snapshot/graph.rs | 243 +- .../graph/detect_conflicts_and_updates.rs | 325 +- .../tests/detect_conflicts_and_updates.rs | 3046 ++--------------- .../workspace_snapshot/graph/tests/rebase.rs | 4 +- lib/dal/src/workspace_snapshot/node_weight.rs | 2 +- .../node_weight/action_node_weight.rs | 2 +- .../action_prototype_node_weight.rs | 2 +- ...ttribute_prototype_argument_node_weight.rs | 2 +- .../attribute_value_node_weight.rs | 2 +- .../node_weight/category_node_weight.rs | 2 +- .../node_weight/component_node_weight.rs | 2 +- .../node_weight/content_node_weight.rs | 2 +- .../dependent_value_root_node_weight.rs | 2 +- .../node_weight/func_argument_node_weight.rs | 2 +- .../node_weight/func_node_weight.rs | 2 +- .../node_weight/ordering_node_weight.rs | 2 +- .../node_weight/prop_node_weight.rs | 2 +- .../node_weight/secret_node_weight.rs | 2 +- lib/dal/src/workspace_snapshot/update.rs | 25 +- lib/dal/tests/integration_test/rebaser.rs | 4 + .../src/change_set_requests/handlers.rs | 2 +- lib/rebaser-server/src/rebase.rs | 238 +- lib/sdf-server/src/server/server.rs | 4 + .../service/change_set/rebase_on_base.rs | 35 +- lib/si-events-rs/src/lib.rs | 1 + lib/si-events-rs/src/rebase_batch_address.rs | 121 + lib/si-layer-cache/src/activities/rebase.rs | 56 +- lib/si-layer-cache/src/db.rs | 29 +- lib/si-layer-cache/src/db/cache_updates.rs | 49 +- lib/si-layer-cache/src/db/rebase_batch.rs | 170 + lib/si-layer-cache/src/event.rs | 2 + .../src/migrations/U0008__rebase_batches.sql | 10 + lib/si-layer-cache/src/persister.rs | 2 + .../tests/integration_test/activities.rs | 2 +- .../integration_test/activities/rebase.rs | 42 +- .../tests/integration_test/db/cas.rs | 2 +- .../tests/integration_test/db/func_run.rs | 2 +- .../tests/integration_test/db/func_run_log.rs | 2 +- .../integration_test/db/workspace_snapshot.rs | 2 +- lib/veritech-client/src/lib.rs | 2 +- 54 files changed, 1218 insertions(+), 3824 deletions(-) create mode 100644 lib/si-events-rs/src/rebase_batch_address.rs create mode 100644 lib/si-layer-cache/src/db/rebase_batch.rs create mode 100644 lib/si-layer-cache/src/migrations/U0008__rebase_batches.sql diff --git a/lib/dal/examples/rebase/main.rs b/lib/dal/examples/rebase/main.rs index 6941070878..966f1960b1 100644 --- a/lib/dal/examples/rebase/main.rs +++ b/lib/dal/examples/rebase/main.rs @@ -2,9 +2,7 @@ use std::{env, fs::File, io::prelude::*}; use si_layer_cache::db::serialize; -use dal::{ - workspace_snapshot::node_weight::NodeWeight, NodeWeightDiscriminants, WorkspaceSnapshotGraphV1, -}; +use dal::WorkspaceSnapshotGraphV1; type Result = std::result::Result>; @@ -39,41 +37,10 @@ fn main() -> Result<()> { onto_vector_clock_id, )?; - let mut last_ordering_node = None; for update in &conflicts_and_updates.updates { - match update { - dal::workspace_snapshot::update::Update::NewEdge { - source, - destination, - .. - } => { - if matches!(source.node_weight_kind, NodeWeightDiscriminants::Ordering) { - if let Some(ordering_node) = &last_ordering_node { - if let NodeWeight::Ordering(ordering) = ordering_node { - dbg!(destination, ordering.order()); - } - } - } - } - dal::workspace_snapshot::update::Update::RemoveEdge { .. } => {} - dal::workspace_snapshot::update::Update::ReplaceSubgraph { onto, .. } => { - if matches!(onto.node_weight_kind, NodeWeightDiscriminants::Ordering) { - last_ordering_node = onto_graph - .get_node_weight_opt(onto.index) - .expect("couldn't get node") - .map(ToOwned::to_owned); - } - } - dal::workspace_snapshot::update::Update::MergeCategoryNodes { .. } => {} - } + dbg!(update); } - dbg!(to_rebase_graph.perform_updates( - to_rebase_vector_clock_id, - &onto_graph, - &conflicts_and_updates.updates, - ))?; - Ok(()) } diff --git a/lib/dal/examples/snapshot-analyzer/main.rs b/lib/dal/examples/snapshot-analyzer/main.rs index 94737d18a9..f3f268d813 100644 --- a/lib/dal/examples/snapshot-analyzer/main.rs +++ b/lib/dal/examples/snapshot-analyzer/main.rs @@ -1,15 +1,8 @@ -use std::{collections::HashMap, env, fs::File, io::prelude::*}; +use std::{env, fs::File, io::Read as _}; use si_layer_cache::db::serialize; -use dal::{ - workspace_snapshot::{ - content_address::ContentAddressDiscriminants, - node_weight::{NodeWeight, NodeWeightDiscriminants}, - vector_clock::HasVectorClocks, - }, - EdgeWeightKindDiscriminants, WorkspaceSnapshotGraph, -}; +use dal::WorkspaceSnapshotGraph; use tokio::time::Instant; type Result = std::result::Result>; @@ -31,102 +24,7 @@ async fn main() -> Result<()> { let now = Instant::now(); let graph: WorkspaceSnapshotGraph = serialize::from_bytes(&snap_bytes)?; println!("deserialization took: {:?}", now.elapsed()); - let inner_graph = graph.graph(); - - let mut edge_kind_counts = HashMap::new(); - let mut node_kind_counts = HashMap::new(); - - let mut edge_count = 0; - let mut node_count = 0; - let mut edge_vector_clock_first_seen_entries = 0; - let mut edge_vector_clock_write_entries = 0; - - for edge_weight in inner_graph.edge_weights() { - edge_count += 1; - edge_vector_clock_first_seen_entries += edge_weight.vector_clock_first_seen().len(); - edge_vector_clock_write_entries += edge_weight.vector_clock_write().len(); - - let kind: EdgeWeightKindDiscriminants = edge_weight.kind().into(); - let kind_string = format!("{:?}", kind); - - edge_kind_counts - .entry(kind_string) - .and_modify(|count| *count += 1) - .or_insert(1); - } - - let mut node_first_seen_entries = 0; - let mut node_recently_seen_entries = 0; - let mut node_write_entries = 0; - - for node_weight in inner_graph.node_weights() { - node_count += 1; - - node_first_seen_entries += node_weight.vector_clock_first_seen().len(); - node_recently_seen_entries += node_weight.vector_clock_recently_seen().len(); - node_write_entries += node_weight.vector_clock_write().len(); - - let kind_string = { - if let NodeWeight::Content(content_node) = node_weight { - let cad_discrim: ContentAddressDiscriminants = - content_node.content_address().into(); - cad_discrim.to_string() - } else { - let kind: NodeWeightDiscriminants = node_weight.into(); - kind.to_string() - } - }; - - node_kind_counts - .entry(kind_string) - .and_modify(|count| *count += 1) - .or_insert(1); - } - - println!("edges: {edge_count}, nodes: {node_count}"); - - println!( - "\nedge vector clock first seen entries: {edge_vector_clock_first_seen_entries}, {} per edge", - edge_vector_clock_first_seen_entries / edge_count - ); - println!( - "edge vector clock write entries: {edge_vector_clock_write_entries}, {} per edge", - edge_vector_clock_write_entries / edge_count - ); - - // 128 bit id, 64 bit timestamp = 24bytes - let rough_bytes = (edge_vector_clock_first_seen_entries + edge_vector_clock_write_entries) * 24; - - println!( - "edge vector clocks are ~{} bytes, which is {}% of the total snapshot", - rough_bytes, - (rough_bytes as f64 / decompressed.len() as f64) * 100.0 - ); - - println!("\nEdge kinds:"); - - for (k, v) in edge_kind_counts { - println!("\t{k}: {v}"); - } - - println!( - "\nnode vector clock first seen entries: {node_first_seen_entries}, {} per node", - node_first_seen_entries / node_count - ); - println!( - "node vector clock recently seen entries: {node_recently_seen_entries}, {} per node", - node_recently_seen_entries / node_count - ); - println!( - "node vector clock write entries: {node_write_entries}, {} per node", - node_write_entries / node_count - ); - - println!("\nNode kinds:"); - - for (k, v) in node_kind_counts { - println!("\t{k}: {v}"); - } + let _inner_graph = graph.graph(); Ok(()) } diff --git a/lib/dal/src/attribute/value/dependent_value_graph.rs b/lib/dal/src/attribute/value/dependent_value_graph.rs index 0aae94a52a..ccce056f0b 100644 --- a/lib/dal/src/attribute/value/dependent_value_graph.rs +++ b/lib/dal/src/attribute/value/dependent_value_graph.rs @@ -88,7 +88,7 @@ impl DependentValueGraph { // at the time the job was created. if workspace_snapshot .try_get_node_index_by_id(initial_id) - .await? + .await .is_none() { debug!(%initial_id, "missing node, skipping it in DependentValueGraph"); diff --git a/lib/dal/src/builtins/func.rs b/lib/dal/src/builtins/func.rs index a6cf5d242a..e8a1657bf1 100644 --- a/lib/dal/src/builtins/func.rs +++ b/lib/dal/src/builtins/func.rs @@ -4,6 +4,7 @@ use crate::module::Module; use crate::{ func::intrinsics::IntrinsicFunc, pkg::import_pkg_from_pkg, BuiltinsResult, DalContext, }; +use telemetry::prelude::*; /// We want the src/builtins/func/** files to be available at run time inside of the Docker container /// that we build, but it would be nice to not have to include arbitrary bits of the source tree when @@ -36,8 +37,11 @@ pub async fn migrate_intrinsics(ctx: &DalContext) -> BuiltinsResult<()> { .await? .is_none() { + info!("importing"); import_pkg_from_pkg(ctx, &intrinsics_pkg, None).await?; + info!("imported, commiting"); ctx.blocking_commit().await?; + info!("commit finished"); } Ok(()) diff --git a/lib/dal/src/change_set.rs b/lib/dal/src/change_set.rs index a629a396e4..cd1f59d31d 100644 --- a/lib/dal/src/change_set.rs +++ b/lib/dal/src/change_set.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; use std::str::FromStr; +use std::sync::Arc; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -14,6 +15,7 @@ use si_events::{ulid::Ulid, WorkspaceSnapshotAddress}; use telemetry::prelude::*; use crate::context::{Conflicts, RebaseRequest}; +use crate::slow_rt::SlowRuntimeError; use crate::workspace_snapshot::vector_clock::VectorClockId; use crate::{ action::{ActionError, ActionId}, @@ -46,6 +48,8 @@ pub enum ChangeSetError { InvalidActor(UserPk), #[error("invalid user system init")] InvalidUserSystemInit, + #[error("tokio join error: {0}")] + Join(#[from] tokio::task::JoinError), #[error("layer db error: {0}")] LayerDb(#[from] LayerDbError), #[error("ulid monotonic error: {0}")] @@ -62,6 +66,8 @@ pub enum ChangeSetError { Pg(#[from] PgError), #[error("serde json error: {0}")] SerdeJson(#[from] serde_json::Error), + #[error("slow runtime error: {0}")] + SlowRuntime(#[from] SlowRuntimeError), #[error("transactions error: {0}")] Transactions(#[from] TransactionsError), #[error("ulid decode error: {0}")] @@ -453,15 +459,26 @@ impl ChangeSet { let to_rebase_change_set_id = self .base_change_set_id .ok_or(ChangeSetError::NoBaseChangeSet(self.id))?; - let onto_workspace_snapshot_address = self - .workspace_snapshot_address - .ok_or(ChangeSetError::NoWorkspaceSnapshot(self.id))?; - let rebase_request = RebaseRequest { - onto_workspace_snapshot_address, - onto_vector_clock_id: ctx.vector_clock_id()?, + + let to_rebase_workspace_snapshot = Arc::new( + WorkspaceSnapshot::find_for_change_set(ctx, to_rebase_change_set_id) + .await + .map_err(Box::new)?, + ); + + if let Some(rebase_batch) = WorkspaceSnapshot::calculate_rebase_batch( to_rebase_change_set_id, - }; - ctx.do_rebase_request(rebase_request).await?; + to_rebase_workspace_snapshot, + ctx.workspace_snapshot().map_err(Box::new)?, + ) + .await + .map_err(Box::new)? + { + let rebase_batch_address = ctx.write_rebase_batch(rebase_batch).await?; + + let rebase_request = RebaseRequest::new(to_rebase_change_set_id, rebase_batch_address); + ctx.do_rebase_request(rebase_request).await?; + } self.update_status(ctx, ChangeSetStatus::Applied).await?; let user = Self::extract_userid_from_context(ctx).await; diff --git a/lib/dal/src/component.rs b/lib/dal/src/component.rs index a22ec4e409..ee8fbcad67 100644 --- a/lib/dal/src/component.rs +++ b/lib/dal/src/component.rs @@ -1376,11 +1376,7 @@ impl Component { component_id: ComponentId, ) -> ComponentResult> { let id: Ulid = component_id.into(); - if let Some(node_index) = ctx - .workspace_snapshot()? - .try_get_node_index_by_id(id) - .await? - { + if let Some(node_index) = ctx.workspace_snapshot()?.try_get_node_index_by_id(id).await { let node_weight = ctx .workspace_snapshot()? .get_node_weight(node_index) @@ -2465,7 +2461,7 @@ impl Component { match ctx .workspace_snapshot()? .try_get_node_index_by_id(component_id) - .await? + .await { Some(component_idx) => { let component_node_weight = ctx diff --git a/lib/dal/src/component/frame.rs b/lib/dal/src/component/frame.rs index f4c097f020..abe10ad9d3 100644 --- a/lib/dal/src/component/frame.rs +++ b/lib/dal/src/component/frame.rs @@ -138,8 +138,6 @@ impl Frame { parent_id: ComponentId, child_id: ComponentId, ) -> FrameResult<()> { - let total_start = std::time::Instant::now(); - // cache current map of input <-> output sockets based on what the parent knows about right now!!!! let initial_impacted_values: HashSet = Self::get_all_inferred_connections_for_component_tree(ctx, parent_id, child_id).await?; @@ -150,10 +148,6 @@ impl Frame { if let Some(current_parent_id) = Component::get_parent_by_id(ctx, child_id).await? { //remove the edge Component::remove_edge_from_frame(ctx, current_parent_id, child_id).await?; - info!( - "Remove existing edge from frame took: {:?}", - total_start.elapsed() - ); // get the map of input <-> output sockets after the edge was removed. so we can determine if more // updates need to be made due to the upsert // note we need to see what the current parent's tree looked like, as there could be nested impacts @@ -172,10 +166,6 @@ impl Frame { Component::add_edge_to_frame(ctx, parent_id, child_id, EdgeWeightKind::FrameContains) .await?; drop(cycle_check_guard); - info!( - "Cycle Check Guard dropped, add edge took {:?}", - total_start.elapsed() - ); // now figure out what needs to rerun! let mut values_to_run: HashSet = HashSet::new(); diff --git a/lib/dal/src/context.rs b/lib/dal/src/context.rs index 98aac9eb63..5cbdfccb1c 100644 --- a/lib/dal/src/context.rs +++ b/lib/dal/src/context.rs @@ -7,6 +7,7 @@ use si_crypto::SymmetricCryptoService; use si_crypto::VeritechEncryptionKey; use si_data_nats::{NatsClient, NatsError, NatsTxn}; use si_data_pg::{InstrumentedClient, PgError, PgPool, PgPoolError, PgPoolResult, PgTxn}; +use si_events::rebase_batch_address::RebaseBatchAddress; use si_events::ulid::Ulid; use si_events::VectorClockActorId; use si_events::VectorClockChangeSetId; @@ -27,8 +28,12 @@ use veritech_client::Client as VeritechClient; use crate::feature_flags::FeatureFlagService; use crate::job::definition::AttributeValueBasedJobIdentifier; use crate::layer_db_types::ContentTypes; +use crate::slow_rt::SlowRuntimeError; use crate::workspace_snapshot::{ - conflict::Conflict, graph::WorkspaceSnapshotGraph, update::Update, vector_clock::VectorClockId, + conflict::Conflict, + graph::{RebaseBatch, WorkspaceSnapshotGraph}, + update::Update, + vector_clock::VectorClockId, }; use crate::{ change_set::{ChangeSet, ChangeSetId}, @@ -42,9 +47,9 @@ use crate::{ AttributeValueId, HistoryActor, StandardModel, Tenancy, TenancyError, Visibility, WorkspacePk, WorkspaceSnapshot, }; -use crate::{EncryptedSecret, Workspace}; +use crate::{slow_rt, EncryptedSecret, Workspace}; -pub type DalLayerDb = LayerDb; +pub type DalLayerDb = LayerDb; /// A context type which contains handles to common core service dependencies. /// @@ -374,20 +379,6 @@ impl DalContext { } } - fn get_rebase_request( - &self, - onto_workspace_snapshot_address: WorkspaceSnapshotAddress, - ) -> Result { - let vector_clock_id = self.vector_clock_id()?; - Ok(RebaseRequest { - onto_workspace_snapshot_address, - // the vector clock id of the current change set is just the id - // of the current change set - to_rebase_change_set_id: self.change_set_id(), - onto_vector_clock_id: vector_clock_id, - }) - } - pub async fn do_rebase_request( &self, rebase_request: RebaseRequest, @@ -450,14 +441,53 @@ impl DalContext { } } + pub async fn write_rebase_batch( + &self, + rebase_batch: RebaseBatch, + ) -> Result { + let layer_db = self.layer_db().clone(); + let events_tenancy = self.events_tenancy(); + let events_actor = self.events_actor(); + + let rebase_batch_address = slow_rt::spawn(async move { + let (rebase_batch_address, _) = layer_db + .rebase_batch() + .write(Arc::new(rebase_batch), None, events_tenancy, events_actor) + .await?; + + Ok::(rebase_batch_address) + })? + .await??; + + Ok(rebase_batch_address) + } + + pub async fn write_current_rebase_batch( + &self, + ) -> Result, TransactionsError> { + Ok(if let Some(snapshot) = &self.workspace_snapshot { + if let Some(rebase_batch) = snapshot + .current_rebase_batch(self.change_set_id(), self.vector_clock_id()?) + .await + .map_err(Box::new)? + { + Some(self.write_rebase_batch(rebase_batch).await?) + } else { + None + } + } else { + None + }) + } + /// Consumes all inner transactions and committing all changes made within them. pub async fn commit(&self) -> Result<(), TransactionsError> { - let rebase_request = match self.write_snapshot().await? { - Some(workspace_snapshot_address) => { - Some(self.get_rebase_request(workspace_snapshot_address)?) - } - None => None, - }; + let rebase_request = self + .write_current_rebase_batch() + .await? + .map(|rebase_batch_address| { + RebaseRequest::new(self.change_set_id(), rebase_batch_address) + }); if let Some(conflicts) = if self.blocking { self.blocking_commit_internal(rebase_request).await? @@ -564,12 +594,14 @@ impl DalContext { /// Consumes all inner transactions, committing all changes made within them, and /// blocks until all queued jobs have reported as finishing. pub async fn blocking_commit(&self) -> Result, TransactionsError> { - let rebase_request = match self.write_snapshot().await? { - Some(workspace_snapshot_address) => { - Some(self.get_rebase_request(workspace_snapshot_address)?) - } - None => None, - }; + let rebase_request = self + .write_current_rebase_batch() + .await? + .map(|rebase_batch_address| { + RebaseRequest::new(self.change_set_id(), rebase_batch_address) + }); + + info!("rebase_request: {:?}", rebase_request); self.blocking_commit_internal(rebase_request).await } @@ -1108,6 +1140,8 @@ pub enum TransactionsError { ConflictsOccurred(Conflicts), #[error(transparent)] JobQueueProcessor(#[from] JobQueueProcessorError), + #[error("tokio join error: {0}")] + Join(#[from] tokio::task::JoinError), #[error(transparent)] LayerDb(#[from] LayerDbError), #[error(transparent)] @@ -1118,10 +1152,12 @@ pub enum TransactionsError { Pg(#[from] PgError), #[error(transparent)] PgPool(#[from] PgPoolError), - #[error("rebase of snapshot {0} change set id {1} failed {2}")] - RebaseFailed(WorkspaceSnapshotAddress, ChangeSetId, String), + #[error("rebase of batch {0} for change set id {1} failed: {2}")] + RebaseFailed(RebaseBatchAddress, ChangeSetId, String), #[error(transparent)] SerdeJson(#[from] serde_json::Error), + #[error("slow rt error: {0}")] + SlowRuntime(#[from] SlowRuntimeError), #[error(transparent)] Tenancy(#[from] TenancyError), #[error("Unable to acquire lock: {0}")] @@ -1139,7 +1175,7 @@ pub enum TransactionsError { #[error("workspace not set on DalContext")] WorkspaceNotSet, #[error("workspace snapshot error: {0}")] - WorkspaceSnapshot(Box), + WorkspaceSnapshot(#[from] Box), } impl TransactionsError { @@ -1214,8 +1250,19 @@ pub struct Transactions { #[derive(Clone, Debug)] pub struct RebaseRequest { pub to_rebase_change_set_id: ChangeSetId, - pub onto_workspace_snapshot_address: WorkspaceSnapshotAddress, - pub onto_vector_clock_id: VectorClockId, + pub rebase_batch_address: RebaseBatchAddress, +} + +impl RebaseRequest { + pub fn new( + to_rebase_change_set_id: ChangeSetId, + rebase_batch_address: RebaseBatchAddress, + ) -> Self { + Self { + to_rebase_change_set_id, + rebase_batch_address, + } + } } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -1275,8 +1322,7 @@ async fn rebase( .rebase() .rebase_and_wait( rebase_request.to_rebase_change_set_id.into(), - rebase_request.onto_workspace_snapshot_address, - rebase_request.onto_vector_clock_id, + rebase_request.rebase_batch_address, metadata, ) .await?; @@ -1287,16 +1333,15 @@ async fn rebase( ); match rebase_finished_activity.payload { ActivityPayload::RebaseFinished(rebase_finished) => match rebase_finished.status() { - RebaseStatus::Success { updates_performed } => { - let updates = Updates { - updates_found: serde_json::from_str(updates_performed)?, - }; - span.record("si.updates", updates_performed); - span.record("si.updates.count", updates.updates_found.len().to_string()); + RebaseStatus::Success { + updates_performed: _, + } => { + //span.record("si.updates", updates_performed); + //span.record("si.updates.count", updates.updates_found.len().to_string()); Ok(None) } RebaseStatus::Error { message } => Err(TransactionsError::RebaseFailed( - rebase_request.onto_workspace_snapshot_address, + rebase_request.rebase_batch_address, rebase_request.to_rebase_change_set_id, message.to_string(), )), diff --git a/lib/dal/src/diagram.rs b/lib/dal/src/diagram.rs index fb2771ec7d..42fc68a5f0 100644 --- a/lib/dal/src/diagram.rs +++ b/lib/dal/src/diagram.rs @@ -392,7 +392,9 @@ impl Diagram { pub async fn assemble(ctx: &DalContext) -> DiagramResult { let mut diagram_edges: Vec = vec![]; let mut diagram_inferred_edges: Vec = vec![]; + let components = Component::list(ctx).await?; + let mut virtual_and_real_components_by_id: HashMap = components.iter().cloned().map(|c| (c.id(), c)).collect(); let mut component_views = Vec::with_capacity(components.len()); @@ -403,13 +405,17 @@ impl Diagram { .iter() .copied() .collect(); - let new_attribute_prototype_argument_ids: HashSet = ctx - .workspace_snapshot()? - .socket_edges_added_relative_to_base(ctx) - .await? - .iter() - .copied() - .collect(); + + // let new_attribute_prototype_argument_ids: HashSet = ctx + // .workspace_snapshot()? + // .socket_edges_added_relative_to_base(ctx) + // .await? + // .iter() + // .copied() + // .collect(); + + let new_attribute_prototype_argument_ids: HashSet = + HashSet::new(); for component in &components { let component_change_status = if new_component_ids.contains(&component.id()) { @@ -441,7 +447,6 @@ impl Diagram { edge_change_status, )?); } - for inferred_incoming_connection in component.inferred_incoming_connections(ctx).await? { diagram_inferred_edges.push(SummaryDiagramInferredEdge::assemble( @@ -502,10 +507,12 @@ impl Diagram { // We need to bring in any AttributePrototypeArguments for incoming & outgoing // connections that have been removed. - let removed_attribute_prototype_argument_ids = ctx - .workspace_snapshot()? - .socket_edges_removed_relative_to_base(ctx) - .await?; + let removed_attribute_prototype_argument_ids: Vec = + vec![]; + // = ctx + // .workspace_snapshot()? + // .socket_edges_removed_relative_to_base(ctx) + // .await?; let mut incoming_connections_by_component_id: HashMap< ComponentId, Vec, diff --git a/lib/dal/src/job/definition/compute_validation.rs b/lib/dal/src/job/definition/compute_validation.rs index d54d874662..6be4b0b129 100644 --- a/lib/dal/src/job/definition/compute_validation.rs +++ b/lib/dal/src/job/definition/compute_validation.rs @@ -93,7 +93,7 @@ impl JobConsumer for ComputeValidation { // the job was created. if workspace_snapshot .try_get_node_index_by_id(av_id) - .await? + .await .is_none() { debug!("Attribute Value {av_id} missing, skipping it in ComputeValidations"); diff --git a/lib/dal/src/job/definition/dependent_values_update.rs b/lib/dal/src/job/definition/dependent_values_update.rs index fb7265db86..c7df55a0e4 100644 --- a/lib/dal/src/job/definition/dependent_values_update.rs +++ b/lib/dal/src/job/definition/dependent_values_update.rs @@ -270,6 +270,7 @@ impl DependentValuesUpdate { } debug!("DependentValuesUpdate took: {:?}", start.elapsed()); + Ok(match ctx.commit().await { Ok(_) => JobCompletionState::Done, Err(err) => match err { diff --git a/lib/dal/src/job/processor/nats_processor.rs b/lib/dal/src/job/processor/nats_processor.rs index bb6e57228f..d6921287b3 100644 --- a/lib/dal/src/job/processor/nats_processor.rs +++ b/lib/dal/src/job/processor/nats_processor.rs @@ -136,8 +136,6 @@ impl JobQueueProcessor for NatsProcessor { } } - info!("processed_queue"); - if !results.is_empty() { Err(BlockingJobError::JobExecution( results diff --git a/lib/dal/src/module.rs b/lib/dal/src/module.rs index 64614b9c2b..c739b7eac5 100644 --- a/lib/dal/src/module.rs +++ b/lib/dal/src/module.rs @@ -515,8 +515,7 @@ impl Module { debug!(?synced_modules.installable, "collected installable modules"); debug!(?synced_modules.upgradeable, "collected upgradeable modules"); - - info!("syncing modules took: {:?}", start.elapsed()); + debug!("syncing modules took: {:?}", start.elapsed()); Ok(synced_modules) } diff --git a/lib/dal/src/workspace_snapshot.rs b/lib/dal/src/workspace_snapshot.rs index f56d125576..a4ca266cac 100644 --- a/lib/dal/src/workspace_snapshot.rs +++ b/lib/dal/src/workspace_snapshot.rs @@ -32,7 +32,7 @@ pub mod node_weight; pub mod update; pub mod vector_clock; -use graph::WorkspaceSnapshotGraph; +use graph::{RebaseBatch, WorkspaceSnapshotGraph}; use std::collections::HashSet; use std::sync::atomic::AtomicBool; use std::sync::Arc; @@ -78,7 +78,6 @@ use crate::{ pk!(NodeId); #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct NodeInformation { - pub index: NodeIndex, pub node_weight_kind: NodeWeightDiscriminants, pub id: NodeId, } @@ -98,6 +97,8 @@ pub enum WorkspaceSnapshotError { ChangeSetMissingWorkspaceSnapshotAddress(ChangeSetId), #[error("Component error: {0}")] Component(#[from] Box), + #[error("conflicts detected when generating rebase batch")] + ConflictsInRebaseBatch, #[error("edge weight error: {0}")] EdgeWeight(#[from] EdgeWeightError), #[error("join error: {0}")] @@ -106,6 +107,8 @@ pub enum WorkspaceSnapshotError { LayerDb(#[from] si_layer_cache::LayerDbError), #[error("missing content from store for id: {0}")] MissingContentFromStore(Ulid), + #[error("could not find a max vector clock for change set id {0}")] + MissingVectorClockForChangeSet(ChangeSetId), #[error("monotonic error: {0}")] Monotonic(#[from] ulid::MonotonicError), #[error("NodeWeight error: {0}")] @@ -351,6 +354,88 @@ impl WorkspaceSnapshot { self.cycle_check.load(std::sync::atomic::Ordering::Relaxed) } + pub async fn current_rebase_batch( + &self, + to_rebase_change_set_id: ChangeSetId, + vector_clock_id: VectorClockId, + ) -> WorkspaceSnapshotResult> { + let self_clone = self.clone(); + let conflicts_and_updates = slow_rt::spawn(async move { + let mut working_copy = self_clone.working_copy_mut().await; + working_copy.cleanup(); + working_copy.mark_graph_seen(vector_clock_id)?; + + let to_rebase_vector_clock_id = self_clone + .read_only_graph + .max_recently_seen_clock_id(Some(to_rebase_change_set_id)) + .ok_or(WorkspaceSnapshotError::MissingVectorClockForChangeSet( + to_rebase_change_set_id, + ))?; + + let onto_vector_clock_id = working_copy.max_recently_seen_clock_id(None).ok_or( + WorkspaceSnapshotError::MissingVectorClockForChangeSet(to_rebase_change_set_id), + )?; + + let conflicts_and_updates = self_clone.read_only_graph.detect_conflicts_and_updates( + to_rebase_vector_clock_id, + &working_copy, + onto_vector_clock_id, + )?; + Ok::(conflicts_and_updates) + })? + .await??; + + Ok(if !conflicts_and_updates.conflicts.is_empty() { + Err(WorkspaceSnapshotError::ConflictsInRebaseBatch)? + } else if conflicts_and_updates.updates.is_empty() { + None + } else { + Some(RebaseBatch::new(conflicts_and_updates.updates)) + }) + } + + pub async fn calculate_rebase_batch( + to_rebase_change_set_id: ChangeSetId, + to_rebase_workspace_snapshot: Arc, + onto_workspace_snapshot: Arc, + ) -> WorkspaceSnapshotResult> { + let conflicts_and_updates = slow_rt::spawn(async move { + let to_rebase_vector_clock_id = to_rebase_workspace_snapshot + .max_recently_seen_clock_id(Some(to_rebase_change_set_id)) + .await? + .ok_or(WorkspaceSnapshotError::MissingVectorClockForChangeSet( + to_rebase_change_set_id, + ))?; + + let onto_vector_clock_id = onto_workspace_snapshot + .max_recently_seen_clock_id(None) + .await? + .ok_or(WorkspaceSnapshotError::MissingVectorClockForChangeSet( + to_rebase_change_set_id, + ))?; + + let conflicts_and_updates = to_rebase_workspace_snapshot + .detect_conflicts_and_updates( + to_rebase_vector_clock_id, + &onto_workspace_snapshot, + onto_vector_clock_id, + ) + .await?; + + Ok::(conflicts_and_updates) + })? + .await??; + + Ok(if !conflicts_and_updates.conflicts.is_empty() { + // error + Err(WorkspaceSnapshotError::ConflictsInRebaseBatch)? + } else if conflicts_and_updates.updates.is_empty() { + None + } else { + Some(RebaseBatch::new(conflicts_and_updates.updates)) + }) + } + #[instrument( name = "workspace_snapshot.write", level = "debug", @@ -783,10 +868,14 @@ impl WorkspaceSnapshot { } /// Write the snapshot to disk. *WARNING* can panic! Use only for debugging - pub async fn write_to_disk(&self, file_suffix: &str) { + pub async fn write_working_copy_to_disk(&self, file_suffix: &str) { self.working_copy().await.write_to_disk(file_suffix); } + pub fn write_readonly_graph_to_disk(&self, file_suffix: &str) { + self.read_only_graph.write_to_disk(file_suffix); + } + #[instrument( name = "workspace_snapshot.get_node_index_by_id", level = "debug", @@ -806,11 +895,8 @@ impl WorkspaceSnapshot { skip_all, fields() )] - pub async fn try_get_node_index_by_id( - &self, - id: impl Into, - ) -> WorkspaceSnapshotResult> { - Ok(self.working_copy().await.try_get_node_index_by_id(id)?) + pub async fn try_get_node_index_by_id(&self, id: impl Into) -> Option { + self.working_copy().await.try_get_node_index_by_id(id) } #[instrument( @@ -831,8 +917,6 @@ impl WorkspaceSnapshot { ctx: &DalContext, workspace_snapshot_addr: WorkspaceSnapshotAddress, ) -> WorkspaceSnapshotResult { - let start = tokio::time::Instant::now(); - let snapshot = match ctx .layer_db() .workspace_snapshot() @@ -852,8 +936,6 @@ impl WorkspaceSnapshot { }, }; - debug!("snapshot fetch took: {:?}", start.elapsed()); - Ok(Self { address: Arc::new(RwLock::new(workspace_snapshot_addr)), read_only_graph: snapshot, @@ -1314,14 +1396,12 @@ impl WorkspaceSnapshot { pub async fn perform_updates( &self, to_rebase_vector_clock_id: VectorClockId, - onto: &WorkspaceSnapshot, updates: &[Update], ) -> WorkspaceSnapshotResult<()> { - Ok(self.working_copy_mut().await.perform_updates( - to_rebase_vector_clock_id, - &*onto.working_copy().await, - updates, - )?) + Ok(self + .working_copy_mut() + .await + .perform_updates(to_rebase_vector_clock_id, updates)?) } /// Mark whether a prop can be used as an input to a function. Props below @@ -1451,20 +1531,6 @@ impl WorkspaceSnapshot { base_change_set_id, ))?; - // We need to use the index of the Category node in the base snapshot, as that's the - // `to_rebase` when detecting conflicts & updates later, and the source node index in the - // Update is from the `to_rebase` graph. - let component_category_idx = if let Some((_category_id, category_idx)) = base_snapshot - .read_only_graph - .get_category_node(None, CategoryNodeKind::Component)? - { - category_idx - } else { - // Can't have any new Components if there's no Component category node to put them - // under. - return Ok(new_component_ids); - }; - // If there is no vector clock in this snapshot for the current change // set, that's because the snapshot has *just* been forked from the base // change set, and has not been written to yet @@ -1481,35 +1547,19 @@ impl WorkspaceSnapshot { for update in &conflicts_and_updates.updates { match update { - Update::RemoveEdge { - source: _, - destination: _, - edge_kind: _, - } - | Update::ReplaceSubgraph { - onto: _, - to_rebase: _, - } - | Update::MergeCategoryNodes { - to_rebase_category_id: _, - onto_category_id: _, - } => { - /* Updates unused for determining if a Component is new with regards to the updates */ - } - Update::NewEdge { - source, - destination, - edge_weight: _, - } => { - if !(source.index == component_category_idx - && destination.node_weight_kind == NodeWeightDiscriminants::Component) - { - // We only care about new edges coming from the Component category node, - // and going to Component nodes, so keep looking. - continue; + Update::RemoveEdge { .. } + | Update::ReplaceNode { .. } + | Update::NewEdge { .. } => {} + Update::NewNode { node_weight } => { + if let NodeWeight::Component(inner) = node_weight { + if base_snapshot + .read_only_graph + .try_get_node_index_by_id(inner.id) + .is_none() + { + new_component_ids.push(inner.id.into()) + } } - - new_component_ids.push(ComponentId::from(Ulid::from(destination.id))); } } } @@ -1557,14 +1607,11 @@ impl WorkspaceSnapshot { base_change_set_id, ))?; - // We need to use the index of the Category node in the base snapshot, as that's the - // `to_rebase` when detecting conflicts & updates later, and the source node index in the - // Update is from the `to_rebase` graph. - let component_category_idx = if let Some((_category_id, category_idx)) = base_snapshot + let component_category_id = if let Some((category_id, _)) = base_snapshot .read_only_graph .get_category_node(None, CategoryNodeKind::Component)? { - category_idx + category_id } else { // Can't have any new Components if there's no Component category node to put them // under. @@ -1586,14 +1633,7 @@ impl WorkspaceSnapshot { for update in &conflicts_and_updates.updates { match update { - Update::ReplaceSubgraph { - onto: _, - to_rebase: _, - } - | Update::MergeCategoryNodes { - to_rebase_category_id: _, - onto_category_id: _, - } => { + Update::ReplaceNode { .. } | Update::NewNode { .. } => { /* Updates unused for determining if a Component is removed in regard to the updates */ } Update::NewEdge { @@ -1602,7 +1642,7 @@ impl WorkspaceSnapshot { edge_weight: _, } => { // get updates that add an edge from the Components category to a component, which implies component creation - if source.index != component_category_idx + if source.id != component_category_id.into() || destination.node_weight_kind != NodeWeightDiscriminants::Component { continue; @@ -1616,7 +1656,7 @@ impl WorkspaceSnapshot { edge_kind: _, } => { // get updates that remove an edge from the Components category to a component, which implies component deletion - if source.index != component_category_idx + if source.id != component_category_id.into() || destination.node_weight_kind != NodeWeightDiscriminants::Component { continue; @@ -1692,19 +1732,9 @@ impl WorkspaceSnapshot { for update in &conflicts_and_updates.updates { match update { - Update::RemoveEdge { - source: _, - destination: _, - edge_kind: _, - } - | Update::ReplaceSubgraph { - onto: _, - to_rebase: _, - } - | Update::MergeCategoryNodes { - to_rebase_category_id: _, - onto_category_id: _, - } => { + Update::NewNode { .. } + | Update::RemoveEdge { .. } + | Update::ReplaceNode { .. } => { // Updates unused for determining if a socket to socket connection (in frontend // terms) is new. } @@ -1854,19 +1884,9 @@ impl WorkspaceSnapshot { for update in conflicts_and_updates.updates { match update { - Update::ReplaceSubgraph { - onto: _, - to_rebase: _, - } - | Update::NewEdge { - source: _, - destination: _, - edge_weight: _, - } - | Update::MergeCategoryNodes { - to_rebase_category_id: _, - onto_category_id: _, - } => { + Update::ReplaceNode { .. } + | Update::NewEdge { .. } + | Update::NewNode { .. } => { /* Updates unused for determining if a connection between sockets has been removed */ } Update::RemoveEdge { diff --git a/lib/dal/src/workspace_snapshot/graph.rs b/lib/dal/src/workspace_snapshot/graph.rs index 32f8796f4e..f01ccf6de1 100644 --- a/lib/dal/src/workspace_snapshot/graph.rs +++ b/lib/dal/src/workspace_snapshot/graph.rs @@ -137,6 +137,21 @@ pub struct ConflictsAndUpdates { pub updates: Vec, } +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct RebaseBatch { + updates: Vec, +} + +impl RebaseBatch { + pub fn new(updates: Vec) -> Self { + Self { updates } + } + + pub fn updates(&self) -> &[Update] { + &self.updates + } +} + impl std::fmt::Debug for WorkspaceSnapshotGraphV1 { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("WorkspaceSnapshotGraph") @@ -997,13 +1012,10 @@ impl WorkspaceSnapshotGraphV1 { } #[inline(always)] - pub(crate) fn try_get_node_index_by_id( - &self, - id: impl Into, - ) -> WorkspaceSnapshotGraphResult> { + pub(crate) fn try_get_node_index_by_id(&self, id: impl Into) -> Option { let id = id.into(); - Ok(self.node_index_by_id.get(&id).copied()) + self.node_index_by_id.get(&id).copied() } fn get_node_index_by_lineage(&self, lineage_id: Ulid) -> HashSet { @@ -1054,81 +1066,6 @@ impl WorkspaceSnapshotGraphV1 { algo::has_path_connecting(&self.graph, self.root_index, node, None) } - /// Import a single node from the `other` [`WorkspaceSnapshotGraph`] into `self`, if it does - /// not already exist in `self`. - pub fn find_in_self_or_import_node_from_other( - &mut self, - other: &WorkspaceSnapshotGraphV1, - other_node_index: NodeIndex, - ) -> WorkspaceSnapshotGraphResult<()> { - let node_weight_to_import = other.get_node_weight(other_node_index)?.clone(); - let node_weight_id = node_weight_to_import.id(); - let node_weight_lineage_id = node_weight_to_import.lineage_id(); - - if let Some(equivalent_node_index) = - self.find_equivalent_node(node_weight_id, node_weight_lineage_id)? - { - let equivalent_node_weight = self.get_node_weight(equivalent_node_index)?; - if !equivalent_node_weight - .vector_clock_write() - .is_newer_than(node_weight_to_import.vector_clock_write()) - { - self.add_node(node_weight_to_import)?; - - self.replace_references(equivalent_node_index)?; - } - } else { - self.add_node(node_weight_to_import)?; - }; - - Ok(()) - } - - pub fn import_subgraph( - &mut self, - other: &WorkspaceSnapshotGraphV1, - root_index: NodeIndex, - ) -> WorkspaceSnapshotGraphResult<()> { - let mut dfs = petgraph::visit::DfsPostOrder::new(&other.graph, root_index); - while let Some(node_index_to_copy) = dfs.next(&other.graph) { - let node_weight_to_copy = other.get_node_weight(node_index_to_copy)?.clone(); - let node_weight_id = node_weight_to_copy.id(); - let node_weight_lineage_id = node_weight_to_copy.lineage_id(); - - // The following assumes there are no conflicts between "self" and "other". If there - // are conflicts between them, we shouldn't be running updates. - let node_index = if let Some(equivalent_node_index) = - self.find_equivalent_node(node_weight_id, node_weight_lineage_id)? - { - let equivalent_node_weight = self.get_node_weight(equivalent_node_index)?; - if equivalent_node_weight - .vector_clock_write() - .is_newer_than(node_weight_to_copy.vector_clock_write()) - { - equivalent_node_index - } else { - let new_node_index = self.add_node(node_weight_to_copy)?; - - self.replace_references(equivalent_node_index)?; - self.get_latest_node_idx(new_node_index)? - } - } else { - self.add_node(node_weight_to_copy)? - }; - - for edge in other.graph.edges_directed(node_index_to_copy, Outgoing) { - let target_id = other.get_node_weight(edge.target())?.id(); - let latest_target = self.get_node_index_by_id(target_id)?; - self.graph - .update_edge(node_index, latest_target, edge.weight().clone()); - } - - self.update_merkle_tree_hash(node_index)?; - } - - Ok(()) - } - pub fn import_component_subgraph( &mut self, vector_clock_id: VectorClockId, @@ -1338,12 +1275,9 @@ impl WorkspaceSnapshotGraphV1 { self.get_node_weight(container_ordering_index)? { for ordered_id in ordering_weight.order() { - ordered_child_indexes.push(*self.node_index_by_id.get(ordered_id).ok_or( - WorkspaceSnapshotGraphError::OrderingNodeHasNonexistentNodeInOrder( - ordering_weight.id(), - *ordered_id, - ), - )?); + if let Some(child_index) = self.node_index_by_id.get(ordered_id).copied() { + ordered_child_indexes.push(child_index); + } } } } else { @@ -1746,7 +1680,6 @@ impl WorkspaceSnapshotGraphV1 { pub fn perform_updates( &mut self, to_rebase_vector_clock_id: VectorClockId, - onto: &WorkspaceSnapshotGraphV1, updates: &[Update], ) -> WorkspaceSnapshotGraphResult<()> { for update in updates { @@ -1756,81 +1689,53 @@ impl WorkspaceSnapshotGraphV1 { destination, edge_weight, } => { - let updated_source = self.get_latest_node_idx(source.index)?; - let destination = - self.reuse_from_self_or_import_subgraph_from_onto(destination.index, onto)?; + let updated_source = self.try_get_node_index_by_id(source.id); + let destination = self.try_get_node_index_by_id(destination.id); - self.add_edge(updated_source, edge_weight.clone(), destination)?; + if let (Some(updated_source), Some(destination)) = (updated_source, destination) + { + self.add_edge(updated_source, edge_weight.clone(), destination)?; + } } Update::RemoveEdge { source, destination, edge_kind, } => { - let updated_source = self.get_latest_node_idx(source.index)?; - let destination = self.get_latest_node_idx(destination.index)?; - self.remove_edge_inner( - to_rebase_vector_clock_id, - updated_source, - destination, - *edge_kind, - // Updating the vector clocks here may cause us to pick - // the to_rebase node incorrectly in ReplaceSubgraph, if - // ReplaceSubgraph comes after this RemoveEdge - false, - )?; + let updated_source = self.try_get_node_index_by_id(source.id); + let destination = self.try_get_node_index_by_id(destination.id); + + if let (Some(updated_source), Some(destination)) = (updated_source, destination) + { + self.remove_edge_inner( + to_rebase_vector_clock_id, + updated_source, + destination, + *edge_kind, + // Updating the vector clocks here may cause us to pick + // the to_rebase node incorrectly in ReplaceNode, if + // ReplaceNode comes after this RemoveEdge + false, + )?; + } } - Update::ReplaceSubgraph { - onto: onto_subgraph_root, - to_rebase: to_rebase_subgraph_root, - } => { - let updated_to_rebase = - self.get_latest_node_idx(to_rebase_subgraph_root.index)?; - // We really only want to import the single node here, not the entire subgraph, - // as we also generate Updates on how to handle bringing in the child - // nodes. If we try to bring in the entire subgraph, we'll have a bad - // interaction with `replace_references` where we'll end up with the set - // union of the edges outgoing from both the "old" and "new" node. We only - // want the edges from the "old" node as we'll have further Update events - // that will handle bringing in the appropriate "new" edges. - self.find_in_self_or_import_node_from_other(onto, onto_subgraph_root.index)?; - self.replace_references(updated_to_rebase)?; + Update::NewNode { node_weight } => { + if self.try_get_node_index_by_id(node_weight.id()).is_none() { + self.add_node(node_weight.to_owned())?; + } } - Update::MergeCategoryNodes { - to_rebase_category_id, - onto_category_id, - } => { - self.merge_category_nodes(*to_rebase_category_id, *onto_category_id)?; + Update::ReplaceNode { node_weight } => { + let updated_to_rebase = self.try_get_node_index_by_id(node_weight.id()); + if let Some(updated_to_rebase) = updated_to_rebase { + self.add_node(node_weight.to_owned())?; + self.replace_references(updated_to_rebase)?; + } } } } Ok(()) } - fn merge_category_nodes( - &mut self, - to_rebase_category_id: Ulid, - onto_category_id: Ulid, - ) -> WorkspaceSnapshotGraphResult<()> { - // both categories should be on the graph at this point - let onto_cat_node_index = self.get_node_index_by_id(onto_category_id)?; - let new_targets_for_cat_node: Vec<(NodeIndex, EdgeWeight)> = self - .edges_directed(onto_cat_node_index, Outgoing) - .map(|edge_ref| (edge_ref.target(), edge_ref.weight().to_owned())) - .collect(); - - for (target_idx, edge_weight) in new_targets_for_cat_node { - let to_rebase_cat_node_idx = self.get_node_index_by_id(to_rebase_category_id)?; - self.add_edge(to_rebase_cat_node_idx, edge_weight, target_idx)?; - } - - let onto_cat_node_index = self.get_node_index_by_id(onto_category_id)?; - self.remove_node(onto_cat_node_index); - self.remove_node_id(onto_category_id); - - Ok(()) - } - /// Update node weight in place with a lambda. Use with caution. Generally /// we treat node weights as immutable and replace them by creating a new /// node with a new node weight and replacing references to point to the new @@ -1852,50 +1757,6 @@ impl WorkspaceSnapshotGraphV1 { Ok(()) } - - /// Given the node index for a node in other, find if a node exists in self that has the same - /// id as the node found in other. - fn find_latest_idx_in_self_from_other_idx( - &mut self, - other: &WorkspaceSnapshotGraphV1, - other_idx: NodeIndex, - ) -> WorkspaceSnapshotGraphResult> { - let other_id = other.get_node_weight(other_idx)?.id(); - - Ok(self.get_node_index_by_id(other_id).ok()) - } - - /// Find in self (and use it as-is) where self is the "to rebase" side or import the entire subgraph from "onto". - fn reuse_from_self_or_import_subgraph_from_onto( - &mut self, - unchecked: NodeIndex, - onto: &WorkspaceSnapshotGraphV1, - ) -> WorkspaceSnapshotGraphResult { - let unchecked_node_weight = onto.get_node_weight(unchecked)?; - - let found_or_created = { - let equivalent_node = if let Some(found) = - self.find_latest_idx_in_self_from_other_idx(onto, unchecked)? - { - Some(found) - } else { - self.find_equivalent_node( - unchecked_node_weight.id(), - unchecked_node_weight.lineage_id(), - )? - }; - - match equivalent_node { - Some(found_equivalent_node) => found_equivalent_node, - None => { - self.import_subgraph(onto, unchecked)?; - self.find_latest_idx_in_self_from_other_idx(onto, unchecked)? - .ok_or(WorkspaceSnapshotGraphError::NodeWeightNotFound)? - } - } - }; - Ok(found_or_created) - } } fn ordering_node_indexes_for_node_index( diff --git a/lib/dal/src/workspace_snapshot/graph/detect_conflicts_and_updates.rs b/lib/dal/src/workspace_snapshot/graph/detect_conflicts_and_updates.rs index 7e82378d13..659282bd7e 100644 --- a/lib/dal/src/workspace_snapshot/graph/detect_conflicts_and_updates.rs +++ b/lib/dal/src/workspace_snapshot/graph/detect_conflicts_and_updates.rs @@ -7,15 +7,10 @@ use si_events::{ulid::Ulid, VectorClockId}; use crate::{ workspace_snapshot::{ - conflict::Conflict, - edge_info::EdgeInfo, - graph::WorkspaceSnapshotGraphError, - node_weight::{category_node_weight::CategoryNodeKind, NodeWeight}, - update::Update, - vector_clock::HasVectorClocks, - NodeInformation, + conflict::Conflict, edge_info::EdgeInfo, graph::WorkspaceSnapshotGraphError, + node_weight::NodeWeight, update::Update, vector_clock::HasVectorClocks, NodeInformation, }, - EdgeWeightKind, EdgeWeightKindDiscriminants, WorkspaceSnapshotGraphV1, + EdgeWeightKind, WorkspaceSnapshotGraphV1, }; use super::{ConflictsAndUpdates, WorkspaceSnapshotGraphResult}; @@ -30,7 +25,12 @@ pub struct DetectConflictsAndUpdates<'a, 'b> { enum ConflictsAndUpdatesControl { Continue(Vec, Vec), - Prune(Vec, Vec), +} + +#[derive(PartialEq, Eq, Clone, Debug)] +enum OntoNodeDifference { + MerkleTreeHash, + NewNode, } impl ConflictsAndUpdatesControl { @@ -39,9 +39,6 @@ impl ConflictsAndUpdatesControl { ConflictsAndUpdatesControl::Continue(conflicts, updates) => { (petgraph::visit::Control::Continue, conflicts, updates) } - ConflictsAndUpdatesControl::Prune(conflicts, updates) => { - (petgraph::visit::Control::Prune, conflicts, updates) - } } } } @@ -80,7 +77,7 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { return Err(WorkspaceSnapshotGraphError::GraphTraversal(traversal_error)); } - updates.extend(self.maybe_merge_category_nodes()?); + // updates.extend(self.maybe_merge_category_nodes()?); // Now that we have the full set of updates to be performed, we can check to see if we'd be // breaking any "exclusive edge" constraints. We need to wait until after we've detected @@ -88,7 +85,7 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { // violation of the constraint if we are not also removing the first one. We need to ensure // that the net result is that there is only one of that edge kind. - conflicts.extend(self.detect_exclusive_edge_conflicts_in_updates(&updates)?); + // conflicts.extend(self.detect_exclusive_edge_conflicts_in_updates(&updates)?); Ok(ConflictsAndUpdates { conflicts, updates }) } @@ -101,6 +98,51 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { ) -> Result, petgraph::visit::DfsEvent> { match event { DfsEvent::Discover(onto_node_index, _) => { + let node_diff = self + .onto_node_difference_from_to_rebase(onto_node_index) + .map_err(|err| { + error!( + err=?err, + "Error detecting conflicts and updates for onto {:?}", + onto_node_index, + ); + event + })?; + + Ok(match node_diff { + None => petgraph::visit::Control::Prune, + Some(OntoNodeDifference::NewNode) => { + let node_weight= self.onto_graph.get_node_weight(onto_node_index) + .map_err(|err| { + error!(err=?err, "Error detecting conflicts and updates for onto: {:?}", onto_node_index); + event + })?.to_owned(); + + updates.push(Update::NewNode { node_weight }); + petgraph::visit::Control::Continue + } + Some(OntoNodeDifference::MerkleTreeHash) => petgraph::visit::Control::Continue, + }) + } + DfsEvent::Finish(onto_node_index, _time) => { + // Even though we're pruning in `DfsEvent::Discover`, we'll still get a `Finish` + // for the node where we returned a `petgraph::visit::Control::Prune`. Since we + // already know that there won't be any conflicts/updates with a nodes that have + // identical merkle tree hashes, we can `Continue` + let node_diff = self + .onto_node_difference_from_to_rebase(onto_node_index) + .map_err(|err| { + error!( + err=?err, + "Error detecting conflicts and updates for onto {:?}", + onto_node_index, + ); + event + })?; + if node_diff.is_none() { + return Ok(petgraph::visit::Control::Continue); + } + let (petgraph_control, node_conflicts, node_updates) = self .detect_conflicts_and_updates_for_node_index(onto_node_index) .map_err(|err| { @@ -122,6 +164,61 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { } } + fn onto_node_difference_from_to_rebase( + &self, + onto_node_index: NodeIndex, + ) -> WorkspaceSnapshotGraphResult> { + let onto_node_weight = self.onto_graph.get_node_weight(onto_node_index)?; + let mut to_rebase_node_indexes = HashSet::new(); + if onto_node_index == self.onto_graph.root_index { + // There can only be one (valid/current) `ContentAddress::Root` at any + // given moment, and the `lineage_id` isn't really relevant as it's not + // globally stable (even though it is locally stable). This matters as we + // may be dealing with a `WorkspaceSnapshotGraph` that is coming to us + // externally from a module that we're attempting to import. The external + // `WorkspaceSnapshotGraph` will be `self`, and the "local" one will be + // `onto`. + to_rebase_node_indexes.insert(self.to_rebase_graph.root()); + } else { + // Only retain node indexes... or indices... if they are part of the current + // graph. There may still be garbage from previous updates to the graph + // laying around. + let mut potential_to_rebase_node_indexes = self + .to_rebase_graph + .get_node_index_by_lineage(onto_node_weight.lineage_id()); + potential_to_rebase_node_indexes + .retain(|node_index| self.to_rebase_graph.has_path_to_root(*node_index)); + + to_rebase_node_indexes.extend(potential_to_rebase_node_indexes); + } + + if to_rebase_node_indexes.is_empty() { + return Ok(Some(OntoNodeDifference::NewNode)); + } + + // If everything with the same `lineage_id` is identical, then we can prune the + // graph traversal, and avoid unnecessary lookups/comparisons. + let mut any_content_with_lineage_is_different = false; + + for to_rebase_node_index in to_rebase_node_indexes { + let to_rebase_node_weight = + self.to_rebase_graph.get_node_weight(to_rebase_node_index)?; + if onto_node_weight.merkle_tree_hash() == to_rebase_node_weight.merkle_tree_hash() { + // If the merkle tree hashes are the same, then the entire sub-graph is + // identical, and we don't need to check any further. + continue; + } + + any_content_with_lineage_is_different = true + } + + Ok(if any_content_with_lineage_is_different { + Some(OntoNodeDifference::MerkleTreeHash) + } else { + None + }) + } + fn detect_conflicts_and_updates_for_node_index( &self, onto_node_index: NodeIndex, @@ -153,9 +250,25 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { to_rebase_node_indexes.extend(potential_to_rebase_node_indexes); } - // If everything with the same `lineage_id` is identical, then we can prune the - // graph traversal, and avoid unnecessary lookups/comparisons. - let mut any_content_with_lineage_has_changed = false; + if to_rebase_node_indexes.is_empty() { + // this node exists in onto, but not in to rebase. We should have + // produced a NewNode update already. Now we just need to produce + // new edge updates for all outgoing edges of this node. + for edgeref in self.onto_graph.edges_directed(onto_node_index, Outgoing) { + let edge_info = EdgeInfo { + source_node_index: edgeref.source(), + target_node_index: edgeref.target(), + edge_kind: edgeref.weight().kind().into(), + edge_index: edgeref.id(), + }; + updates.push(Update::new_edge( + self.onto_graph, + &edge_info, + edgeref.weight().to_owned(), + )?); + } + return Ok(ConflictsAndUpdatesControl::Continue(conflicts, updates)); + } for to_rebase_node_index in to_rebase_node_indexes { let to_rebase_node_weight = @@ -170,7 +283,6 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { ); continue; } - any_content_with_lineage_has_changed = true; // Check if there's a difference in the node itself (and whether it is a // conflict if there is a difference). @@ -186,22 +298,11 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { .vector_clock_write() .is_newer_than(to_rebase_node_weight.vector_clock_write()) { - let onto_node_information = NodeInformation { - index: onto_node_index, - id: onto_node_weight.id().into(), - node_weight_kind: onto_node_weight.clone().into(), - }; - let to_rebase_node_information = NodeInformation { - index: to_rebase_node_index, - id: to_rebase_node_weight.id().into(), - node_weight_kind: to_rebase_node_weight.clone().into(), - }; // `onto` has changes, but has already seen all of the changes in // `to_rebase`. There is no conflict, and we should update to use the // `onto` node. - updates.push(Update::ReplaceSubgraph { - onto: onto_node_information, - to_rebase: to_rebase_node_information, + updates.push(Update::ReplaceNode { + node_weight: onto_node_weight.to_owned(), }); } else { // There are changes on both sides that have not @@ -245,12 +346,10 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { }; if common_onto_items != common_to_rebase_items { let to_rebase_node_information = NodeInformation { - index: to_rebase_node_index, id: to_rebase_node_weight.id().into(), node_weight_kind: to_rebase_node_weight.into(), }; let onto_node_information = NodeInformation { - index: onto_node_index, id: onto_node_weight.id().into(), node_weight_kind: onto_node_weight.into(), }; @@ -262,12 +361,10 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { } } else { let to_rebase_node_information = NodeInformation { - index: to_rebase_node_index, id: to_rebase_node_weight.id().into(), node_weight_kind: to_rebase_node_weight.into(), }; let onto_node_information = NodeInformation { - index: onto_node_index, id: onto_node_weight.id().into(), node_weight_kind: onto_node_weight.into(), }; @@ -290,16 +387,13 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { conflicts.extend(container_conflicts); } - if any_content_with_lineage_has_changed { - // There was at least one thing with a merkle tree hash difference, so we need - // to examine further down the tree to see where the difference(s) are, and - // where there are conflicts, if there are any. - Ok(ConflictsAndUpdatesControl::Continue(conflicts, updates)) - } else { - // Everything to be rebased is identical, so there's no need to examine the - // rest of the tree looking for differences & conflicts that won't be there. - Ok(ConflictsAndUpdatesControl::Prune(conflicts, updates)) - } + // This function is run in `DfsEvent::Finish`, so regardless of whether there are any + // updates/conflicts, we need to return `Continue`. We shouldn't ever get here if there + // aren't any differences at all, as we prune the graph during `DfsEvent::Discover`, but + // the differences might not result in any changes/conflicts in the direction we're doing + // the comparison. + + Ok(ConflictsAndUpdatesControl::Continue(conflicts, updates)) } fn find_container_membership_conflicts_and_updates( @@ -443,7 +537,6 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { // Item has been modified in `to_rebase` since // `onto` last saw `to_rebase` let node_information = NodeInformation { - index: only_to_rebase_edge_info.target_node_index, id: to_rebase_item_weight.id().into(), node_weight_kind: to_rebase_item_weight.into(), }; @@ -451,7 +544,6 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { .to_rebase_graph .get_node_weight(to_rebase_container_index)?; let container_node_information = NodeInformation { - index: to_rebase_container_index, id: container_node_weight.id().into(), node_weight_kind: container_node_weight.into(), }; @@ -468,12 +560,10 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { .to_rebase_graph .get_node_weight(only_to_rebase_edge_info.target_node_index)?; let source_node_information = NodeInformation { - index: only_to_rebase_edge_info.source_node_index, id: source_node_weight.id().into(), node_weight_kind: source_node_weight.into(), }; let target_node_information = NodeInformation { - index: only_to_rebase_edge_info.target_node_index, id: target_node_weight.id().into(), node_weight_kind: target_node_weight.into(), }; @@ -544,29 +634,26 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { let onto_node_weight = self .onto_graph .get_node_weight(only_onto_edge_info.target_node_index)?; - let container_node_information = NodeInformation { - index: to_rebase_container_index, + let _container_node_information = NodeInformation { id: container_node_weight.id().into(), node_weight_kind: container_node_weight.into(), }; - let removed_item_node_information = NodeInformation { - index: only_onto_edge_info.target_node_index, + let _removed_item_node_information = NodeInformation { id: onto_node_weight.id().into(), node_weight_kind: onto_node_weight.into(), }; - conflicts.push(Conflict::RemoveModifiedItem { - container: container_node_information, - removed_item: removed_item_node_information, - }); + // NOTE: The clocks don't actually matter anymore. -- Adam & Fletcher + //conflicts.push(Conflict::RemoveModifiedItem { + // container: container_node_information, + // removed_item: removed_item_node_information, + //}); } } None => { // This edge has never been seen by to_rebase updates.push(Update::new_edge( - self.to_rebase_graph, self.onto_graph, - to_rebase_container_index, only_onto_edge_info, onto_edge_weight.to_owned(), )?); @@ -578,124 +665,4 @@ impl<'a, 'b> DetectConflictsAndUpdates<'a, 'b> { // - Sets same: No conflicts/updates Ok((conflicts, updates)) } - - fn maybe_merge_category_nodes(&self) -> WorkspaceSnapshotGraphResult> { - Ok( - match ( - self.to_rebase_graph - .get_category_node(None, CategoryNodeKind::DependentValueRoots)?, - self.onto_graph - .get_category_node(None, CategoryNodeKind::DependentValueRoots)?, - ) { - (Some((to_rebase_category_id, _)), Some((onto_category_id, _))) - if to_rebase_category_id != onto_category_id => - { - vec![Update::MergeCategoryNodes { - to_rebase_category_id, - onto_category_id, - }] - } - _ => vec![], - }, - ) - } - - fn detect_exclusive_edge_conflicts_in_updates( - &self, - updates: &Vec, - ) -> WorkspaceSnapshotGraphResult> { - let mut conflicts = Vec::new(); - - #[derive(Debug, Default, Clone)] - struct NodeEdgeWeightUpdates { - additions: Vec<(NodeInformation, NodeInformation)>, - removals: Vec<(NodeInformation, NodeInformation)>, - } - - let mut edge_updates: HashMap< - NodeIndex, - HashMap, - > = HashMap::new(); - - for update in updates { - match update { - Update::NewEdge { - source, - destination, - edge_weight, - } => { - let source_entry = edge_updates.entry(source.index).or_default(); - let edge_weight_entry = source_entry - .entry(edge_weight.kind().clone().into()) - .or_default(); - edge_weight_entry.additions.push((*source, *destination)); - } - Update::RemoveEdge { - source, - destination, - edge_kind, - } => { - let source_entry = edge_updates.entry(source.index).or_default(); - let edge_weight_entry = source_entry.entry(*edge_kind).or_default(); - edge_weight_entry.removals.push((*source, *destination)); - } - _ => { /* Other updates are unused for exclusive edge conflict detection */ } - } - } - - for (source_node_index, source_node_updates) in &edge_updates { - for (edge_weight_kind, edge_kind_updates) in source_node_updates { - if edge_kind_updates.additions.is_empty() { - // There haven't been any new edges added for this EdgeWeightKind, so we can't - // have created any Conflict::ExclusiveEdge - continue; - } - - if !self - .to_rebase_graph - .get_node_weight(*source_node_index)? - .is_exclusive_outgoing_edge(*edge_weight_kind) - { - // Nothing to check. This edge weight kind isn't considered exclusive. - continue; - } - - // We can only have (removals.len() + (1 - existing edge count)) additions - // _at most_, or we'll be creating a Conflict::ExclusiveEdge because there will be - // multiple of the same edge kind outgoing from the same node. - let existing_outgoing_edges_of_kind = self - .to_rebase_graph - .graph() - .edges_directed(*source_node_index, Outgoing) - .filter(|edge| *edge_weight_kind == edge.weight().kind().into()) - .count(); - - if edge_kind_updates.additions.len() - > (edge_kind_updates.removals.len() + (1 - existing_outgoing_edges_of_kind)) - { - warn!( - "ExclusiveEdgeMismatch: Found {} pre-existing edges. Requested {} removals, {} additions.", - existing_outgoing_edges_of_kind, - edge_kind_updates.removals.len(), - edge_kind_updates.additions.len(), - ); - // The net count of outgoing edges of this kind is >1. Consider *ALL* of the - // additions to be in conflict. - for ( - edge_addition_source_node_information, - edge_addition_destination_node_information, - ) in &edge_kind_updates.additions - { - conflicts.push(Conflict::ExclusiveEdgeMismatch { - source: *edge_addition_source_node_information, - destination: *edge_addition_destination_node_information, - edge_kind: *edge_weight_kind, - }); - } - } - } - } - - Ok(conflicts) - } } diff --git a/lib/dal/src/workspace_snapshot/graph/tests/detect_conflicts_and_updates.rs b/lib/dal/src/workspace_snapshot/graph/tests/detect_conflicts_and_updates.rs index 32d06fa872..d267a0ca8a 100644 --- a/lib/dal/src/workspace_snapshot/graph/tests/detect_conflicts_and_updates.rs +++ b/lib/dal/src/workspace_snapshot/graph/tests/detect_conflicts_and_updates.rs @@ -8,80 +8,20 @@ mod test { use si_events::ContentHash; use si_events::VectorClockId; use std::collections::HashMap; - use std::collections::HashSet; - use crate::workspace_snapshot::content_address::{ContentAddress, ContentAddressDiscriminants}; + use crate::workspace_snapshot::content_address::ContentAddress; use crate::workspace_snapshot::edge_weight::{ EdgeWeight, EdgeWeightKind, EdgeWeightKindDiscriminants, }; use crate::workspace_snapshot::graph::tests::add_prop_nodes_to_graph; - use crate::workspace_snapshot::node_weight::category_node_weight::CategoryNodeKind::DependentValueRoots; use crate::workspace_snapshot::node_weight::NodeWeight; use crate::workspace_snapshot::update::Update; use crate::workspace_snapshot::{ - conflict::Conflict, graph::ConflictsAndUpdates, vector_clock::HasVectorClocks, - NodeInformation, + conflict::Conflict, graph::ConflictsAndUpdates, NodeInformation, }; use crate::NodeWeightDiscriminants; use crate::{PropKind, WorkspaceSnapshotGraphV1}; - #[derive(Debug, Eq, PartialEq)] - enum UpdateWithEdgeWeightKind { - NewEdge { - source: NodeInformation, - destination: NodeInformation, - edge_weight_kind: EdgeWeightKind, - }, - RemoveEdge { - source: NodeInformation, - destination: NodeInformation, - edge_kind: EdgeWeightKindDiscriminants, - }, - ReplaceSubgraph { - onto: NodeInformation, - to_rebase: NodeInformation, - }, - MergeCategoryNodes { - to_rebase_category_id: Ulid, - onto_category_id: Ulid, - }, - } - - impl From for UpdateWithEdgeWeightKind { - fn from(value: Update) -> Self { - match value { - Update::NewEdge { - source, - destination, - edge_weight, - } => Self::NewEdge { - source, - destination, - edge_weight_kind: edge_weight.kind().clone(), - }, - Update::RemoveEdge { - source, - destination, - edge_kind, - } => Self::RemoveEdge { - source, - destination, - edge_kind, - }, - Update::ReplaceSubgraph { onto, to_rebase } => { - Self::ReplaceSubgraph { onto, to_rebase } - } - Update::MergeCategoryNodes { - to_rebase_category_id, - onto_category_id, - } => Self::MergeCategoryNodes { - to_rebase_category_id, - onto_category_id, - }, - } - } - } - fn get_root_node_info(graph: &WorkspaceSnapshotGraphV1) -> NodeInformation { let root_id = graph .get_node_weight(graph.root_index) @@ -89,7 +29,6 @@ mod test { .id(); NodeInformation { - index: graph.root_index, node_weight_kind: NodeWeightDiscriminants::Content, id: root_id.into(), } @@ -315,17 +254,12 @@ mod test { assert!(conflicts_and_updates.conflicts.is_empty()); - let new_onto_component_index = base_graph + let _new_onto_component_index = base_graph .get_node_index_by_id(new_onto_component_id) .expect("Unable to get NodeIndex"); match conflicts_and_updates.updates.as_slice() { - [Update::NewEdge { - source, - destination, - edge_weight, - }] => { - assert_eq!(new_graph.root_index, source.index); - assert_eq!(new_onto_component_index, destination.index); + [Update::NewNode { .. }, Update::NewEdge { edge_weight, .. }, Update::NewEdge { .. }] => + { assert_eq!(&EdgeWeightKind::new_use(), edge_weight.kind()); } other => panic!("Unexpected updates: {:?}", other), @@ -411,13 +345,7 @@ mod test { assert!(conflicts_and_updates.conflicts.is_empty()); match conflicts_and_updates.updates.as_slice() { - [Update::NewEdge { - source, - destination, - edge_weight, - }] => { - assert_eq!(base_graph.root_index, source.index); - assert_eq!(new_component_index, destination.index); + [Update::NewNode { .. }, Update::NewEdge { edge_weight, .. }] => { assert_eq!(&EdgeWeightKind::new_use(), edge_weight.kind()); } other => panic!("Unexpected updates: {:?}", other), @@ -565,17 +493,12 @@ mod test { assert!(conflicts_and_updates.conflicts.is_empty()); - let new_onto_component_index = base_graph + let _new_onto_component_index = base_graph .get_node_index_by_id(new_onto_component_id) .expect("Unable to get NodeIndex"); match conflicts_and_updates.updates.as_slice() { - [Update::NewEdge { - source, - destination, - edge_weight, - }] => { - assert_eq!(new_graph.root_index, source.index); - assert_eq!(new_onto_component_index, destination.index); + [Update::NewNode { .. }, Update::NewEdge { edge_weight, .. }, Update::NewEdge { .. }] => + { assert_eq!(&EdgeWeightKind::new_use(), edge_weight.kind()); } other => panic!("Unexpected updates: {:?}", other), @@ -711,16 +634,10 @@ mod test { vec![Conflict::NodeContent { onto: NodeInformation { id: component_id.into(), - index: base_graph - .get_node_index_by_id(component_id) - .expect("Unable to get component NodeIndex"), node_weight_kind: NodeWeightDiscriminants::Content, }, to_rebase: NodeInformation { id: component_id.into(), - index: new_graph - .get_node_index_by_id(component_id) - .expect("Unable to get component NodeIndex"), node_weight_kind: NodeWeightDiscriminants::Content, }, }], @@ -729,157 +646,6 @@ mod test { assert!(conflicts_and_updates.updates.is_empty()); } - #[test] - fn detect_conflicts_and_updates_simple_with_content_conflict_but_no_conflict_same_vector_clocks( - ) { - let actor_id = Ulid::new(); - let vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut base_graph = WorkspaceSnapshotGraphV1::new(vector_clock_id) - .expect("Unable to create WorkspaceSnapshotGraph"); - - let schema_id = base_graph.generate_ulid().expect("Unable to generate Ulid"); - let schema_index = base_graph - .add_node( - NodeWeight::new_content( - vector_clock_id, - schema_id, - Ulid::new(), - ContentAddress::Schema(ContentHash::from("Schema A")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema A"); - let schema_variant_id = base_graph.generate_ulid().expect("Unable to generate Ulid"); - let schema_variant_index = base_graph - .add_node( - NodeWeight::new_content( - vector_clock_id, - schema_variant_id, - Ulid::new(), - ContentAddress::SchemaVariant(ContentHash::from("Schema Variant A")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema Variant A"); - - base_graph - .add_edge( - base_graph.root_index, - EdgeWeight::new(vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - schema_index, - ) - .expect("Unable to add root -> schema edge"); - base_graph - .add_edge( - base_graph - .get_node_index_by_id(schema_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - schema_variant_index, - ) - .expect("Unable to add schema -> schema variant edge"); - - let component_id = base_graph.generate_ulid().expect("Unable to generate Ulid"); - let component_index = base_graph - .add_node( - NodeWeight::new_content( - vector_clock_id, - component_id, - Ulid::new(), - ContentAddress::Component(ContentHash::from("Component A")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Component A"); - base_graph - .add_edge( - base_graph.root_index, - EdgeWeight::new(vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - component_index, - ) - .expect("Unable to add root -> component edge"); - base_graph - .add_edge( - base_graph - .get_node_index_by_id(component_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - base_graph - .get_node_index_by_id(schema_variant_id) - .expect("Unable to get NodeIndex"), - ) - .expect("Unable to add component -> schema variant edge"); - - base_graph.cleanup(); - base_graph.dot(); - base_graph - .mark_graph_seen(vector_clock_id) - .expect("mark graph seen"); - - let mut new_graph = base_graph.clone(); - - new_graph - .update_content( - vector_clock_id, - component_id, - ContentHash::from("Updated Component A"), - ) - .expect("Unable to update Component A"); - - new_graph.cleanup(); - new_graph.dot(); - new_graph - .mark_graph_seen(vector_clock_id) - .expect("mark graph seen"); - - base_graph - .update_content( - vector_clock_id, - component_id, - ContentHash::from("Base Updated Component A"), - ) - .expect("Unable to update Component A"); - - base_graph.cleanup(); - base_graph.dot(); - base_graph - .mark_graph_seen(vector_clock_id) - .expect("mark graph seen"); - - let conflicts_and_updates = new_graph - .detect_conflicts_and_updates(vector_clock_id, &base_graph, vector_clock_id) - .expect("Unable to detect conflicts and updates"); - - // With identical vector clocks, we should not hit conflicts - assert!(conflicts_and_updates.conflicts.is_empty()); - - // Instead, we receive the "inverse" of the NodeContent conflict: A - // ReplaceSubgraph update for the node. - assert_eq!( - vec![Update::ReplaceSubgraph { - onto: NodeInformation { - id: component_id.into(), - index: base_graph - .get_node_index_by_id(component_id) - .expect("Unable to get component NodeIndex"), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - to_rebase: NodeInformation { - id: component_id.into(), - index: new_graph - .get_node_index_by_id(component_id) - .expect("Unable to get component NodeIndex"), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - }], - conflicts_and_updates.updates - ); - } - #[test] fn detect_conflicts_and_updates_simple_with_modify_removed_item_conflict() { let actor_id = Ulid::new(); @@ -1015,9 +781,6 @@ mod test { container, modified_item: NodeInformation { id: component_id.into(), - index: new_graph - .get_node_index_by_id(component_id) - .expect("Unable to get NodeIndex"), node_weight_kind: NodeWeightDiscriminants::Content, } }], @@ -1163,9 +926,6 @@ mod test { container, modified_item: NodeInformation { id: component_id.into(), - index: new_graph - .get_node_index_by_id(component_id) - .expect("Unable to get NodeIndex"), node_weight_kind: NodeWeightDiscriminants::Content, } }], @@ -1331,18 +1091,8 @@ mod test { destination, edge_weight, }] => { - assert_eq!( - base_graph - .get_node_index_by_id(base_prop_id) - .expect("Unable to get prop NodeIndex"), - source.index, - ); - assert_eq!( - base_graph - .get_node_index_by_id(attribute_prototype_id) - .expect("Unable to get prop NodeIndex"), - destination.index, - ); + assert_eq!(base_prop_id, source.id.into(),); + assert_eq!(attribute_prototype_id, destination.id.into(),); assert_eq!(&EdgeWeightKind::Prototype(None), edge_weight.kind()); } other => panic!("Unexpected updates: {:?}", other), @@ -1350,1620 +1100,100 @@ mod test { } #[test] - fn detect_conflicts_and_updates_complex() { + fn detect_conflicts_and_updates_simple_ordering_no_conflicts_no_updates_in_base() { let actor_id = Ulid::new(); let initial_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut base_graph = WorkspaceSnapshotGraphV1::new(initial_vector_clock_id) + let mut initial_graph = WorkspaceSnapshotGraphV1::new(initial_vector_clock_id) .expect("Unable to create WorkspaceSnapshotGraph"); - // Docker Image Schema - let docker_image_schema_id = base_graph.generate_ulid().expect("Unable to generate Ulid"); - let docker_image_schema_index = base_graph + let schema_id = initial_graph + .generate_ulid() + .expect("Unable to generate Ulid"); + let schema_index = initial_graph .add_node( NodeWeight::new_content( initial_vector_clock_id, - docker_image_schema_id, + schema_id, Ulid::new(), - ContentAddress::Schema(ContentHash::from("first")), + ContentAddress::Schema(ContentHash::from("Schema A")), ) .expect("Unable to create NodeWeight"), ) .expect("Unable to add Schema A"); - base_graph - .add_edge( - base_graph.root_index, - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - docker_image_schema_index, - ) - .expect("Unable to add root -> schema edge"); - - println!("Add edge from root to {} in onto", docker_image_schema_id); - - // Docker Image Schema Variant - let docker_image_schema_variant_id = - base_graph.generate_ulid().expect("Unable to generate Ulid"); - let docker_image_schema_variant_index = base_graph + let schema_variant_id = initial_graph + .generate_ulid() + .expect("Unable to generate Ulid"); + let schema_variant_index = initial_graph .add_node( NodeWeight::new_content( initial_vector_clock_id, - docker_image_schema_variant_id, + schema_variant_id, Ulid::new(), - ContentAddress::SchemaVariant(ContentHash::from("first")), + ContentAddress::SchemaVariant(ContentHash::from("Schema Variant A")), ) .expect("Unable to create NodeWeight"), ) .expect("Unable to add Schema Variant A"); - base_graph + + initial_graph .add_edge( - base_graph - .get_node_index_by_id(docker_image_schema_id) + initial_graph.root_index, + EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) + .expect("Unable to create EdgeWeight"), + schema_index, + ) + .expect("Unable to add root -> schema edge"); + initial_graph + .add_edge( + initial_graph + .get_node_index_by_id(schema_id) .expect("Unable to get NodeIndex"), EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) .expect("Unable to create EdgeWeight"), - docker_image_schema_variant_index, + schema_variant_index, ) .expect("Unable to add schema -> schema variant edge"); - println!( - "Add edge from {} to {} in onto", - docker_image_schema_id, docker_image_schema_variant_id - ); - - // Nginx Docker Image Component - let nginx_docker_image_component_id = - base_graph.generate_ulid().expect("Unable to generate Ulid"); - let nginx_docker_image_component_index = base_graph - .add_node( + let container_prop_id = initial_graph + .generate_ulid() + .expect("Unable to generate Ulid"); + let container_prop_index = initial_graph + .add_ordered_node( + initial_vector_clock_id, NodeWeight::new_content( initial_vector_clock_id, - nginx_docker_image_component_id, + container_prop_id, Ulid::new(), - ContentAddress::Component(ContentHash::from("first")), + ContentAddress::Prop(ContentHash::new( + container_prop_id.to_string().as_bytes(), + )), ) .expect("Unable to create NodeWeight"), ) - .expect("Unable to add Component A"); - base_graph - .add_edge( - base_graph.root_index, - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - nginx_docker_image_component_index, - ) - .expect("Unable to add root -> component edge"); - - println!( - "Add edge from root to {} in onto", - nginx_docker_image_component_id - ); - - base_graph + .expect("Unable to add container prop"); + initial_graph .add_edge( - base_graph - .get_node_index_by_id(nginx_docker_image_component_id) + initial_graph + .get_node_index_by_id(schema_variant_id) .expect("Unable to get NodeIndex"), EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) .expect("Unable to create EdgeWeight"), - base_graph - .get_node_index_by_id(docker_image_schema_variant_id) - .expect("Unable to get NodeIndex"), + container_prop_index, ) - .expect("Unable to add component -> schema variant edge"); - - println!( - "Add edge from {} to {} in onto", - nginx_docker_image_component_id, docker_image_schema_variant_id - ); + .expect("Unable to add schema variant -> container prop edge"); - // Alpine Component - let alpine_component_id = base_graph.generate_ulid().expect("Unable to generate Ulid"); - let alpine_component_index = base_graph + let ordered_prop_1_id = initial_graph + .generate_ulid() + .expect("Unable to generate Ulid"); + let ordered_prop_1_index = initial_graph .add_node( NodeWeight::new_content( initial_vector_clock_id, - alpine_component_id, + ordered_prop_1_id, Ulid::new(), - ContentAddress::Component(ContentHash::from("first")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Component A"); - base_graph - .add_edge( - base_graph.root_index, - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - alpine_component_index, - ) - .expect("Unable to add root -> component edge"); - - println!("Add edge from root to {} in onto", alpine_component_id); - - base_graph - .add_edge( - base_graph - .get_node_index_by_id(alpine_component_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - base_graph - .get_node_index_by_id(docker_image_schema_variant_id) - .expect("Unable to get NodeIndex"), - ) - .expect("Unable to add component -> schema variant edge"); - - println!( - "Add edge from {} to {} in onto", - alpine_component_id, docker_image_schema_variant_id - ); - - // Butane Schema - let butane_schema_id = base_graph.generate_ulid().expect("Unable to generate Ulid"); - let butane_schema_index = base_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - butane_schema_id, - Ulid::new(), - ContentAddress::Schema(ContentHash::from("first")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema A"); - base_graph - .add_edge( - base_graph.root_index, - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - butane_schema_index, - ) - .expect("Unable to add root -> schema edge"); - - println!("Add edge from root to {} in onto", butane_schema_id); - - // Butane Schema Variant - let butane_schema_variant_id = base_graph.generate_ulid().expect("Unable to generate Ulid"); - let butane_schema_variant_index = base_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - butane_schema_variant_id, - Ulid::new(), - ContentAddress::SchemaVariant(ContentHash::from("first")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema Variant A"); - base_graph - .add_edge( - base_graph - .get_node_index_by_id(butane_schema_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - butane_schema_variant_index, - ) - .expect("Unable to add schema -> schema variant edge"); - - println!( - "Add edge from {} to {} in onto", - butane_schema_id, butane_schema_variant_id - ); - - // Nginx Butane Component - let nginx_butane_component_id = - base_graph.generate_ulid().expect("Unable to generate Ulid"); - let nginx_butane_node_index = base_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - nginx_butane_component_id, - Ulid::new(), - ContentAddress::Component(ContentHash::from("first")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema Variant A"); - base_graph - .add_edge( - base_graph.root_index, - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - nginx_butane_node_index, - ) - .expect("Unable to add root -> component edge"); - - println!( - "Add edge from root to {} in onto", - nginx_butane_component_id - ); - - base_graph - .add_edge( - base_graph - .get_node_index_by_id(nginx_butane_component_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - base_graph - .get_node_index_by_id(butane_schema_variant_id) - .expect("Unable to get NodeIndex"), - ) - .expect("Unable to add component -> schema variant edge"); - - println!( - "Add edge from {} to {} in onto", - nginx_butane_component_id, butane_schema_variant_id - ); - - base_graph.cleanup(); - //base_graph.dot(); - base_graph - .mark_graph_seen(initial_vector_clock_id) - .expect("unable to mark graph seen"); - - // Create a new change set to cause some problems! - let new_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut new_graph = base_graph.clone(); - - println!("fork onto into to_rebase"); - - // Create a modify removed item conflict. - base_graph - .remove_edge( - initial_vector_clock_id, - base_graph.root_index, - base_graph - .get_node_index_by_id(nginx_butane_component_id) - .expect("Unable to get NodeIndex"), - EdgeWeightKindDiscriminants::Use, - ) - .expect("Unable to update the component"); - - println!( - "Remove edge from root to {} in onto", - nginx_butane_component_id - ); - - new_graph - .update_content( - new_vector_clock_id, - nginx_butane_component_id, - ContentHash::from("second"), - ) - .expect("Unable to update the component"); - - println!( - "Update content of {} in to_rebase (should produce ModifyRemovedItem conflict)", - nginx_butane_component_id - ); - - // Create a node content conflict. - base_graph - .update_content( - initial_vector_clock_id, - docker_image_schema_variant_id, - ContentHash::from("oopsie"), - ) - .expect("Unable to update the component"); - - println!( - "Update content of {} in onto", - docker_image_schema_variant_id - ); - - new_graph - .update_content( - new_vector_clock_id, - docker_image_schema_variant_id, - ContentHash::from("poopsie"), - ) - .expect("Unable to update the component"); - - println!( - "Update content of {} in to_rebase (should produce update content conflict)", - docker_image_schema_variant_id - ); - - // Create a pure update. - base_graph - .update_content( - initial_vector_clock_id, - docker_image_schema_id, - ContentHash::from("bg3"), - ) - .expect("Unable to update the schema"); - - println!("Update content of {} in onto", docker_image_schema_id); - - new_graph.cleanup(); - new_graph - .mark_graph_seen(new_vector_clock_id) - .expect("unable to mark graph seen"); - base_graph.cleanup(); - base_graph - .mark_graph_seen(initial_vector_clock_id) - .expect("unable to mark graph seen"); - - // new_graph.tiny_dot_to_file(Some("to_rebase")); - // base_graph.tiny_dot_to_file(Some("onto")); - - let ConflictsAndUpdates { conflicts, updates } = new_graph - .detect_conflicts_and_updates(new_vector_clock_id, &base_graph, initial_vector_clock_id) - .expect("Unable to detect conflicts and updates"); - - // base_graph.dot(); - // new_graph.dot(); - let container = get_root_node_info(&new_graph); - - let expected_conflicts = vec![ - Conflict::ModifyRemovedItem { - container, - modified_item: NodeInformation { - index: new_graph - .get_node_index_by_id(nginx_butane_component_id) - .expect("Unable to get component NodeIndex"), - id: nginx_butane_component_id.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - }, - Conflict::NodeContent { - onto: NodeInformation { - index: base_graph - .get_node_index_by_id(docker_image_schema_variant_id) - .expect("Unable to get component NodeIndex"), - id: docker_image_schema_variant_id.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - to_rebase: NodeInformation { - index: new_graph - .get_node_index_by_id(docker_image_schema_variant_id) - .expect("Unable to get component NodeIndex"), - id: docker_image_schema_variant_id.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - }, - ]; - let expected_updates = vec![Update::ReplaceSubgraph { - onto: NodeInformation { - index: base_graph - .get_node_index_by_id(docker_image_schema_id) - .expect("Unable to get NodeIndex"), - id: docker_image_schema_id.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - to_rebase: NodeInformation { - index: new_graph - .get_node_index_by_id(docker_image_schema_id) - .expect("Unable to get NodeIndex"), - id: docker_image_schema_id.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - }]; - - assert_eq!( - ConflictsAndUpdates { - conflicts: expected_conflicts, - updates: expected_updates, - }, - ConflictsAndUpdates { conflicts, updates }, - ); - } - - #[test] - fn detect_conflicts_and_updates_simple_ordering_no_conflicts_no_updates_in_base() { - let actor_id = Ulid::new(); - let initial_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut initial_graph = WorkspaceSnapshotGraphV1::new(initial_vector_clock_id) - .expect("Unable to create WorkspaceSnapshotGraph"); - - let schema_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let schema_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - schema_id, - Ulid::new(), - ContentAddress::Schema(ContentHash::from("Schema A")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema A"); - let schema_variant_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let schema_variant_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - schema_variant_id, - Ulid::new(), - ContentAddress::SchemaVariant(ContentHash::from("Schema Variant A")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema Variant A"); - - initial_graph - .add_edge( - initial_graph.root_index, - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - schema_index, - ) - .expect("Unable to add root -> schema edge"); - initial_graph - .add_edge( - initial_graph - .get_node_index_by_id(schema_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - schema_variant_index, - ) - .expect("Unable to add schema -> schema variant edge"); - - let container_prop_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let container_prop_index = initial_graph - .add_ordered_node( - initial_vector_clock_id, - NodeWeight::new_content( - initial_vector_clock_id, - container_prop_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - container_prop_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add container prop"); - initial_graph - .add_edge( - initial_graph - .get_node_index_by_id(schema_variant_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - container_prop_index, - ) - .expect("Unable to add schema variant -> container prop edge"); - - let ordered_prop_1_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_1_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_1_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_1_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 1"); - initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_1_index, - ) - .expect("Unable to add container prop -> ordered prop 1 edge"); - - let ordered_prop_2_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_2_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_2_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_2_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 2"); - initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_2_index, - ) - .expect("Unable to add container prop -> ordered prop 2 edge"); - - let ordered_prop_3_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_3_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_3_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_3_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 3"); - initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_3_index, - ) - .expect("Unable to add container prop -> ordered prop 3 edge"); - - let ordered_prop_4_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_4_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_4_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_4_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 4"); - initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_4_index, - ) - .expect("Unable to add container prop -> ordered prop 4 edge"); - - initial_graph.cleanup(); - initial_graph.dot(); - - initial_graph - .mark_graph_seen(initial_vector_clock_id) - .expect("unable to mark graph seen"); - - let new_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut new_graph = initial_graph.clone(); - - let ordered_prop_5_id = new_graph.generate_ulid().expect("Unable to generate Ulid"); - let ordered_prop_5_index = new_graph - .add_node( - NodeWeight::new_content( - new_vector_clock_id, - ordered_prop_5_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_5_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 5"); - new_graph - .add_ordered_edge( - new_vector_clock_id, - new_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(new_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_5_index, - ) - .expect("Unable to add container prop -> ordered prop 5 edge"); - - new_graph.cleanup(); - new_graph.dot(); - new_graph - .mark_graph_seen(new_vector_clock_id) - .expect("unable to mark graph seen"); - - let ConflictsAndUpdates { conflicts, updates } = new_graph - .detect_conflicts_and_updates( - new_vector_clock_id, - &initial_graph, - initial_vector_clock_id, - ) - .expect("Unable to detect conflicts and updates"); - - assert_eq!(Vec::::new(), conflicts); - assert_eq!(Vec::::new(), updates); - } - - #[test] - fn detect_conflicts_and_updates_simple_ordering_no_conflicts_with_updates_in_base() { - let actor_id = Ulid::new(); - let initial_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut initial_graph = WorkspaceSnapshotGraphV1::new(initial_vector_clock_id) - .expect("Unable to create WorkspaceSnapshotGraph"); - - let schema_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let schema_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - schema_id, - Ulid::new(), - ContentAddress::Schema(ContentHash::from("Schema A")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema A"); - let schema_variant_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let schema_variant_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - schema_variant_id, - Ulid::new(), - ContentAddress::SchemaVariant(ContentHash::from("Schema Variant A")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema Variant A"); - - initial_graph - .add_edge( - initial_graph.root_index, - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - schema_index, - ) - .expect("Unable to add root -> schema edge"); - initial_graph - .add_edge( - initial_graph - .get_node_index_by_id(schema_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - schema_variant_index, - ) - .expect("Unable to add schema -> schema variant edge"); - - let container_prop_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let container_prop_index = initial_graph - .add_ordered_node( - initial_vector_clock_id, - NodeWeight::new_content( - initial_vector_clock_id, - container_prop_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - container_prop_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add container prop"); - initial_graph - .add_edge( - initial_graph - .get_node_index_by_id(schema_variant_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - container_prop_index, - ) - .expect("Unable to add schema variant -> container prop edge"); - - let ordered_prop_1_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_1_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_1_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_1_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 1"); - initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_1_index, - ) - .expect("Unable to add container prop -> ordered prop 1 edge"); - - let ordered_prop_2_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_2_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_2_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_2_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 2"); - initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_2_index, - ) - .expect("Unable to add container prop -> ordered prop 2 edge"); - - let ordered_prop_3_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_3_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_3_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_3_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 3"); - initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_3_index, - ) - .expect("Unable to add container prop -> ordered prop 3 edge"); - - let ordered_prop_4_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_4_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_4_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_4_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 4"); - initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_4_index, - ) - .expect("Unable to add container prop -> ordered prop 4 edge"); - - initial_graph.dot(); - initial_graph - .mark_graph_seen(initial_vector_clock_id) - .expect("unable to mark graph seen"); - - let new_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut new_graph = initial_graph.clone(); - - let ordered_prop_5_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_5_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_5_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_5_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 5"); - let new_edge_weight = EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"); - let (_, maybe_ordinal_edge_information) = initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - new_edge_weight.clone(), - ordered_prop_5_index, - ) - .expect("Unable to add container prop -> ordered prop 5 edge"); - let ( - ordinal_edge_index, - source_node_index_for_ordinal_edge, - destination_node_index_for_ordinal_edge, - ) = maybe_ordinal_edge_information.expect("ordinal edge information not found"); - let ordinal_edge_weight = initial_graph - .get_edge_weight_opt(ordinal_edge_index) - .expect("should not error when getting edge") - .expect("could not get edge weight for index") - .to_owned(); - let source_node_id_for_ordinal_edge = initial_graph - .get_node_weight(source_node_index_for_ordinal_edge) - .expect("could not get node weight") - .id(); - let destination_node_id_for_ordinal_edge = initial_graph - .get_node_weight(destination_node_index_for_ordinal_edge) - .expect("could not get node weight") - .id(); - initial_graph - .mark_graph_seen(initial_vector_clock_id) - .expect("unable to mark graph seen"); - new_graph - .mark_graph_seen(new_vector_clock_id) - .expect("unable to mark graph seen"); - - new_graph.dot(); - - let ConflictsAndUpdates { conflicts, updates } = new_graph - .detect_conflicts_and_updates( - new_vector_clock_id, - &initial_graph, - initial_vector_clock_id, - ) - .expect("Unable to detect conflicts and updates"); - - let initial_ordering_node_index_for_container = initial_graph - .ordering_node_index_for_container( - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get container NodeIndex"), - ) - .expect("Unable to get new ordering NodeIndex") - .expect("Ordering NodeIndex not found"); - let initial_ordering_node_weight_for_container = initial_graph - .get_node_weight(initial_ordering_node_index_for_container) - .expect("Unable to get ordering node weight"); - let new_ordering_node_index_for_container = new_graph - .ordering_node_index_for_container( - new_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get container NodeIndex"), - ) - .expect("Unable to get new ordering NodeIndex") - .expect("Ordering NodeIndex not found"); - let new_ordering_node_weight_for_container = new_graph - .get_node_weight(new_ordering_node_index_for_container) - .expect("Unable to get ordering node weight"); - assert_eq!(Vec::::new(), conflicts); - assert_eq!( - vec![ - UpdateWithEdgeWeightKind::NewEdge { - source: NodeInformation { - index: new_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - id: container_prop_id.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - destination: NodeInformation { - index: initial_graph - .get_node_index_by_id(ordered_prop_5_id) - .expect("Unable to get NodeIndex"), - id: ordered_prop_5_id.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - edge_weight_kind: new_edge_weight.kind().clone(), - }, - UpdateWithEdgeWeightKind::ReplaceSubgraph { - onto: NodeInformation { - index: initial_ordering_node_index_for_container, - id: initial_ordering_node_weight_for_container.id().into(), - node_weight_kind: NodeWeightDiscriminants::Ordering, - }, - to_rebase: NodeInformation { - index: new_ordering_node_index_for_container, - id: new_ordering_node_weight_for_container.id().into(), - node_weight_kind: NodeWeightDiscriminants::Ordering, - }, - }, - UpdateWithEdgeWeightKind::NewEdge { - source: NodeInformation { - index: new_graph - .get_node_index_by_id(source_node_id_for_ordinal_edge) - .expect("could not get node index by id"), - id: source_node_id_for_ordinal_edge.into(), - node_weight_kind: NodeWeightDiscriminants::Ordering, - }, - destination: NodeInformation { - index: initial_graph - .get_node_index_by_id(destination_node_id_for_ordinal_edge) - .expect("could not get node index by id"), - id: destination_node_id_for_ordinal_edge.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - edge_weight_kind: ordinal_edge_weight.kind().clone(), - } - ], - updates - .into_iter() - .map(Into::::into) - .collect::>(), - ); - } - - #[test] - fn detect_conflicts_and_updates_simple_ordering_with_conflicting_ordering_updates() { - let actor_id = Ulid::new(); - let initial_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut initial_graph = WorkspaceSnapshotGraphV1::new(initial_vector_clock_id) - .expect("Unable to create WorkspaceSnapshotGraph"); - - let schema_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let schema_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - schema_id, - Ulid::new(), - ContentAddress::Schema(ContentHash::from("Schema A")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema A"); - let schema_variant_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let schema_variant_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - schema_variant_id, - Ulid::new(), - ContentAddress::SchemaVariant(ContentHash::from("Schema Variant A")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema Variant A"); - - initial_graph - .add_edge( - initial_graph.root_index, - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - schema_index, - ) - .expect("Unable to add root -> schema edge"); - initial_graph - .add_edge( - initial_graph - .get_node_index_by_id(schema_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - schema_variant_index, - ) - .expect("Unable to add schema -> schema variant edge"); - - let container_prop_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let container_prop_index = initial_graph - .add_ordered_node( - initial_vector_clock_id, - NodeWeight::new_content( - initial_vector_clock_id, - container_prop_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - container_prop_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add container prop"); - initial_graph - .add_edge( - initial_graph - .get_node_index_by_id(schema_variant_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - container_prop_index, - ) - .expect("Unable to add schema variant -> container prop edge"); - - let ordered_prop_1_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_1_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_1_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_1_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 1"); - initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_1_index, - ) - .expect("Unable to add container prop -> ordered prop 1 edge"); - - let ordered_prop_2_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_2_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_2_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_2_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 2"); - initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_2_index, - ) - .expect("Unable to add container prop -> ordered prop 2 edge"); - - let ordered_prop_3_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_3_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_3_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_3_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 3"); - initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_3_index, - ) - .expect("Unable to add container prop -> ordered prop 3 edge"); - - let ordered_prop_4_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_4_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_4_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_4_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 4"); - initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - ordered_prop_4_index, - ) - .expect("Unable to add container prop -> ordered prop 4 edge"); - - initial_graph.dot(); - initial_graph - .mark_graph_seen(initial_vector_clock_id) - .expect("unable to mark graph seen"); - - let new_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut new_graph = initial_graph.clone(); - - let new_order = vec![ - ordered_prop_2_id, - ordered_prop_1_id, - ordered_prop_4_id, - ordered_prop_3_id, - ]; - new_graph - .update_order(new_vector_clock_id, container_prop_id, new_order) - .expect("Unable to update order of container prop's children"); - - let ordered_prop_5_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_5_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_5_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_5_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 5"); - let new_edge_weight = EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"); - let (_, maybe_ordinal_edge_information) = initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - new_edge_weight.clone(), - ordered_prop_5_index, - ) - .expect("Unable to add container prop -> ordered prop 5 edge"); - let ( - ordinal_edge_index, - source_node_index_for_ordinal_edge, - destination_node_index_for_ordinal_edge, - ) = maybe_ordinal_edge_information.expect("ordinal edge information not found"); - let ordinal_edge_weight = initial_graph - .get_edge_weight_opt(ordinal_edge_index) - .expect("should not error when getting edge") - .expect("could not get edge weight for index") - .to_owned(); - let source_node_id_for_ordinal_edge = initial_graph - .get_node_weight(source_node_index_for_ordinal_edge) - .expect("could not get node weight") - .id(); - let destination_node_id_for_ordinal_edge = initial_graph - .get_node_weight(destination_node_index_for_ordinal_edge) - .expect("could not get node weight") - .id(); - - initial_graph - .mark_graph_seen(initial_vector_clock_id) - .expect("unable to mark graph seen"); - - new_graph.cleanup(); - new_graph - .mark_graph_seen(new_vector_clock_id) - .expect("unable to mark graph seen"); - new_graph.dot(); - - let ConflictsAndUpdates { conflicts, updates } = new_graph - .detect_conflicts_and_updates( - new_vector_clock_id, - &initial_graph, - initial_vector_clock_id, - ) - .expect("Unable to detect conflicts and updates"); - - let initial_container_ordering_node_index = initial_graph - .ordering_node_index_for_container( - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get container node index"), - ) - .expect("Unable to get ordering node index") - .expect("No ordering node"); - let new_container_ordering_node_index = new_graph - .ordering_node_index_for_container( - new_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get container node index"), - ) - .expect("Unable to get ordering node index") - .expect("No ordering node"); - - assert_eq!( - vec![Conflict::ChildOrder { - onto: NodeInformation { - index: initial_container_ordering_node_index, - id: initial_graph - .get_node_weight(initial_container_ordering_node_index) - .expect("Unable to get ordering node") - .id() - .into(), - node_weight_kind: NodeWeightDiscriminants::Ordering, - }, - to_rebase: NodeInformation { - index: new_container_ordering_node_index, - id: new_graph - .get_node_weight(new_container_ordering_node_index) - .expect("Unable to get new ordering node") - .id() - .into(), - node_weight_kind: NodeWeightDiscriminants::Ordering, - }, - }], - conflicts - ); - - assert_eq!( - vec![ - UpdateWithEdgeWeightKind::NewEdge { - source: NodeInformation { - index: new_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get new prop index"), - id: container_prop_id.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - destination: NodeInformation { - index: initial_graph - .get_node_index_by_id(ordered_prop_5_id) - .expect("Unable to get ordered prop 5 index"), - id: ordered_prop_5_id.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - edge_weight_kind: new_edge_weight.kind().clone(), - }, - UpdateWithEdgeWeightKind::NewEdge { - source: NodeInformation { - index: new_graph - .get_node_index_by_id(source_node_id_for_ordinal_edge) - .expect("could not get node index by id"), - id: source_node_id_for_ordinal_edge.into(), - node_weight_kind: NodeWeightDiscriminants::Ordering, - }, - destination: NodeInformation { - index: initial_graph - .get_node_index_by_id(destination_node_id_for_ordinal_edge) - .expect("could not get node index by id"), - id: destination_node_id_for_ordinal_edge.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - edge_weight_kind: ordinal_edge_weight.kind().to_owned(), - } - ], - updates - .into_iter() - .map(Into::::into) - .collect::>(), - ); - } - - #[test] - fn simple_ordering_no_conflicts_same_vector_clocks() { - let vector_clock_id = VectorClockId::new(Ulid::new(), Ulid::new()); - let mut to_rebase_graph = WorkspaceSnapshotGraphV1::new(vector_clock_id) - .expect("Unable to create WorkspaceSnapshotGraph"); - - let ordered_node = "ordered_container"; - let initial_children = vec!["a", "b", "c"]; - let onto_children = vec!["d", "e", "f"]; - - let mut node_id_map = - add_prop_nodes_to_graph(&mut to_rebase_graph, vector_clock_id, &[ordered_node], true); - node_id_map.extend(add_prop_nodes_to_graph( - &mut to_rebase_graph, - vector_clock_id, - &initial_children, - false, - )); - let ordered_id = *node_id_map.get(ordered_node).expect("should be there"); - let ordered_idx = to_rebase_graph - .get_node_index_by_id(ordered_id) - .expect("should have a node index"); - let root_idx = to_rebase_graph.root(); - to_rebase_graph - .add_edge( - root_idx, - EdgeWeight::new(vector_clock_id, EdgeWeightKind::new_use()) - .expect("failed to make edge weight"), - ordered_idx, - ) - .expect("should be able to make an edge"); - - for child in &initial_children { - let ordered_idx = to_rebase_graph - .get_node_index_by_id(ordered_id) - .expect("should have a node index"); - let child_id = node_id_map.get(*child).copied().expect("node should exist"); - let child_idx = to_rebase_graph - .get_node_index_by_id(child_id) - .expect("should have a node index"); - to_rebase_graph - .add_ordered_edge( - vector_clock_id, - ordered_idx, - EdgeWeight::new(vector_clock_id, EdgeWeightKind::new_use()) - .expect("failed to make edge weight"), - child_idx, - ) - .expect("should be able to make an edge"); - } - - to_rebase_graph.cleanup(); - to_rebase_graph - .mark_graph_seen(vector_clock_id) - .expect("mark twain"); - - let mut onto_graph = to_rebase_graph.clone(); - for child in &initial_children { - let ordered_idx = onto_graph - .get_node_index_by_id(ordered_id) - .expect("should have a node index"); - let child_id = node_id_map.get(*child).copied().expect("node should exist"); - let child_idx = onto_graph - .get_node_index_by_id(child_id) - .expect("should have a node index"); - onto_graph - .remove_edge( - vector_clock_id, - ordered_idx, - child_idx, - EdgeWeightKindDiscriminants::Use, - ) - .expect("unable to remove edge"); - } - - node_id_map.extend(add_prop_nodes_to_graph( - &mut onto_graph, - vector_clock_id, - &onto_children, - false, - )); - - for child in &onto_children { - let child_id = node_id_map.get(*child).copied().expect("node should exist"); - let ordered_idx = onto_graph - .get_node_index_by_id(ordered_id) - .expect("should have a node index"); - let child_idx = onto_graph - .get_node_index_by_id(child_id) - .expect("should have a node index"); - onto_graph - .add_ordered_edge( - vector_clock_id, - ordered_idx, - EdgeWeight::new(vector_clock_id, EdgeWeightKind::new_use()) - .expect("failed to make edge weight"), - child_idx, - ) - .expect("should be able to make an edge"); - } - - onto_graph.cleanup(); - onto_graph - .mark_graph_seen(vector_clock_id) - .expect("call me mark, mr seen is my father"); - - let conflicts_and_updates = to_rebase_graph - .detect_conflicts_and_updates(vector_clock_id, &onto_graph, vector_clock_id) - .expect("unable to detect conflicts and updates"); - - assert!(conflicts_and_updates.conflicts.is_empty()); - - to_rebase_graph - .perform_updates(vector_clock_id, &onto_graph, &conflicts_and_updates.updates) - .expect("unable to perform updates"); - to_rebase_graph.cleanup(); - - let ordered_idx = to_rebase_graph - .get_node_index_by_id(ordered_id) - .expect("should have a node index"); - let ordering_node = to_rebase_graph - .ordering_node_for_container(ordered_idx) - .expect("should not fail") - .expect("ordering node should exist"); - - let expected_order_ids: Vec = onto_children - .iter() - .map(|&name| node_id_map.get(name).copied().expect("get id for name")) - .collect(); - assert_eq!(&expected_order_ids, ordering_node.order()); - - let container_children: Vec = to_rebase_graph - .edges_directed_for_edge_weight_kind( - ordered_idx, - Outgoing, - EdgeWeightKindDiscriminants::Use, - ) - .iter() - .filter_map(|(_, _, target_idx)| to_rebase_graph.node_index_to_id(*target_idx)) - .collect(); - - assert_eq!(expected_order_ids.len(), container_children.len()); - for container_child in &container_children { - assert!(expected_order_ids.contains(container_child)); - } - - let ordering_node_idx = to_rebase_graph - .get_node_index_by_id(ordering_node.id()) - .expect("should have an index for ordering node"); - - let ordering_node_children: Vec = to_rebase_graph - .edges_directed(ordering_node_idx, Outgoing) - .filter_map(|edge_ref| to_rebase_graph.node_index_to_id(edge_ref.target())) - .collect(); - - for child in &ordering_node_children { - assert!(expected_order_ids.contains(child)); - } - } - - #[test] - fn detect_conflicts_and_updates_simple_ordering_with_no_conflicts_add_in_onto_remove_in_to_rebase( - ) { - let actor_id = Ulid::new(); - let initial_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut initial_graph = WorkspaceSnapshotGraphV1::new(initial_vector_clock_id) - .expect("Unable to create WorkspaceSnapshotGraph"); - - let schema_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let schema_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - schema_id, - Ulid::new(), - ContentAddress::Schema(ContentHash::from("Schema A")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema A"); - let schema_variant_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let schema_variant_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - schema_variant_id, - Ulid::new(), - ContentAddress::SchemaVariant(ContentHash::from("Schema Variant A")), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add Schema Variant A"); - - initial_graph - .add_edge( - initial_graph.root_index, - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - schema_index, - ) - .expect("Unable to add root -> schema edge"); - initial_graph - .add_edge( - initial_graph - .get_node_index_by_id(schema_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - schema_variant_index, - ) - .expect("Unable to add schema -> schema variant edge"); - - let container_prop_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let container_prop_index = initial_graph - .add_ordered_node( - initial_vector_clock_id, - NodeWeight::new_content( - initial_vector_clock_id, - container_prop_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - container_prop_id.to_string().as_bytes(), - )), - ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add container prop"); - initial_graph - .add_edge( - initial_graph - .get_node_index_by_id(schema_variant_id) - .expect("Unable to get NodeIndex"), - EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"), - container_prop_index, - ) - .expect("Unable to add schema variant -> container prop edge"); - - let ordered_prop_1_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_1_index = initial_graph - .add_node( - NodeWeight::new_content( - initial_vector_clock_id, - ordered_prop_1_id, - Ulid::new(), - ContentAddress::Prop(ContentHash::new( - ordered_prop_1_id.to_string().as_bytes(), - )), + ContentAddress::Prop(ContentHash::new( + ordered_prop_1_id.to_string().as_bytes(), + )), ) .expect("Unable to create NodeWeight"), ) @@ -2980,8 +1210,6 @@ mod test { ) .expect("Unable to add container prop -> ordered prop 1 edge"); - println!("added ordered edge from {container_prop_id} to {ordered_prop_1_id} in onto"); - let ordered_prop_2_id = initial_graph .generate_ulid() .expect("Unable to generate Ulid"); @@ -3010,8 +1238,6 @@ mod test { ) .expect("Unable to add container prop -> ordered prop 2 edge"); - println!("added ordered edge from {container_prop_id} to {ordered_prop_2_id} in onto"); - let ordered_prop_3_id = initial_graph .generate_ulid() .expect("Unable to generate Ulid"); @@ -3040,8 +1266,6 @@ mod test { ) .expect("Unable to add container prop -> ordered prop 3 edge"); - println!("added ordered edge from {container_prop_id} to {ordered_prop_3_id} in onto"); - let ordered_prop_4_id = initial_graph .generate_ulid() .expect("Unable to generate Ulid"); @@ -3070,146 +1294,218 @@ mod test { ) .expect("Unable to add container prop -> ordered prop 4 edge"); - println!("added ordered edge from {container_prop_id} to {ordered_prop_4_id} in onto"); - initial_graph.cleanup(); + initial_graph.dot(); + initial_graph .mark_graph_seen(initial_vector_clock_id) - .expect("Unable to update recently seen information"); - // initial_graph.dot(); + .expect("unable to mark graph seen"); let new_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut new_graph = initial_graph.clone(); - new_graph - .remove_edge( - new_vector_clock_id, - new_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get container NodeIndex"), - ordered_prop_2_index, - EdgeWeightKindDiscriminants::Use, - ) - .expect("Unable to remove container prop -> prop 2 edge"); - - println!("removed edge from {container_prop_id} to {ordered_prop_2_id} in to_rebase"); - - let ordered_prop_5_id = initial_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - let ordered_prop_5_index = initial_graph + let ordered_prop_5_id = new_graph.generate_ulid().expect("Unable to generate Ulid"); + let ordered_prop_5_index = new_graph .add_node( NodeWeight::new_content( - initial_vector_clock_id, + new_vector_clock_id, ordered_prop_5_id, Ulid::new(), ContentAddress::Prop(ContentHash::new( ordered_prop_5_id.to_string().as_bytes(), )), ) - .expect("Unable to create NodeWeight"), - ) - .expect("Unable to add ordered prop 5"); + .expect("Unable to create NodeWeight"), + ) + .expect("Unable to add ordered prop 5"); + new_graph + .add_ordered_edge( + new_vector_clock_id, + new_graph + .get_node_index_by_id(container_prop_id) + .expect("Unable to get NodeIndex"), + EdgeWeight::new(new_vector_clock_id, EdgeWeightKind::new_use()) + .expect("Unable to create EdgeWeight"), + ordered_prop_5_index, + ) + .expect("Unable to add container prop -> ordered prop 5 edge"); + + new_graph.cleanup(); + new_graph.dot(); + new_graph + .mark_graph_seen(new_vector_clock_id) + .expect("unable to mark graph seen"); + + let ConflictsAndUpdates { conflicts, updates } = new_graph + .detect_conflicts_and_updates( + new_vector_clock_id, + &initial_graph, + initial_vector_clock_id, + ) + .expect("Unable to detect conflicts and updates"); + + assert_eq!(Vec::::new(), conflicts); + assert_eq!(Vec::::new(), updates); + } + + #[test] + fn simple_ordering_no_conflicts_same_vector_clocks() { + let vector_clock_id = VectorClockId::new(Ulid::new(), Ulid::new()); + let mut to_rebase_graph = WorkspaceSnapshotGraphV1::new(vector_clock_id) + .expect("Unable to create WorkspaceSnapshotGraph"); + + let ordered_node = "ordered_container"; + let initial_children = vec!["a", "b", "c"]; + let onto_children = vec!["d", "e", "f"]; + + let mut node_id_map = + add_prop_nodes_to_graph(&mut to_rebase_graph, vector_clock_id, &[ordered_node], true); + node_id_map.extend(add_prop_nodes_to_graph( + &mut to_rebase_graph, + vector_clock_id, + &initial_children, + false, + )); + let ordered_id = *node_id_map.get(ordered_node).expect("should be there"); + let ordered_idx = to_rebase_graph + .get_node_index_by_id(ordered_id) + .expect("should have a node index"); + let root_idx = to_rebase_graph.root(); + to_rebase_graph + .add_edge( + root_idx, + EdgeWeight::new(vector_clock_id, EdgeWeightKind::new_use()) + .expect("failed to make edge weight"), + ordered_idx, + ) + .expect("should be able to make an edge"); + + for child in &initial_children { + let ordered_idx = to_rebase_graph + .get_node_index_by_id(ordered_id) + .expect("should have a node index"); + let child_id = node_id_map.get(*child).copied().expect("node should exist"); + let child_idx = to_rebase_graph + .get_node_index_by_id(child_id) + .expect("should have a node index"); + to_rebase_graph + .add_ordered_edge( + vector_clock_id, + ordered_idx, + EdgeWeight::new(vector_clock_id, EdgeWeightKind::new_use()) + .expect("failed to make edge weight"), + child_idx, + ) + .expect("should be able to make an edge"); + } + + to_rebase_graph.cleanup(); + to_rebase_graph + .mark_graph_seen(vector_clock_id) + .expect("mark twain"); + + let mut onto_graph = to_rebase_graph.clone(); + for child in &initial_children { + let ordered_idx = onto_graph + .get_node_index_by_id(ordered_id) + .expect("should have a node index"); + let child_id = node_id_map.get(*child).copied().expect("node should exist"); + let child_idx = onto_graph + .get_node_index_by_id(child_id) + .expect("should have a node index"); + onto_graph + .remove_edge( + vector_clock_id, + ordered_idx, + child_idx, + EdgeWeightKindDiscriminants::Use, + ) + .expect("unable to remove edge"); + } + + node_id_map.extend(add_prop_nodes_to_graph( + &mut onto_graph, + vector_clock_id, + &onto_children, + false, + )); + + for child in &onto_children { + let child_id = node_id_map.get(*child).copied().expect("node should exist"); + let ordered_idx = onto_graph + .get_node_index_by_id(ordered_id) + .expect("should have a node index"); + let child_idx = onto_graph + .get_node_index_by_id(child_id) + .expect("should have a node index"); + onto_graph + .add_ordered_edge( + vector_clock_id, + ordered_idx, + EdgeWeight::new(vector_clock_id, EdgeWeightKind::new_use()) + .expect("failed to make edge weight"), + child_idx, + ) + .expect("should be able to make an edge"); + } - let new_edge_weight = EdgeWeight::new(initial_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create EdgeWeight"); - let (_, maybe_ordinal_edge_information) = initial_graph - .add_ordered_edge( - initial_vector_clock_id, - initial_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get NodeIndex"), - new_edge_weight.clone(), - ordered_prop_5_index, - ) - .expect("Unable to add container prop -> ordered prop 5 edge"); - println!("added ordered edge from {container_prop_id} to {ordered_prop_5_id} in onto"); - let ( - ordinal_edge_index, - source_node_index_for_ordinal_edge, - destination_node_index_for_ordinal_edge, - ) = maybe_ordinal_edge_information.expect("ordinal edge information not found"); - let ordinal_edge_weight = initial_graph - .get_edge_weight_opt(ordinal_edge_index) - .expect("should not error when getting edge") - .expect("could not get edge weight for index") - .to_owned(); - let source_node_id_for_ordinal_edge = initial_graph - .get_node_weight(source_node_index_for_ordinal_edge) - .expect("could not get node weight") - .id(); - let destination_node_id_for_ordinal_edge = initial_graph - .get_node_weight(destination_node_index_for_ordinal_edge) - .expect("could not get node weight") - .id(); + onto_graph.cleanup(); + onto_graph + .mark_graph_seen(vector_clock_id) + .expect("call me mark, mr seen is my father"); - initial_graph.cleanup(); - //initial_graph.dot(); - initial_graph - .mark_graph_seen(initial_vector_clock_id) - .expect("unable mark graph seen"); + let conflicts_and_updates = to_rebase_graph + .detect_conflicts_and_updates(vector_clock_id, &onto_graph, vector_clock_id) + .expect("unable to detect conflicts and updates"); - new_graph.cleanup(); - //new_graph.dot(); - new_graph - .mark_graph_seen(new_vector_clock_id) - .expect("unable to mark graph seen"); + assert!(conflicts_and_updates.conflicts.is_empty()); - // initial_graph.tiny_dot_to_file(Some("onto")); - // new_graph.tiny_dot_to_file(Some("to_rebase")); + to_rebase_graph + .perform_updates(vector_clock_id, &conflicts_and_updates.updates) + .expect("unable to perform updates"); + to_rebase_graph.cleanup(); - let ConflictsAndUpdates { conflicts, updates } = new_graph - .detect_conflicts_and_updates( - new_vector_clock_id, - &initial_graph, - initial_vector_clock_id, + let ordered_idx = to_rebase_graph + .get_node_index_by_id(ordered_id) + .expect("should have a node index"); + let ordering_node = to_rebase_graph + .ordering_node_for_container(ordered_idx) + .expect("should not fail") + .expect("ordering node should exist"); + + let expected_order_ids: Vec = onto_children + .iter() + .map(|&name| node_id_map.get(name).copied().expect("get id for name")) + .collect(); + assert_eq!(&expected_order_ids, ordering_node.order()); + + let container_children: Vec = to_rebase_graph + .edges_directed_for_edge_weight_kind( + ordered_idx, + Outgoing, + EdgeWeightKindDiscriminants::Use, ) - .expect("Unable to detect conflicts and updates"); + .iter() + .filter_map(|(_, _, target_idx)| to_rebase_graph.node_index_to_id(*target_idx)) + .collect(); - assert_eq!(Vec::::new(), conflicts); - assert_eq!( - vec![ - UpdateWithEdgeWeightKind::NewEdge { - source: NodeInformation { - index: new_graph - .get_node_index_by_id(container_prop_id) - .expect("Unable to get new_graph container NodeIndex"), - id: container_prop_id.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - destination: NodeInformation { - index: initial_graph - .get_node_index_by_id(ordered_prop_5_id) - .expect("Unable to get ordered prop 5 NodeIndex"), - id: ordered_prop_5_id.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - edge_weight_kind: new_edge_weight.kind().clone(), - }, - UpdateWithEdgeWeightKind::NewEdge { - source: NodeInformation { - index: new_graph - .get_node_index_by_id(source_node_id_for_ordinal_edge) - .expect("could not get node index by id"), - id: source_node_id_for_ordinal_edge.into(), - node_weight_kind: NodeWeightDiscriminants::Ordering, - }, - destination: NodeInformation { - index: initial_graph - .get_node_index_by_id(destination_node_id_for_ordinal_edge) - .expect("could not get node index by id"), - id: destination_node_id_for_ordinal_edge.into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }, - edge_weight_kind: ordinal_edge_weight.kind().clone(), - } - ], - updates - .into_iter() - .map(Into::::into) - .collect::>(), - ); + assert_eq!(expected_order_ids.len(), container_children.len()); + for container_child in &container_children { + assert!(expected_order_ids.contains(container_child)); + } + + let ordering_node_idx = to_rebase_graph + .get_node_index_by_id(ordering_node.id()) + .expect("should have an index for ordering node"); + + let ordering_node_children: Vec = to_rebase_graph + .edges_directed(ordering_node_idx, Outgoing) + .filter_map(|edge_ref| to_rebase_graph.node_index_to_id(edge_ref.target())) + .collect(); + + for child in &ordering_node_children { + assert!(expected_order_ids.contains(child)); + } } #[test] @@ -3381,421 +1677,23 @@ mod test { let ConflictsAndUpdates { conflicts, updates } = base_graph .detect_conflicts_and_updates(base_vector_clock_id, &new_graph, new_vector_clock_id) - .expect("Unable to detect conflicts and updates"); - - assert_eq!( - vec![Update::RemoveEdge { - source: NodeInformation { - index: a_idx, - id: a_id.into(), - node_weight_kind: NodeWeightDiscriminants::Prop, - }, - destination: NodeInformation { - index: c_idx, - id: c_id.into(), - node_weight_kind: NodeWeightDiscriminants::Prop, - }, - edge_kind: EdgeWeightKindDiscriminants::Use, - }], - updates - ); - assert!(conflicts.is_empty()); - } - - #[test] - fn detect_exclusive_edge_conflict() { - let actor_id = Ulid::new(); - let base_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut base_graph = - WorkspaceSnapshotGraphV1::new(base_vector_clock_id).expect("Unable to make base graph"); - - let av_id = base_graph.generate_ulid().expect("Unable to generate Ulid"); - base_graph - .add_node( - NodeWeight::new_attribute_value( - base_vector_clock_id, - av_id, - Ulid::new(), - None, - None, - ) - .expect("Unable to create AttributeValue"), - ) - .expect("Unable to add AttributeValue"); - base_graph - .add_edge( - base_graph.root_index, - EdgeWeight::new(base_vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create edge weight"), - base_graph - .get_node_index_by_id(av_id) - .expect("Unable to get node index for AttributeValue"), - ) - .expect("Unable to add root -> AV Use edge"); - base_graph.cleanup(); - base_graph - .mark_graph_seen(base_vector_clock_id) - .expect("Unable to mark base graph as seen"); - - let new_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut new_changes_graph = base_graph.clone(); - - let prototype_a_id = new_changes_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - new_changes_graph - .add_node( - NodeWeight::new_content( - new_vector_clock_id, - prototype_a_id, - Ulid::new(), - ContentAddress::AttributePrototype(ContentHash::new( - "Prototype A".to_string().as_bytes(), - )), - ) - .expect("Unable to create content node weight"), - ) - .expect("Unable to add prototype a to graph"); - new_changes_graph - .add_edge( - new_changes_graph - .get_node_index_by_id(av_id) - .expect("Unable to get AV node index"), - EdgeWeight::new(new_vector_clock_id, EdgeWeightKind::Prototype(None)) - .expect("Unable to create edge weight"), - new_changes_graph - .get_node_index_by_id(prototype_a_id) - .expect("Unable to get node index for AttributePrototype"), - ) - .expect("Unable to add AV -> AttributePrototype Prototype edge"); - new_changes_graph.cleanup(); - new_changes_graph - .mark_graph_seen(new_vector_clock_id) - .expect("Unable to mark new changes graph as seen"); - - let ConflictsAndUpdates { - conflicts, - updates: _, - } = base_graph - .detect_conflicts_and_updates( - base_vector_clock_id, - &new_changes_graph, - new_vector_clock_id, - ) - .expect("able to detect conflicts and updates"); - - assert_eq!(Vec::::new(), conflicts); - - let conflicting_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut conflicting_graph = new_changes_graph.clone(); - - let prototype_b_id = conflicting_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - conflicting_graph - .add_node( - NodeWeight::new_content( - conflicting_vector_clock_id, - prototype_b_id, - Ulid::new(), - ContentAddress::AttributePrototype(ContentHash::new( - "Prototype B".to_string().as_bytes(), - )), - ) - .expect("Unable to create content node weight"), - ) - .expect("Unable to add prototype b to graph"); - conflicting_graph - .add_edge( - conflicting_graph - .get_node_index_by_id(av_id) - .expect("Unable to get AV node index"), - EdgeWeight::new(conflicting_vector_clock_id, EdgeWeightKind::Prototype(None)) - .expect("Unable to create edge weight"), - conflicting_graph - .get_node_index_by_id(prototype_b_id) - .expect("Unable to get node index for AttributePrototype"), - ) - .expect("Unable to add AV -> AttributePrototype Prototype edge"); - - conflicting_graph.cleanup(); - conflicting_graph - .mark_graph_seen(conflicting_vector_clock_id) - .expect("Unable to mark conflicting graph as seen"); - - let ConflictsAndUpdates { - conflicts, - updates: _, - } = new_changes_graph - .detect_conflicts_and_updates( - new_vector_clock_id, - &conflicting_graph, - conflicting_vector_clock_id, - ) - .expect("able to detect conflicts and updates"); - - assert_eq!( - vec![Conflict::ExclusiveEdgeMismatch { - source: NodeInformation { - index: new_changes_graph - .get_node_index_by_id(av_id) - .expect("Unable to get AV node index"), - node_weight_kind: NodeWeightDiscriminants::AttributeValue, - id: av_id.into(), - }, - destination: NodeInformation { - index: conflicting_graph - .get_node_index_by_id(prototype_b_id) - .expect("Unable to get Prototype B node index"), - node_weight_kind: NodeWeightDiscriminants::Content, - id: prototype_b_id.into(), - }, - edge_kind: EdgeWeightKindDiscriminants::Prototype, - }], - conflicts, - ); - - let ConflictsAndUpdates { - conflicts, - updates: _, - } = base_graph - .detect_conflicts_and_updates( - base_vector_clock_id, - &conflicting_graph, - conflicting_vector_clock_id, - ) - .expect("able to detect conflicts and updates"); - - let expected_conflicts: HashSet = [ - Conflict::ExclusiveEdgeMismatch { - source: NodeInformation { - index: base_graph - .get_node_index_by_id(av_id) - .expect("Unable to get AV node index"), - node_weight_kind: NodeWeightDiscriminants::AttributeValue, - id: av_id.into(), - }, - destination: NodeInformation { - index: conflicting_graph - .get_node_index_by_id(prototype_a_id) - .expect("Unable to get Prototype A node index"), - node_weight_kind: NodeWeightDiscriminants::Content, - id: prototype_a_id.into(), - }, - edge_kind: EdgeWeightKindDiscriminants::Prototype, - }, - Conflict::ExclusiveEdgeMismatch { - source: NodeInformation { - index: base_graph - .get_node_index_by_id(av_id) - .expect("Unable to get AV node index"), - node_weight_kind: NodeWeightDiscriminants::AttributeValue, - id: av_id.into(), - }, - destination: NodeInformation { - index: conflicting_graph - .get_node_index_by_id(prototype_b_id) - .expect("Unable to get Prototype B node index"), - node_weight_kind: NodeWeightDiscriminants::Content, - id: prototype_b_id.into(), - }, - edge_kind: EdgeWeightKindDiscriminants::Prototype, - }, - ] - .iter() - .copied() - .collect(); - let actual_conflicts: HashSet = conflicts.iter().copied().collect(); - - assert_eq!(expected_conflicts, actual_conflicts); - } - - #[test] - fn detect_exclusive_edge_conflict_same_vector_clocks() { - let actor_id = Ulid::new(); - let vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut base_graph = - WorkspaceSnapshotGraphV1::new(vector_clock_id).expect("Unable to make base graph"); - - let av_id = base_graph.generate_ulid().expect("Unable to generate Ulid"); - base_graph - .add_node( - NodeWeight::new_attribute_value(vector_clock_id, av_id, Ulid::new(), None, None) - .expect("Unable to create AttributeValue"), - ) - .expect("Unable to add AttributeValue"); - base_graph - .add_edge( - base_graph.root_index, - EdgeWeight::new(vector_clock_id, EdgeWeightKind::new_use()) - .expect("Unable to create edge weight"), - base_graph - .get_node_index_by_id(av_id) - .expect("Unable to get node index for AttributeValue"), - ) - .expect("Unable to add root -> AV Use edge"); - base_graph.cleanup(); - base_graph - .mark_graph_seen(vector_clock_id) - .expect("Unable to mark base graph as seen"); - - let mut new_changes_graph = base_graph.clone(); - - let prototype_a_id = new_changes_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - new_changes_graph - .add_node( - NodeWeight::new_content( - vector_clock_id, - prototype_a_id, - Ulid::new(), - ContentAddress::AttributePrototype(ContentHash::new( - "Prototype A".to_string().as_bytes(), - )), - ) - .expect("Unable to create content node weight"), - ) - .expect("Unable to add prototype a to graph"); - new_changes_graph - .add_edge( - new_changes_graph - .get_node_index_by_id(av_id) - .expect("Unable to get AV node index"), - EdgeWeight::new(vector_clock_id, EdgeWeightKind::Prototype(None)) - .expect("Unable to create edge weight"), - new_changes_graph - .get_node_index_by_id(prototype_a_id) - .expect("Unable to get node index for AttributePrototype"), - ) - .expect("Unable to add AV -> AttributePrototype Prototype edge"); - new_changes_graph.cleanup(); - new_changes_graph - .mark_graph_seen(vector_clock_id) - .expect("Unable to mark new changes graph as seen"); - - let ConflictsAndUpdates { - conflicts, - updates: _, - } = base_graph - .detect_conflicts_and_updates(vector_clock_id, &new_changes_graph, vector_clock_id) - .expect("able to detect conflicts and updates"); - - assert_eq!(Vec::::new(), conflicts); - - let mut conflicting_graph = new_changes_graph.clone(); - - let prototype_b_id = conflicting_graph - .generate_ulid() - .expect("Unable to generate Ulid"); - conflicting_graph - .add_node( - NodeWeight::new_content( - vector_clock_id, - prototype_b_id, - Ulid::new(), - ContentAddress::AttributePrototype(ContentHash::new( - "Prototype B".to_string().as_bytes(), - )), - ) - .expect("Unable to create content node weight"), - ) - .expect("Unable to add prototype b to graph"); - conflicting_graph - .add_edge( - conflicting_graph - .get_node_index_by_id(av_id) - .expect("Unable to get AV node index"), - EdgeWeight::new(vector_clock_id, EdgeWeightKind::Prototype(None)) - .expect("Unable to create edge weight"), - conflicting_graph - .get_node_index_by_id(prototype_b_id) - .expect("Unable to get node index for AttributePrototype"), - ) - .expect("Unable to add AV -> AttributePrototype Prototype edge"); - - conflicting_graph.cleanup(); - conflicting_graph - .mark_graph_seen(vector_clock_id) - .expect("Unable to mark conflicting graph as seen"); - - let ConflictsAndUpdates { - conflicts, - updates: _, - } = new_changes_graph - .detect_conflicts_and_updates(vector_clock_id, &conflicting_graph, vector_clock_id) - .expect("able to detect conflicts and updates"); - - assert_eq!( - vec![Conflict::ExclusiveEdgeMismatch { - source: NodeInformation { - index: new_changes_graph - .get_node_index_by_id(av_id) - .expect("Unable to get AV node index"), - node_weight_kind: NodeWeightDiscriminants::AttributeValue, - id: av_id.into(), - }, - destination: NodeInformation { - index: conflicting_graph - .get_node_index_by_id(prototype_b_id) - .expect("Unable to get Prototype B node index"), - node_weight_kind: NodeWeightDiscriminants::Content, - id: prototype_b_id.into(), - }, - edge_kind: EdgeWeightKindDiscriminants::Prototype, - }], - conflicts, - ); - - let ConflictsAndUpdates { - conflicts, - updates: _, - } = base_graph - .detect_conflicts_and_updates(vector_clock_id, &conflicting_graph, vector_clock_id) - .expect("able to detect conflicts and updates"); + .expect("Unable to detect conflicts and updates"); - let expected_conflicts: HashSet = [ - Conflict::ExclusiveEdgeMismatch { - source: NodeInformation { - index: base_graph - .get_node_index_by_id(av_id) - .expect("Unable to get AV node index"), - node_weight_kind: NodeWeightDiscriminants::AttributeValue, - id: av_id.into(), - }, - destination: NodeInformation { - index: conflicting_graph - .get_node_index_by_id(prototype_a_id) - .expect("Unable to get Prototype A node index"), - node_weight_kind: NodeWeightDiscriminants::Content, - id: prototype_a_id.into(), - }, - edge_kind: EdgeWeightKindDiscriminants::Prototype, - }, - Conflict::ExclusiveEdgeMismatch { + assert_eq!( + vec![Update::RemoveEdge { source: NodeInformation { - index: base_graph - .get_node_index_by_id(av_id) - .expect("Unable to get AV node index"), - node_weight_kind: NodeWeightDiscriminants::AttributeValue, - id: av_id.into(), + id: a_id.into(), + node_weight_kind: NodeWeightDiscriminants::Prop, }, destination: NodeInformation { - index: conflicting_graph - .get_node_index_by_id(prototype_b_id) - .expect("Unable to get Prototype B node index"), - node_weight_kind: NodeWeightDiscriminants::Content, - id: prototype_b_id.into(), + id: c_id.into(), + node_weight_kind: NodeWeightDiscriminants::Prop, }, - edge_kind: EdgeWeightKindDiscriminants::Prototype, - }, - ] - .iter() - .copied() - .collect(); - let actual_conflicts: HashSet = conflicts.iter().copied().collect(); - - assert_eq!(expected_conflicts, actual_conflicts); + edge_kind: EdgeWeightKindDiscriminants::Use, + }], + updates + ); + assert!(conflicts.is_empty()); } #[test] @@ -3873,420 +1771,4 @@ mod test { } )); } - - #[test] - fn detect_conflicts_and_updates_remove_modified_item_conflict() { - let actor_id = Ulid::new(); - let to_rebase_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut to_rebase_graph = WorkspaceSnapshotGraphV1::new(to_rebase_vector_clock_id) - .expect("unable to make to_rebase_graph"); - - let prototype_node_id = to_rebase_graph.generate_ulid().expect("gen ulid"); - let prototype_node = NodeWeight::new_content( - to_rebase_vector_clock_id, - prototype_node_id, - Ulid::new(), - ContentAddress::AttributePrototype(ContentHash::from("prototype")), - ) - .expect("unable to create prototype node weight"); - - to_rebase_graph - .add_node(prototype_node) - .expect("unable to add node"); - to_rebase_graph - .add_edge( - to_rebase_graph.root(), - EdgeWeight::new(to_rebase_vector_clock_id, EdgeWeightKind::Prototype(None)) - .expect("make edge weight"), - to_rebase_graph - .get_node_index_by_id(prototype_node_id) - .expect("get_node_index_by_id"), - ) - .expect("unable to add edge"); - - // "write" the graph - to_rebase_graph.cleanup(); - to_rebase_graph - .mark_graph_seen(to_rebase_vector_clock_id) - .expect("mark_graph_seen"); - - // "fork" a working changeset from the current one - let onto_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut onto_graph = to_rebase_graph.clone(); - - // After the fork, remove the edge in to_rebase, but modify the edge in onto - to_rebase_graph - .remove_edge( - onto_vector_clock_id, - onto_graph.root(), - to_rebase_graph - .get_node_index_by_id(prototype_node_id) - .expect("get_node_index_by_id"), - EdgeWeightKindDiscriminants::Prototype, - ) - .expect("remove_edge"); - to_rebase_graph.cleanup(); - to_rebase_graph - .mark_graph_seen(to_rebase_vector_clock_id) - .expect("mark_graph_seen"); - - let onto_content_node_idx = onto_graph - .get_node_index_by_id(prototype_node_id) - .expect("get_node_index_by_id"); - - let mut content_node = onto_graph - .get_node_weight(onto_content_node_idx) - .expect("get_node_weight") - .get_content_node_weight_of_kind(ContentAddressDiscriminants::AttributePrototype) - .expect("get_content_node_weight_of_kind"); - - // Modifying this node in onto, after it has been removed in to_rebase, - // will produce a RemoveModifiedItem conflict - content_node - .new_content_hash(ContentHash::from("prototype_change")) - .expect("update_content_hash"); - content_node.increment_vector_clocks(onto_vector_clock_id); - onto_graph - .add_node(NodeWeight::Content(content_node)) - .expect("add_node"); - onto_graph - .replace_references(onto_content_node_idx) - .expect("replace_references"); - onto_graph.cleanup(); - onto_graph - .mark_graph_seen(onto_vector_clock_id) - .expect("mark_graph_seen"); - - let ConflictsAndUpdates { conflicts, updates } = to_rebase_graph - .detect_conflicts_and_updates( - to_rebase_vector_clock_id, - &onto_graph, - onto_vector_clock_id, - ) - .expect("detect_conflicts_and_updates"); - - // Since the node in question is removed in to_rebase, there will be no - // ReplaceSubgraph update - assert!(updates.is_empty()); - assert_eq!(1, conflicts.len()); - - let container = NodeInformation { - index: to_rebase_graph.root_index, - id: to_rebase_graph - .get_node_weight(to_rebase_graph.root()) - .expect("Unable to get root node") - .id() - .into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }; - let removed_index = onto_graph - .get_node_index_by_id(prototype_node_id) - .expect("get_node_index_by_id"); - let removed_item = NodeInformation { - index: removed_index, - id: onto_graph - .get_node_weight(removed_index) - .expect("Unable to get removed item node weight") - .id() - .into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }; - assert_eq!( - conflicts[0], - Conflict::RemoveModifiedItem { - container, - removed_item - } - ); - } - - #[test] - fn detect_conflicts_and_updates_remove_modified_item_conflict_same_vector_clocks() { - let actor_id = Ulid::new(); - let vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut to_rebase_graph = - WorkspaceSnapshotGraphV1::new(vector_clock_id).expect("unable to make to_rebase_graph"); - - let prototype_node_id = to_rebase_graph.generate_ulid().expect("gen ulid"); - let prototype_node = NodeWeight::new_content( - vector_clock_id, - prototype_node_id, - Ulid::new(), - ContentAddress::AttributePrototype(ContentHash::from("prototype")), - ) - .expect("unable to create prototype node weight"); - - to_rebase_graph - .add_node(prototype_node) - .expect("unable to add node"); - to_rebase_graph - .add_edge( - to_rebase_graph.root(), - EdgeWeight::new(vector_clock_id, EdgeWeightKind::Prototype(None)) - .expect("make edge weight"), - to_rebase_graph - .get_node_index_by_id(prototype_node_id) - .expect("get_node_index_by_id"), - ) - .expect("unable to add edge"); - - // "write" the graph - to_rebase_graph.cleanup(); - to_rebase_graph - .mark_graph_seen(vector_clock_id) - .expect("mark_graph_seen"); - - // "fork" a working changeset from the current one - let mut onto_graph = to_rebase_graph.clone(); - - // After the fork, remove the edge in to_rebase, but modify the edge in onto - to_rebase_graph - .remove_edge( - vector_clock_id, - onto_graph.root(), - to_rebase_graph - .get_node_index_by_id(prototype_node_id) - .expect("get_node_index_by_id"), - EdgeWeightKindDiscriminants::Prototype, - ) - .expect("remove_edge"); - to_rebase_graph.cleanup(); - to_rebase_graph - .mark_graph_seen(vector_clock_id) - .expect("mark_graph_seen"); - - let onto_content_node_idx = onto_graph - .get_node_index_by_id(prototype_node_id) - .expect("get_node_index_by_id"); - - let mut content_node = onto_graph - .get_node_weight(onto_content_node_idx) - .expect("get_node_weight") - .get_content_node_weight_of_kind(ContentAddressDiscriminants::AttributePrototype) - .expect("get_content_node_weight_of_kind"); - - // Modifying this node in onto, after it has been removed in to_rebase - // will produce a RemoveModifiedItem conflict, even though the vector - // clocks are the same, since the same conditions still hold. - content_node - .new_content_hash(ContentHash::from("prototype_change")) - .expect("update_content_hash"); - content_node.increment_vector_clocks(vector_clock_id); - onto_graph - .add_node(NodeWeight::Content(content_node)) - .expect("add_node"); - onto_graph - .replace_references(onto_content_node_idx) - .expect("replace_references"); - onto_graph.cleanup(); - onto_graph - .mark_graph_seen(vector_clock_id) - .expect("mark_graph_seen"); - - let ConflictsAndUpdates { conflicts, updates } = to_rebase_graph - .detect_conflicts_and_updates(vector_clock_id, &onto_graph, vector_clock_id) - .expect("detect_conflicts_and_updates"); - - assert!(updates.is_empty()); - assert_eq!(1, conflicts.len()); - - let container = get_root_node_info(&to_rebase_graph); - - let removed_index = onto_graph - .get_node_index_by_id(prototype_node_id) - .expect("get_node_index_by_id"); - let removed_item = NodeInformation { - index: removed_index, - id: onto_graph - .get_node_weight(removed_index) - .expect("Unable to get removed item node weight") - .id() - .into(), - node_weight_kind: NodeWeightDiscriminants::Content, - }; - assert_eq!( - Conflict::RemoveModifiedItem { - container, - removed_item, - }, - conflicts[0], - ); - - // Reversing the detection order should produce the ModifyRemovedItem conflict - let ConflictsAndUpdates { conflicts, updates } = onto_graph - .detect_conflicts_and_updates(vector_clock_id, &to_rebase_graph, vector_clock_id) - .expect("detect_conflicts_and_updates"); - assert!(updates.is_empty()); - let container = get_root_node_info(&onto_graph); - assert_eq!( - Conflict::ModifyRemovedItem { - container, - modified_item: removed_item - }, - conflicts[0] - ); - } - - #[test] - fn test_merge_dependent_value_roots() { - let actor_id = Ulid::new(); - let to_rebase_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut to_rebase_graph = WorkspaceSnapshotGraphV1::new(to_rebase_vector_clock_id) - .expect("unable to make to_rebase_graph"); - - to_rebase_graph - .mark_graph_seen(to_rebase_vector_clock_id) - .expect("unable to mark graph seen"); - - let onto_vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); - let mut onto_graph = to_rebase_graph.clone(); - - let cat_node_idx = to_rebase_graph - .add_category_node( - to_rebase_vector_clock_id, - Ulid::new(), - Ulid::new(), - DependentValueRoots, - ) - .expect("able to add dvu root cat node"); - let to_rebase_cat_node_orig_weight = to_rebase_graph - .get_node_weight(cat_node_idx) - .expect("unable to get node weight") - .to_owned(); - to_rebase_graph - .add_edge( - to_rebase_graph.root_index, - EdgeWeight::new(to_rebase_vector_clock_id, EdgeWeightKind::new_use()) - .expect("unable to make edge weigh"), - cat_node_idx, - ) - .expect("unable add edge "); - - let cat_node_idx = onto_graph - .add_category_node( - onto_vector_clock_id, - Ulid::new(), - Ulid::new(), - DependentValueRoots, - ) - .expect("unable to add dvu root cat node"); - onto_graph - .add_edge( - onto_graph.root_index, - EdgeWeight::new(onto_vector_clock_id, EdgeWeightKind::new_use()) - .expect("unable to make edge weigh"), - cat_node_idx, - ) - .expect("unable add edge "); - - let shared_value_id = to_rebase_graph.generate_ulid().expect("unable to gen ulid"); - - let unique_to_rebase_value_id = to_rebase_graph - .generate_ulid() - .expect("unable to generate ulid"); - - let unique_to_onto_value_id = onto_graph.generate_ulid().expect("unable to generate ulid"); - - let to_rebase_value_ids = [unique_to_rebase_value_id, shared_value_id]; - let onto_value_ids = [unique_to_onto_value_id, shared_value_id]; - - for (graph, vector_clock_id, values) in [ - ( - &mut to_rebase_graph, - to_rebase_vector_clock_id, - &to_rebase_value_ids, - ), - (&mut onto_graph, onto_vector_clock_id, &onto_value_ids), - ] { - let (cat_id, _) = graph - .get_category_node(None, DependentValueRoots) - .expect("unable to get cat node") - .expect("cat node for dvu roots not there"); - - for value_id in values { - let node_weight = NodeWeight::new_dependent_value_root( - vector_clock_id, - Ulid::new(), - Ulid::new(), - *value_id, - ) - .expect("unable to make root node weight"); - let dvu_root_idx = graph.add_node(node_weight).expect("unable to add node"); - graph - .add_edge( - graph - .get_node_index_by_id(cat_id) - .expect("unable to get node index for category"), - EdgeWeight::new(vector_clock_id, EdgeWeightKind::new_use()) - .expect("unable to make edge weight"), - dvu_root_idx, - ) - .expect("unable to add edge"); - } - } - - to_rebase_graph.cleanup(); - onto_graph.cleanup(); - to_rebase_graph - .mark_graph_seen(to_rebase_vector_clock_id) - .expect("unable to mark graph seen"); - onto_graph - .mark_graph_seen(onto_vector_clock_id) - .expect("unable to mark graph seen"); - - let ConflictsAndUpdates { conflicts, updates } = to_rebase_graph - .detect_conflicts_and_updates( - to_rebase_vector_clock_id, - &onto_graph, - onto_vector_clock_id, - ) - .expect("able to detect conflicts and updates"); - - assert!(conflicts.is_empty()); - - to_rebase_graph - .perform_updates(to_rebase_vector_clock_id, &onto_graph, &updates) - .expect("unable to perform updates"); - - to_rebase_graph.cleanup(); - to_rebase_graph - .mark_graph_seen(to_rebase_vector_clock_id) - .expect("unable to mark graph seen"); - - let neighbors_of_root: Vec = to_rebase_graph - .edges_directed(to_rebase_graph.root_index, Outgoing) - .map(|edge_ref| edge_ref.target()) - .collect(); - - assert_eq!(1, neighbors_of_root.len()); - let cat_node_idx = neighbors_of_root[0]; - let cat_node_weight = to_rebase_graph - .get_node_weight(cat_node_idx) - .expect("unable to get cat node weight") - .to_owned(); - - assert_eq!(to_rebase_cat_node_orig_weight.id(), cat_node_weight.id()); - let neighbors_of_cat: Vec = to_rebase_graph - .edges_directed(cat_node_idx, Outgoing) - .map(|edge_ref| edge_ref.target()) - .collect(); - - assert_eq!(4, neighbors_of_cat.len()); - let mut expected_id_set: HashSet = HashSet::new(); - expected_id_set.extend(&to_rebase_value_ids); - expected_id_set.extend(&onto_value_ids); - - let mut found_id_set = HashSet::new(); - for neighbor_idx in neighbors_of_cat { - let node_weight = to_rebase_graph - .get_node_weight(neighbor_idx) - .expect("unable to get node weight") - .get_dependent_value_root_node_weight() - .expect("unable to get dvu root"); - - found_id_set.insert(node_weight.value_id()); - } - - assert_eq!(expected_id_set, found_id_set); - } } diff --git a/lib/dal/src/workspace_snapshot/graph/tests/rebase.rs b/lib/dal/src/workspace_snapshot/graph/tests/rebase.rs index c0acc3f035..5fc2ffbb8b 100644 --- a/lib/dal/src/workspace_snapshot/graph/tests/rebase.rs +++ b/lib/dal/src/workspace_snapshot/graph/tests/rebase.rs @@ -153,7 +153,7 @@ mod test { .expect("could not detect conflicts and updates"); assert!(conflicts.is_empty()); assert_eq!( - 2, // expected + 7, // expected updates.len() // actual ); assert_eq!( @@ -176,7 +176,7 @@ mod test { // Perform the updates. In the future, we may want to see if the onto and resulting to // rebase graphs are logically equivalent after updates are performed. to_rebase - .perform_updates(to_rebase_vector_clock_id, &onto, &updates) + .perform_updates(to_rebase_vector_clock_id, &updates) .expect("could not perform updates"); } } diff --git a/lib/dal/src/workspace_snapshot/node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight.rs index e7d15e6c4f..fa16e145b3 100644 --- a/lib/dal/src/workspace_snapshot/node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight.rs @@ -86,7 +86,7 @@ pub type NodeWeightResult = Result; /// **WARNING**: the order of this enum is important! Do not re-order elements. /// New variants must go at the end, even if it's not in lexical order! -#[derive(Debug, Serialize, Deserialize, Clone, EnumDiscriminants)] +#[derive(Debug, Serialize, Deserialize, Clone, EnumDiscriminants, PartialEq, Eq)] #[strum_discriminants(derive(strum::Display, Hash, Serialize, Deserialize))] pub enum NodeWeight { Action(ActionNodeWeight), diff --git a/lib/dal/src/workspace_snapshot/node_weight/action_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/action_node_weight.rs index 5dba34a039..da1d1dfddf 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/action_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/action_node_weight.rs @@ -12,7 +12,7 @@ use crate::{ use super::{deprecated::DeprecatedActionNodeWeight, NodeWeightResult}; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct ActionNodeWeight { pub id: Ulid, state: ActionState, diff --git a/lib/dal/src/workspace_snapshot/node_weight/action_prototype_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/action_prototype_node_weight.rs index 86ab51eaf0..c5cea3c926 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/action_prototype_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/action_prototype_node_weight.rs @@ -12,7 +12,7 @@ use crate::{ use super::{deprecated::DeprecatedActionPrototypeNodeWeight, NodeWeightResult}; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct ActionPrototypeNodeWeight { pub id: Ulid, kind: ActionKind, diff --git a/lib/dal/src/workspace_snapshot/node_weight/attribute_prototype_argument_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/attribute_prototype_argument_node_weight.rs index bf8cad0627..e436c990a4 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/attribute_prototype_argument_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/attribute_prototype_argument_node_weight.rs @@ -20,7 +20,7 @@ pub struct ArgumentTargets { pub destination_component_id: ComponentId, } -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct AttributePrototypeArgumentNodeWeight { pub id: Ulid, pub lineage_id: LineageId, diff --git a/lib/dal/src/workspace_snapshot/node_weight/attribute_value_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/attribute_value_node_weight.rs index 58166259e6..549e68b3c2 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/attribute_value_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/attribute_value_node_weight.rs @@ -13,7 +13,7 @@ use crate::{ use super::deprecated::DeprecatedAttributeValueNodeWeight; -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct AttributeValueNodeWeight { pub id: Ulid, pub lineage_id: LineageId, diff --git a/lib/dal/src/workspace_snapshot/node_weight/category_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/category_node_weight.rs index e2b59744bf..5043cc62e4 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/category_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/category_node_weight.rs @@ -28,7 +28,7 @@ pub enum CategoryNodeKind { DependentValueRoots, } -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct CategoryNodeWeight { pub id: Ulid, pub lineage_id: LineageId, diff --git a/lib/dal/src/workspace_snapshot/node_weight/component_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/component_node_weight.rs index da58c28d58..b3dcac60a1 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/component_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/component_node_weight.rs @@ -12,7 +12,7 @@ use crate::{ use super::{deprecated::DeprecatedComponentNodeWeight, NodeWeightError, NodeWeightResult}; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct ComponentNodeWeight { pub id: Ulid, pub lineage_id: LineageId, diff --git a/lib/dal/src/workspace_snapshot/node_weight/content_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/content_node_weight.rs index 2ea6e70e35..855d40a667 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/content_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/content_node_weight.rs @@ -13,7 +13,7 @@ use crate::workspace_snapshot::{ }; use crate::EdgeWeightKindDiscriminants; -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct ContentNodeWeight { /// The stable local ID of the object in question. Mainly used by external things like /// the UI to be able to say "do X to _this_ thing" since the `NodeIndex` is an diff --git a/lib/dal/src/workspace_snapshot/node_weight/dependent_value_root_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/dependent_value_root_node_weight.rs index 997985ab13..14498ffdc1 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/dependent_value_root_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/dependent_value_root_node_weight.rs @@ -8,7 +8,7 @@ use crate::EdgeWeightKindDiscriminants; use super::deprecated::DeprecatedDependentValueRootNodeWeight; use super::NodeWeightResult; -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct DependentValueRootNodeWeight { pub id: Ulid, pub lineage_id: Ulid, diff --git a/lib/dal/src/workspace_snapshot/node_weight/func_argument_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/func_argument_node_weight.rs index 091397e2c2..bb961f6865 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/func_argument_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/func_argument_node_weight.rs @@ -14,7 +14,7 @@ use crate::{ use super::deprecated::DeprecatedFuncArgumentNodeWeight; -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct FuncArgumentNodeWeight { pub id: Ulid, pub lineage_id: LineageId, diff --git a/lib/dal/src/workspace_snapshot/node_weight/func_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/func_node_weight.rs index 86284d1118..d96bf9e326 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/func_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/func_node_weight.rs @@ -15,7 +15,7 @@ use crate::EdgeWeightKindDiscriminants; use super::deprecated::DeprecatedFuncNodeWeight; -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct FuncNodeWeight { pub id: Ulid, pub lineage_id: LineageId, diff --git a/lib/dal/src/workspace_snapshot/node_weight/ordering_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/ordering_node_weight.rs index 6a0aad9b96..75244eb290 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/ordering_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/ordering_node_weight.rs @@ -7,7 +7,7 @@ use crate::workspace_snapshot::vector_clock::{HasVectorClocks, VectorClockId}; use crate::workspace_snapshot::{node_weight::NodeWeightResult, vector_clock::VectorClock}; use crate::EdgeWeightKindDiscriminants; -#[derive(Clone, Serialize, Deserialize, Default)] +#[derive(Clone, Serialize, Deserialize, Default, PartialEq, Eq)] pub struct OrderingNodeWeight { pub id: Ulid, pub lineage_id: Ulid, diff --git a/lib/dal/src/workspace_snapshot/node_weight/prop_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/prop_node_weight.rs index d9074703b6..35faacb2df 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/prop_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/prop_node_weight.rs @@ -17,7 +17,7 @@ use crate::{ use super::deprecated::DeprecatedPropNodeWeight; -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct PropNodeWeight { pub id: Ulid, pub lineage_id: LineageId, diff --git a/lib/dal/src/workspace_snapshot/node_weight/secret_node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight/secret_node_weight.rs index a1386a4261..ae742231f6 100644 --- a/lib/dal/src/workspace_snapshot/node_weight/secret_node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight/secret_node_weight.rs @@ -14,7 +14,7 @@ use crate::EdgeWeightKindDiscriminants; use super::deprecated::DeprecatedSecretNodeWeight; -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct SecretNodeWeight { pub id: Ulid, pub lineage_id: LineageId, diff --git a/lib/dal/src/workspace_snapshot/update.rs b/lib/dal/src/workspace_snapshot/update.rs index 445d78ad98..4009bd0f44 100644 --- a/lib/dal/src/workspace_snapshot/update.rs +++ b/lib/dal/src/workspace_snapshot/update.rs @@ -1,6 +1,3 @@ -use petgraph::prelude::NodeIndex; -use si_events::ulid::Ulid; - use serde::{Deserialize, Serialize}; use strum::EnumDiscriminants; @@ -8,6 +5,7 @@ use super::{ edge_info::EdgeInfo, edge_weight::{EdgeWeight, EdgeWeightKindDiscriminants}, graph::WorkspaceSnapshotGraphResult, + node_weight::NodeWeight, }; use crate::{workspace_snapshot::NodeInformation, WorkspaceSnapshotGraphV1}; @@ -26,40 +24,31 @@ pub enum Update { destination: NodeInformation, edge_kind: EdgeWeightKindDiscriminants, }, - // This is not correctly named. We really only want to replace the single node, as we also - // generate Update entries to handle processing the rest of the subgraph. - ReplaceSubgraph { - onto: NodeInformation, - // Check if already exists in "onto". Grab node weight from "to_rebase" and see if there is - // an equivalent node (id and lineage) in "onto". If not, use "import_subgraph". - to_rebase: NodeInformation, + ReplaceNode { + node_weight: NodeWeight, }, - MergeCategoryNodes { - to_rebase_category_id: Ulid, - onto_category_id: Ulid, + NewNode { + node_weight: NodeWeight, }, } impl Update { /// Produce a NewEdge update from an edge that exists only in the "onto" graph pub fn new_edge( - to_rebase_graph: &WorkspaceSnapshotGraphV1, onto_graph: &WorkspaceSnapshotGraphV1, - to_rebase_source_index: NodeIndex, only_onto_edge_info: &EdgeInfo, only_onto_edge_weight: EdgeWeight, ) -> WorkspaceSnapshotGraphResult { - let source_node_weight = to_rebase_graph.get_node_weight(to_rebase_source_index)?; + let source_node_weight = + onto_graph.get_node_weight(only_onto_edge_info.source_node_index)?; let target_node_weight = onto_graph.get_node_weight(only_onto_edge_info.target_node_index)?; let source = NodeInformation { - index: to_rebase_source_index, id: source_node_weight.id().into(), node_weight_kind: source_node_weight.into(), }; let destination = NodeInformation { - index: only_onto_edge_info.target_node_index, id: target_node_weight.id().into(), node_weight_kind: target_node_weight.into(), }; diff --git a/lib/dal/tests/integration_test/rebaser.rs b/lib/dal/tests/integration_test/rebaser.rs index ab9eafe9be..966f178a90 100644 --- a/lib/dal/tests/integration_test/rebaser.rs +++ b/lib/dal/tests/integration_test/rebaser.rs @@ -378,10 +378,12 @@ async fn correctly_detect_unrelated_unmodified_data(ctx: &mut DalContext) { ChangeSetTestHelpers::commit_and_update_snapshot_to_visibility(ctx) .await .expect("Unable to commit_and_update_snapshot_to_visibility"); + // Swifty is in HEAD ChangeSetTestHelpers::apply_change_set_to_base(ctx) .await .expect("Unable to merge to base change set"); + // Both change sets have Swifty in HEAD let change_set_a = ChangeSetTestHelpers::fork_from_head_change_set_with_name(ctx, "Change set A") .await @@ -407,6 +409,7 @@ async fn correctly_detect_unrelated_unmodified_data(ctx: &mut DalContext) { .copied() .expect("si.name attribute value not found") }; + // Swifty in Change Set A has a name of 'Modified in change set A' AttributeValue::update( ctx, cs_a_name_av_id, @@ -414,6 +417,7 @@ async fn correctly_detect_unrelated_unmodified_data(ctx: &mut DalContext) { ) .await .expect("Unable to update shared component name in change set A"); + // Change set A is committed to the rebaser ChangeSetTestHelpers::commit_and_update_snapshot_to_visibility(ctx) .await .expect("Unable to commit_and_update_snapshot_to_visibility for change set A"); diff --git a/lib/rebaser-server/src/change_set_requests/handlers.rs b/lib/rebaser-server/src/change_set_requests/handlers.rs index 80561d805c..a5c2fdea15 100644 --- a/lib/rebaser-server/src/change_set_requests/handlers.rs +++ b/lib/rebaser-server/src/change_set_requests/handlers.rs @@ -121,7 +121,7 @@ pub async fn process_request(State(state): State, msg: InnerMessage) - .finished( rebase_status.clone(), message.payload.to_rebase_change_set_id, - message.payload.onto_workspace_snapshot_address, + message.payload.rebase_batch_address, message.metadata.clone(), message.id, ) diff --git a/lib/rebaser-server/src/rebase.rs b/lib/rebaser-server/src/rebase.rs index a5473bee2a..ed45e1c3be 100644 --- a/lib/rebaser-server/src/rebase.rs +++ b/lib/rebaser-server/src/rebase.rs @@ -1,13 +1,9 @@ use dal::change_set::{ChangeSet, ChangeSetError, ChangeSetId}; -use dal::workspace_snapshot::conflict::Conflict; use dal::workspace_snapshot::graph::ConflictsAndUpdates; -use dal::workspace_snapshot::update::Update; -use dal::workspace_snapshot::vector_clock::{HasVectorClocks, VectorClockId}; -use dal::workspace_snapshot::{NodeId, NodeInformation, WorkspaceSnapshotError}; -use dal::{ - DalContext, EdgeWeight, EdgeWeightKindDiscriminants, TransactionsError, WorkspacePk, - WorkspaceSnapshot, WsEventError, -}; +use dal::workspace_snapshot::vector_clock::VectorClockId; +use dal::workspace_snapshot::WorkspaceSnapshotError; +use dal::{DalContext, TransactionsError, WorkspacePk, WorkspaceSnapshot, WsEventError}; +use si_events::rebase_batch_address::RebaseBatchAddress; use si_events::WorkspaceSnapshotAddress; use si_layer_cache::activities::rebase::RebaseStatus; use si_layer_cache::activities::ActivityRebaseRequest; @@ -25,6 +21,8 @@ pub enum RebaseError { LayerDb(#[from] LayerDbError), #[error("missing change set")] MissingChangeSet(ChangeSetId), + #[error("missing rebase batch {0}")] + MissingRebaseBatch(RebaseBatchAddress), #[error("to_rebase snapshot has no recently seen vector clock for its change set {0}")] MissingVectorClockForChangeSet(ChangeSetId), #[error("snapshot has no recently seen vector clock for any change set")] @@ -79,18 +77,22 @@ pub async fn perform_rebase( debug!("before snapshot fetch and parse: {:?}", start.elapsed()); let to_rebase_workspace_snapshot = WorkspaceSnapshot::find(ctx, to_rebase_workspace_snapshot_address).await?; - let onto_workspace_snapshot: WorkspaceSnapshot = - WorkspaceSnapshot::find(ctx, message.payload.onto_workspace_snapshot_address).await?; - info!( - "to_rebase_id: {}, onto_id: {}", - to_rebase_workspace_snapshot_address, - onto_workspace_snapshot.id().await + + let rebase_batch = ctx + .layer_db() + .rebase_batch() + .read_wait_for_memory(&message.payload.rebase_batch_address) + .await? + .ok_or(RebaseError::MissingRebaseBatch( + message.payload.rebase_batch_address, + ))?; + + debug!( + "to_rebase_address: {}, rebase_batch_address: {}", + to_rebase_workspace_snapshot_address, message.payload.rebase_batch_address ); debug!("after snapshot fetch and parse: {:?}", start.elapsed()); - // Perform the conflicts and updates detection. - //let onto_vector_clock_id: VectorClockId = message.payload.onto_vector_clock_id; - // Choose the most recent vector clock for the to_rebase change set for conflict detection let to_rebase_vector_clock_id = to_rebase_workspace_snapshot .max_recently_seen_clock_id(Some(to_rebase_change_set.id)) @@ -99,55 +101,20 @@ pub async fn perform_rebase( to_rebase_change_set.id, ))?; - let onto_vector_clock_id = onto_workspace_snapshot - .max_recently_seen_clock_id(None) - .await? - .ok_or(RebaseError::MissingVectorClockForChangeSet( - to_rebase_change_set.id, - ))?; - - let mut conflicts_and_updates = to_rebase_workspace_snapshot - .detect_conflicts_and_updates( - to_rebase_vector_clock_id, - &onto_workspace_snapshot, - onto_vector_clock_id, - ) - .await?; - - info!( - "count: conflicts ({}) and updates ({}), {:?}", - conflicts_and_updates.conflicts.len(), - conflicts_and_updates.updates.len(), - start.elapsed() - ); - - let len_before = conflicts_and_updates.conflicts.len(); - if !conflicts_and_updates.conflicts.is_empty() { - conflicts_and_updates = fix_prototype_race_conflicts( - conflicts_and_updates.clone(), - &to_rebase_workspace_snapshot, - ) - .await?; - } - - if conflicts_and_updates.conflicts.len() < len_before { - info!("automatically resolved prototype edge exclusive edge mismatch"); - } + let conflicts_and_updates = ConflictsAndUpdates { + ..Default::default() + }; // If there are conflicts, immediately assemble a reply message that conflicts were found. // Otherwise, we can perform updates and assemble a "success" reply message. let message: RebaseStatus = if conflicts_and_updates.conflicts.is_empty() { to_rebase_workspace_snapshot - .perform_updates( - to_rebase_vector_clock_id, - &onto_workspace_snapshot, - conflicts_and_updates.updates.as_slice(), - ) + .perform_updates(to_rebase_vector_clock_id, rebase_batch.updates()) .await?; - info!("updates complete: {:?}", start.elapsed()); + debug!("updates complete: {:?}", start.elapsed()); - if !conflicts_and_updates.updates.is_empty() { + if !rebase_batch.updates().is_empty() { // Once all updates have been performed, we can write out, mark everything as recently seen // and update the pointer. let workspace_pk = ctx.tenancy().workspace_pk().unwrap_or(WorkspacePk::NONE); @@ -163,18 +130,21 @@ pub async fn perform_rebase( to_rebase_workspace_snapshot .write(ctx, vector_clock_id) .await?; - info!("snapshot written: {:?}", start.elapsed()); + debug!("snapshot written: {:?}", start.elapsed()); to_rebase_change_set .update_pointer(ctx, to_rebase_workspace_snapshot.id().await) .await?; - info!("pointer updated: {:?}", start.elapsed()); + + debug!("pointer updated: {:?}", start.elapsed()); } - let updates_count = conflicts_and_updates.updates.len(); - let updates_performed = serde_json::to_value(conflicts_and_updates.updates)?.to_string(); + let updates_count = rebase_batch.updates().len(); + //let updates_performed = serde_json::to_value(rebase_batch.updates())?.to_string(); - span.record("si.updates", updates_performed.clone()); + //span.record("si.updates", updates_performed.clone()); span.record("si.updates.count", updates_count.to_string()); - RebaseStatus::Success { updates_performed } + RebaseStatus::Success { + updates_performed: message.payload.rebase_batch_address, + } } else { let conflicts_count = conflicts_and_updates.conflicts.len(); let conflicts_found = serde_json::to_value(conflicts_and_updates.conflicts)?.to_string(); @@ -205,11 +175,7 @@ pub async fn perform_rebase( { error!(?error, "Eviction error: {:?}", error); } - if let Err(error) = - evict_unused_snapshots(&ictx, &onto_workspace_snapshot.id().await).await - { - error!(?error, "Eviction error: {:?}", error); - } + // TODO: RebaseBatch eviction? }); } @@ -232,137 +198,3 @@ pub(crate) async fn evict_unused_snapshots( } Ok(()) } - -/// If the same user modifies two attributes in two+ components in quick -/// succession, they can race against themselves, producing a conflict where -/// an out of date attribute that was *not* changed in "onto" attempts to -/// stomp on the more up to date attribute in "to_rebase". We can fix this -/// by just removing the new edge update if there are no other updates for -/// the source id. -async fn fix_prototype_race_conflicts( - mut conflicts_and_updates: ConflictsAndUpdates, - to_rebase_snapshot: &WorkspaceSnapshot, -) -> RebaseResult { - let original_conflicts = conflicts_and_updates.conflicts.clone(); - for conflict in &original_conflicts { - match conflict { - Conflict::ExclusiveEdgeMismatch { - source, edge_kind, .. - } if edge_kind == &EdgeWeightKindDiscriminants::Prototype => { - let mut new_edge_updates = find_new_edge_updates_for_source( - &conflicts_and_updates.updates, - source.id, - *edge_kind, - ); - if new_edge_updates.len() != 1 { - // We can't resolve this one automatically because there - // is either no new edge update or more than one for - // this kind - continue; - } - let to_rebase_edge = to_rebase_snapshot - .edges_directed_for_edge_weight_kind( - source.id, - dal::workspace_snapshot::Direction::Outgoing, - *edge_kind, - ) - .await? - .pop() - .map(|(edge_weight, _, _)| edge_weight); - - if let (Some(to_rebase_edge), Some((_, _, onto_edge_weight))) = - (to_rebase_edge, new_edge_updates.pop()) - { - let to_rebase_clock = to_rebase_edge.vector_clock_write().max(None); - let onto_clock = onto_edge_weight.vector_clock_write().max(None); - if let ( - Some((to_rebase_clock_id, to_rebase_clock_stamp)), - Some((onto_clock_id, onto_clock_stamp)), - ) = (to_rebase_clock, onto_clock) - { - if to_rebase_clock_id == onto_clock_id - && to_rebase_clock_stamp > onto_clock_stamp - { - conflicts_and_updates = - remove_new_edge_update_and_conflict_for_source_if_safe( - conflicts_and_updates, - source.id, - *edge_kind, - ); - } - } - } - } - _ => {} - } - } - - Ok(conflicts_and_updates) -} - -// NOTE: This is only safe when used by the fix_prototype_race_conflicts function... -fn remove_new_edge_update_and_conflict_for_source_if_safe( - mut conflicts_and_updates: ConflictsAndUpdates, - source_id: NodeId, - new_edge_kind: EdgeWeightKindDiscriminants, -) -> ConflictsAndUpdates { - let is_it_safe = !conflicts_and_updates - .updates - .iter() - .any(|update| match update { - Update::NewEdge { - source, - edge_weight, - .. - } => source.id == source_id && new_edge_kind != edge_weight.kind().into(), - Update::RemoveEdge { source, .. } => source.id == source_id, - Update::ReplaceSubgraph { onto, .. } => onto.id == source_id, - _ => false, - }); - - if !is_it_safe { - return conflicts_and_updates; - } - - conflicts_and_updates.updates.retain(|update| match update { - Update::NewEdge { - source, - edge_weight, - .. - } if source.id == source_id => new_edge_kind != edge_weight.kind().into(), - _ => true, - }); - - conflicts_and_updates - .conflicts - .retain(|conflict| match conflict { - Conflict::ExclusiveEdgeMismatch { - source, edge_kind, .. - } if source.id == source_id => edge_kind != &new_edge_kind, - _ => true, - }); - - conflicts_and_updates -} - -fn find_new_edge_updates_for_source( - updates: &[Update], - source_id: NodeId, - kind: EdgeWeightKindDiscriminants, -) -> Vec<(NodeInformation, NodeInformation, EdgeWeight)> { - updates - .iter() - .filter_map(|update| match update { - Update::NewEdge { - source, - destination, - edge_weight, - } if source.id == source_id && kind == edge_weight.kind().into() => Some(( - source.to_owned(), - destination.to_owned(), - edge_weight.to_owned(), - )), - _ => None, - }) - .collect() -} diff --git a/lib/sdf-server/src/server/server.rs b/lib/sdf-server/src/server/server.rs index 5ac88a3913..7c392bb1f3 100644 --- a/lib/sdf-server/src/server/server.rs +++ b/lib/sdf-server/src/server/server.rs @@ -374,6 +374,7 @@ pub async fn migrate_builtins_from_module_index(services_context: &ServicesConte let module_index_client = ModuleIndexClient::unauthenticated_client(module_index_url.try_into()?); let module_list = module_index_client.list_builtins().await?; + info!("builtins install starting"); let install_builtins = install_builtins(ctx, module_list, module_index_client); tokio::pin!(install_builtins); loop { @@ -402,6 +403,9 @@ async fn install_builtins( let dal = &ctx; let client = &module_index_client.clone(); let modules: Vec = module_list.modules; + // .into_iter() + // .filter(|module| module.name.contains("docker-image")) + // .collect(); let total = modules.len(); diff --git a/lib/sdf-server/src/server/service/change_set/rebase_on_base.rs b/lib/sdf-server/src/server/service/change_set/rebase_on_base.rs index f55f9768cf..605c31e427 100644 --- a/lib/sdf-server/src/server/service/change_set/rebase_on_base.rs +++ b/lib/sdf-server/src/server/service/change_set/rebase_on_base.rs @@ -1,8 +1,9 @@ +use std::sync::Arc; + use axum::{extract::OriginalUri, Json}; use serde::{Deserialize, Serialize}; -use dal::{context::RebaseRequest, ChangeSet, Ulid, Visibility, WsEvent}; -use si_events::VectorClockId; +use dal::{context::RebaseRequest, ChangeSet, Visibility, WorkspaceSnapshot, WsEvent}; use super::ChangeSetResult; use crate::server::{ @@ -30,7 +31,7 @@ pub async fn rebase_on_base( OriginalUri(original_uri): OriginalUri, Json(request): Json, ) -> ChangeSetResult> { - let ctx = builder.build(request_ctx.build(request.visibility)).await?; + let ctx: dal::DalContext = builder.build(request_ctx.build(request.visibility)).await?; let change_set = ChangeSet::find(&ctx, request.visibility.change_set_id) .await? @@ -42,23 +43,21 @@ pub async fn rebase_on_base( .await? .ok_or(dal::ChangeSetError::ChangeSetNotFound(base_change_set_id))? } else { - return Err(dal::ChangeSetError::NoBaseChangeSet(request.visibility.change_set_id).into()); - }; - let base_snapshot_address = base_change_set - .workspace_snapshot_address - .ok_or(dal::ChangeSetError::NoWorkspaceSnapshot(base_change_set.id))?; - - // TODO: Check for affected AttributeValues, and enqueue DVU for them. - - let rebase_request = RebaseRequest { - to_rebase_change_set_id: request.visibility.change_set_id, - onto_workspace_snapshot_address: base_snapshot_address, - // Doesn't really matter, as this field is deprecated since we automatically - // figure it out in the rebaser. - onto_vector_clock_id: VectorClockId::new(Ulid::new(), Ulid::new()), + return Err(dal::ChangeSetError::NoBaseChangeSet(ctx.change_set_id()).into()); }; - ctx.do_rebase_request(rebase_request).await?; + let base_snapshot = WorkspaceSnapshot::find_for_change_set(&ctx, base_change_set.id).await?; + if let Some(rebase_batch) = WorkspaceSnapshot::calculate_rebase_batch( + ctx.change_set_id(), + ctx.workspace_snapshot()?, + Arc::new(base_snapshot), + ) + .await? + { + let rebase_batch_address = ctx.write_rebase_batch(rebase_batch).await?; + let rebase_request = RebaseRequest::new(ctx.change_set_id(), rebase_batch_address); + ctx.do_rebase_request(rebase_request).await?; + } let user = ChangeSet::extract_userid_from_context(&ctx).await; // There is no commit, and the rebase request has already gone through & succeeded, so send out diff --git a/lib/si-events-rs/src/lib.rs b/lib/si-events-rs/src/lib.rs index 537db9b972..8e58465d91 100644 --- a/lib/si-events-rs/src/lib.rs +++ b/lib/si-events-rs/src/lib.rs @@ -1,6 +1,7 @@ pub mod content_hash; pub mod encrypted_secret; pub mod merkle_tree_hash; +pub mod rebase_batch_address; pub mod ulid; pub mod workspace_snapshot_address; pub mod xxhash_type; diff --git a/lib/si-events-rs/src/rebase_batch_address.rs b/lib/si-events-rs/src/rebase_batch_address.rs new file mode 100644 index 0000000000..99b0c4f224 --- /dev/null +++ b/lib/si-events-rs/src/rebase_batch_address.rs @@ -0,0 +1,121 @@ +use bytes::BytesMut; +use postgres_types::ToSql; +use serde::{ + de::{self, Visitor}, + Deserialize, Serialize, +}; +use std::{fmt, str::FromStr}; +use thiserror::Error; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub struct RebaseBatchAddress(blake3::Hash); + +impl RebaseBatchAddress { + #[must_use] + pub fn new(input: &[u8]) -> Self { + Self(blake3::hash(input)) + } + + pub fn nil() -> Self { + Self(blake3::Hash::from_bytes([0; 32])) + } +} + +#[derive(Debug, Error)] +#[error("failed to parse hash hex string")] +pub struct RebaseBatchAddressParseError(#[from] blake3::HexError); + +impl FromStr for RebaseBatchAddress { + type Err = RebaseBatchAddressParseError; + + fn from_str(s: &str) -> Result { + Ok(Self(blake3::Hash::from_str(s)?)) + } +} + +impl std::fmt::Display for RebaseBatchAddress { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl Serialize for RebaseBatchAddress { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +struct RebaseBatchAddressVisitor; + +impl<'de> Visitor<'de> for RebaseBatchAddressVisitor { + type Value = RebaseBatchAddress; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a blake3 hash string") + } + + fn visit_str(self, v: &str) -> Result + where + E: de::Error, + { + RebaseBatchAddress::from_str(v).map_err(|e| E::custom(e.to_string())) + } +} + +impl<'de> Deserialize<'de> for RebaseBatchAddress { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_str(RebaseBatchAddressVisitor) + } +} + +impl ToSql for RebaseBatchAddress { + fn to_sql( + &self, + ty: &postgres_types::Type, + out: &mut BytesMut, + ) -> Result> + where + Self: Sized, + { + let self_string = self.to_string(); + + self_string.to_sql(ty, out) + } + + fn accepts(ty: &postgres_types::Type) -> bool + where + Self: Sized, + { + String::accepts(ty) + } + + fn to_sql_checked( + &self, + ty: &postgres_types::Type, + out: &mut BytesMut, + ) -> Result> { + let self_string = self.to_string(); + self_string.to_sql_checked(ty, out) + } +} + +impl<'a> postgres_types::FromSql<'a> for RebaseBatchAddress { + fn from_sql( + ty: &postgres_types::Type, + raw: &'a [u8], + ) -> Result> { + let hash_string: String = postgres_types::FromSql::from_sql(ty, raw)?; + Ok(Self(blake3::Hash::from_str(&hash_string)?)) + } + + fn accepts(ty: &postgres_types::Type) -> bool { + ty == &postgres_types::Type::TEXT + || ty.kind() == &postgres_types::Kind::Domain(postgres_types::Type::TEXT) + } +} diff --git a/lib/si-layer-cache/src/activities/rebase.rs b/lib/si-layer-cache/src/activities/rebase.rs index d2ef087555..83a6f03f26 100644 --- a/lib/si-layer-cache/src/activities/rebase.rs +++ b/lib/si-layer-cache/src/activities/rebase.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; +use si_events::rebase_batch_address::RebaseBatchAddress; use strum::EnumDiscriminants; -use si_events::{VectorClockId, WorkspaceSnapshotAddress}; use telemetry::prelude::*; use telemetry::tracing::instrument; use tokio::sync::mpsc::UnboundedReceiver; @@ -20,27 +20,17 @@ pub struct RebaseRequest { pub to_rebase_change_set_id: Ulid, /// Corresponds to the workspace snapshot that will be the "onto" workspace snapshot when /// rebasing the "to rebase" workspace snapshot. - pub onto_workspace_snapshot_address: WorkspaceSnapshotAddress, - /// *DEPRECATED*: We no longer have "edit sessions", the correct vector - /// clock to choose for the onto workspace is always the most up to date - /// "recently seen" clock in the root node of the onto snapshot. This field - /// is ignored in the request. - pub onto_vector_clock_id: VectorClockId, - /// DEPRECATED: We have to hang on to this to ensure we can deserialize this message - pub dvu_values: Option>, + pub rebase_batch_address: RebaseBatchAddress, } impl RebaseRequest { pub fn new( to_rebase_change_set_id: Ulid, - onto_workspace_snapshot_address: WorkspaceSnapshotAddress, - onto_vector_clock_id: VectorClockId, + rebase_batch_address: RebaseBatchAddress, ) -> RebaseRequest { RebaseRequest { to_rebase_change_set_id, - onto_workspace_snapshot_address, - onto_vector_clock_id, - dvu_values: None, + rebase_batch_address, } } } @@ -49,19 +39,19 @@ impl RebaseRequest { pub struct RebaseFinished { status: RebaseStatus, to_rebase_change_set_id: Ulid, - onto_workspace_snapshot_address: WorkspaceSnapshotAddress, + rebase_batch_address: RebaseBatchAddress, } impl RebaseFinished { pub fn new( status: RebaseStatus, to_rebase_change_set_id: Ulid, - onto_workspace_snapshot_address: WorkspaceSnapshotAddress, + rebase_batch_address: RebaseBatchAddress, ) -> RebaseFinished { RebaseFinished { status, to_rebase_change_set_id, - onto_workspace_snapshot_address, + rebase_batch_address, } } @@ -73,8 +63,8 @@ impl RebaseFinished { &self.to_rebase_change_set_id } - pub fn onto_workspace_snapshot_address(&self) -> &WorkspaceSnapshotAddress { - &self.onto_workspace_snapshot_address + pub fn rebase_batch_address(&self) -> &RebaseBatchAddress { + &self.rebase_batch_address } } @@ -86,7 +76,7 @@ pub enum RebaseStatus { /// Processing the request and performing updates were both successful. Additionally, no conflicts were found. Success { /// The serialized updates performed when rebasing. - updates_performed: String, + updates_performed: RebaseBatchAddress, }, /// Conflicts found when processing the request. ConflictsFound { @@ -117,15 +107,10 @@ impl<'a> ActivityRebase<'a> { pub async fn rebase( &self, to_rebase_change_set_id: Ulid, - onto_workspace_snapshot_address: WorkspaceSnapshotAddress, - onto_vector_clock_id: VectorClockId, + rebase_batch_address: RebaseBatchAddress, metadata: LayeredEventMetadata, ) -> LayerDbResult { - let payload = RebaseRequest::new( - to_rebase_change_set_id, - onto_workspace_snapshot_address, - onto_vector_clock_id, - ); + let payload = RebaseRequest::new(to_rebase_change_set_id, rebase_batch_address); let activity = Activity::rebase(payload, metadata); self.activity_base.publish(&activity).await?; Ok(activity) @@ -135,15 +120,10 @@ impl<'a> ActivityRebase<'a> { pub async fn rebase_and_wait( &self, to_rebase_change_set_id: Ulid, - onto_workspace_snapshot_address: WorkspaceSnapshotAddress, - onto_vector_clock_id: VectorClockId, + rebase_batch_address: RebaseBatchAddress, metadata: LayeredEventMetadata, ) -> LayerDbResult { - let payload = RebaseRequest::new( - to_rebase_change_set_id, - onto_workspace_snapshot_address, - onto_vector_clock_id, - ); + let payload = RebaseRequest::new(to_rebase_change_set_id, rebase_batch_address); let activity = Activity::rebase(payload, metadata); // println!("trigger: sending rebase and waiting for response"); debug!(?activity, "sending rebase and waiting for response"); @@ -182,15 +162,11 @@ impl<'a> ActivityRebase<'a> { &self, status: RebaseStatus, to_rebase_change_set_id: Ulid, - onto_workspace_snapshot_address: WorkspaceSnapshotAddress, + rebase_batch_address: RebaseBatchAddress, metadata: LayeredEventMetadata, parent_activity_id: ActivityId, ) -> LayerDbResult { - let payload = RebaseFinished::new( - status, - to_rebase_change_set_id, - onto_workspace_snapshot_address, - ); + let payload = RebaseFinished::new(status, to_rebase_change_set_id, rebase_batch_address); let activity = Activity::rebase_finished(payload, metadata, parent_activity_id); self.activity_base.publish(&activity).await?; Ok(activity) diff --git a/lib/si-layer-cache/src/db.rs b/lib/si-layer-cache/src/db.rs index 978f9cb35d..8a128b601a 100644 --- a/lib/si-layer-cache/src/db.rs +++ b/lib/si-layer-cache/src/db.rs @@ -23,27 +23,33 @@ use crate::{ persister::{PersisterClient, PersisterTask}, }; -use self::{cache_updates::CacheUpdatesTask, cas::CasDb, workspace_snapshot::WorkspaceSnapshotDb}; +use self::{ + cache_updates::CacheUpdatesTask, cas::CasDb, rebase_batch::RebaseBatchDb, + workspace_snapshot::WorkspaceSnapshotDb, +}; mod cache_updates; pub mod cas; pub mod encrypted_secret; pub mod func_run; pub mod func_run_log; +pub mod rebase_batch; pub mod serialize; pub mod workspace_snapshot; #[derive(Debug, Clone)] -pub struct LayerDb +pub struct LayerDb where CasValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, EncryptedSecretValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, WorkspaceSnapshotValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, + RebaseBatchValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, { cas: CasDb, encrypted_secret: EncryptedSecretDb, func_run: FuncRunDb, func_run_log: FuncRunLogDb, + rebase_batch: RebaseBatchDb, workspace_snapshot: WorkspaceSnapshotDb, pg_pool: PgPool, nats_client: NatsClient, @@ -52,12 +58,13 @@ where instance_id: Ulid, } -impl - LayerDb +impl + LayerDb where CasValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, EncryptedSecretValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, WorkspaceSnapshotValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, + RebaseBatchValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, { #[instrument(name = "layer_db.init.from_config", level = "info", skip_all)] pub async fn from_config( @@ -122,6 +129,13 @@ where memory_cache_config.clone(), )?; + let rebase_batch_cache: LayerCache> = LayerCache::new( + rebase_batch::CACHE_NAME, + disk_path, + pg_pool.clone(), + memory_cache_config.clone(), + )?; + let snapshot_cache: LayerCache> = LayerCache::new( workspace_snapshot::CACHE_NAME, disk_path, @@ -136,6 +150,7 @@ where encrypted_secret_cache.clone(), func_run_cache.clone(), func_run_log_cache.clone(), + rebase_batch_cache.clone(), snapshot_cache.clone(), token.clone(), ) @@ -159,6 +174,7 @@ where let func_run = FuncRunDb::new(func_run_cache, persister_client.clone()); let func_run_log = FuncRunLogDb::new(func_run_log_cache, persister_client.clone()); let workspace_snapshot = WorkspaceSnapshotDb::new(snapshot_cache, persister_client.clone()); + let rebase_batch = RebaseBatchDb::new(rebase_batch_cache, persister_client.clone()); let activity = ActivityClient::new(instance_id, nats_client.clone(), token.clone()); let graceful_shutdown = LayerDbGracefulShutdown { tracker, token }; @@ -174,6 +190,7 @@ where persister_client, nats_client, instance_id, + rebase_batch, }; Ok((layerdb, graceful_shutdown)) @@ -207,6 +224,10 @@ where &self.func_run_log } + pub fn rebase_batch(&self) -> &RebaseBatchDb { + &self.rebase_batch + } + pub fn workspace_snapshot(&self) -> &WorkspaceSnapshotDb { &self.workspace_snapshot } diff --git a/lib/si-layer-cache/src/db/cache_updates.rs b/lib/si-layer-cache/src/db/cache_updates.rs index 612bedaa92..5a7c401a0c 100644 --- a/lib/si-layer-cache/src/db/cache_updates.rs +++ b/lib/si-layer-cache/src/db/cache_updates.rs @@ -26,28 +26,35 @@ enum CacheName { WorkspaceSnapshots, } -pub struct CacheUpdatesTask -where +pub struct CacheUpdatesTask< + CasValue, + EncryptedSecretValue, + WorkspaceSnapshotValue, + RebaseBatchValue, +> where CasValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, EncryptedSecretValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, WorkspaceSnapshotValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, + RebaseBatchValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, { cas_cache: LayerCache>, encrypted_secret_cache: LayerCache>, func_run_cache: LayerCache>, func_run_log_cache: LayerCache>, + rebase_batch_cache: LayerCache>, snapshot_cache: LayerCache>, event_channel: UnboundedReceiver, shutdown_token: CancellationToken, tracker: TaskTracker, } -impl - CacheUpdatesTask +impl + CacheUpdatesTask where CasValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, EncryptedSecretValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, WorkspaceSnapshotValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, + RebaseBatchValue: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, { const NAME: &'static str = "LayerDB::CacheUpdatesTask"; @@ -59,6 +66,7 @@ where encrypted_secret_cache: LayerCache>, func_run_cache: LayerCache>, func_run_log_cache: LayerCache>, + rebase_batch_cache: LayerCache>, snapshot_cache: LayerCache>, shutdown_token: CancellationToken, ) -> LayerDbResult { @@ -74,6 +82,7 @@ where encrypted_secret_cache, func_run_cache, func_run_log_cache, + rebase_batch_cache, snapshot_cache, event_channel, shutdown_token, @@ -103,6 +112,7 @@ where self.func_run_cache.clone(), self.func_run_log_cache.clone(), self.snapshot_cache.clone(), + self.rebase_batch_cache.clone(), ); self.tracker .spawn(async move { cache_update_task.run(event).await }); @@ -110,24 +120,27 @@ where } } -struct CacheUpdateTask +struct CacheUpdateTask where Q: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, R: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, S: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, + T: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, { cas_cache: LayerCache>, encrypted_secret_cache: LayerCache>, func_run_cache: LayerCache>, func_run_log_cache: LayerCache>, snapshot_cache: LayerCache>, + rebase_batch_cache: LayerCache>, } -impl CacheUpdateTask +impl CacheUpdateTask where Q: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, R: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, S: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, + T: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, { fn new( cas_cache: LayerCache>, @@ -135,13 +148,15 @@ where func_run_cache: LayerCache>, func_run_log_cache: LayerCache>, snapshot_cache: LayerCache>, - ) -> CacheUpdateTask { + rebase_batch_cache: LayerCache>, + ) -> CacheUpdateTask { CacheUpdateTask { cas_cache, encrypted_secret_cache, func_run_cache, func_run_log_cache, snapshot_cache, + rebase_batch_cache, } } @@ -174,6 +189,25 @@ where crate::event::LayeredEventKind::Raw => { warn!("Recevied a 'raw' layered event kind - this is for testing only. Bug!"); } + + crate::event::LayeredEventKind::RebaseBatchWrite => { + if !self.rebase_batch_cache.contains(&event.key) { + let memory_value = self + .rebase_batch_cache + .deserialize_memory_value(&event.payload.value)?; + let serialized_value = + Arc::try_unwrap(event.payload.value).unwrap_or_else(|arc| (*arc).clone()); + self.rebase_batch_cache + .insert_from_cache_updates(event.key, memory_value, serialized_value) + .await?; + } + } + crate::event::LayeredEventKind::RebaseBatchEvict => { + self.rebase_batch_cache + .evict_from_cache_updates(event.key) + .await?; + } + crate::event::LayeredEventKind::SnapshotWrite => { if !self.snapshot_cache.contains(&event.key) { let memory_value = self @@ -212,6 +246,7 @@ where .await?; } } + Ok(()) } diff --git a/lib/si-layer-cache/src/db/rebase_batch.rs b/lib/si-layer-cache/src/db/rebase_batch.rs new file mode 100644 index 0000000000..2dfdc390ec --- /dev/null +++ b/lib/si-layer-cache/src/db/rebase_batch.rs @@ -0,0 +1,170 @@ +use std::{sync::Arc, time::Instant}; + +use serde::{de::DeserializeOwned, Serialize}; +use si_events::{rebase_batch_address::RebaseBatchAddress, Actor, Tenancy, WebEvent}; +use telemetry::prelude::*; + +use crate::{ + error::LayerDbResult, + event::{LayeredEvent, LayeredEventKind}, + layer_cache::LayerCache, + persister::{PersisterClient, PersisterStatusReader}, +}; + +use super::serialize; + +pub const DBNAME: &str = "rebase_batches"; +pub const CACHE_NAME: &str = "rebase_batches"; +pub const PARTITION_KEY: &str = "rebase_batches"; + +#[derive(Debug, Clone)] +pub struct RebaseBatchDb +where + V: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, +{ + pub cache: LayerCache>, + persister_client: PersisterClient, +} + +impl RebaseBatchDb +where + V: Serialize + DeserializeOwned + Clone + Send + Sync + 'static, +{ + pub fn new(cache: LayerCache>, persister_client: PersisterClient) -> Self { + Self { + cache, + persister_client, + } + } + + pub async fn write( + &self, + value: Arc, + web_events: Option>, + tenancy: Tenancy, + actor: Actor, + ) -> LayerDbResult<(RebaseBatchAddress, PersisterStatusReader)> { + let value_clone = value.clone(); + let postcard_value = serialize::to_vec(&value)?; + + let key = RebaseBatchAddress::new(&postcard_value); + let cache_key: Arc = key.to_string().into(); + + self.cache.insert(cache_key.clone(), value_clone).await; + + let event = LayeredEvent::new( + LayeredEventKind::RebaseBatchWrite, + Arc::new(DBNAME.to_string()), + cache_key, + Arc::new(postcard_value), + Arc::new("rebase_batches".to_string()), + web_events, + tenancy, + actor, + ); + let reader = self.persister_client.write_event(event)?; + + Ok((key, reader)) + } + + #[instrument( + name = "rebase_batch.read", + level = "debug", + skip_all, + fields( + si.rebase_batch.address = %key, + ) + )] + pub async fn read(&self, key: &RebaseBatchAddress) -> LayerDbResult>> { + self.cache.get(key.to_string().into()).await + } + + #[instrument( + name = "rebase_batch.read_wait_for_memory", + level = "debug", + skip_all, + fields( + si.layer_cache.memory_cache.hit = Empty, + si.layer_cache.memory_cache.read_wait_ms = Empty, + si.layer_cache.memory_cache.retries = Empty, + si.rebase_batch.address = %key, + ) + )] + pub async fn read_wait_for_memory( + &self, + key: &RebaseBatchAddress, + ) -> LayerDbResult>> { + let span = Span::current(); + + let key: Arc = key.to_string().into(); + const MAX_TRIES: i32 = 2000; + let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(1)); + let mut tried = 0; + let read_wait = Instant::now(); + while tried < MAX_TRIES { + if let Some(v) = self.cache.memory_cache().get(&key).await { + span.record("si.layer_cache.memory_cache.hit", true); + span.record( + "si.layer_cache.memory_cache.read_wait_ms", + read_wait.elapsed().as_millis(), + ); + span.record("si.layer_cache.memory_cache.retries", tried); + return Ok(Some(v)); + } + tried += 1; + interval.tick().await; + } + + span.record("si.layer_cache.memory_cache.hit", false); + self.cache.get(key.to_string().into()).await + } + + #[instrument( + name = "rebase_batch.evict", + level = "debug", + skip_all, + fields( + si.rebase_batch.address = %key, + ) + )] + pub async fn evict( + &self, + key: &RebaseBatchAddress, + tenancy: Tenancy, + actor: Actor, + ) -> LayerDbResult { + let cache_key = key.to_string(); + self.cache.remove_from_memory(&cache_key).await; + + let event = LayeredEvent::new( + LayeredEventKind::RebaseBatchEvict, + Arc::new(DBNAME.to_string()), + cache_key.into(), + Arc::new(Vec::new()), + Arc::new("rebase_batch".to_string()), + None, + tenancy, + actor, + ); + let reader = self.persister_client.evict_event(event)?; + + Ok(reader) + } + + #[instrument( + name = "rebase_batch.read_bytes_from_durable_storage", + level = "debug", + skip_all, + fields( + si.rebase_batch.address = %key, + ) + )] + pub async fn read_bytes_from_durable_storage( + &self, + key: &RebaseBatchAddress, + ) -> LayerDbResult>> { + self.cache + .get_bytes_from_durable_storage(key.to_string().into()) + .await + } +} diff --git a/lib/si-layer-cache/src/event.rs b/lib/si-layer-cache/src/event.rs index cfd5c7a7bb..fc1bb81236 100644 --- a/lib/si-layer-cache/src/event.rs +++ b/lib/si-layer-cache/src/event.rs @@ -97,6 +97,8 @@ pub enum LayeredEventKind { FuncRunLogWrite, FuncRunWrite, Raw, + RebaseBatchEvict, + RebaseBatchWrite, SnapshotEvict, SnapshotWrite, } diff --git a/lib/si-layer-cache/src/migrations/U0008__rebase_batches.sql b/lib/si-layer-cache/src/migrations/U0008__rebase_batches.sql new file mode 100644 index 0000000000..34295e7e93 --- /dev/null +++ b/lib/si-layer-cache/src/migrations/U0008__rebase_batches.sql @@ -0,0 +1,10 @@ +CREATE TABLE rebase_batches +( + key text NOT NULL PRIMARY KEY, + sort_key text NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT CLOCK_TIMESTAMP(), + value bytea NOT NULL, + serialization_lib text NOT NULL DEFAULT 'postcard' +); + +CREATE INDEX IF NOT EXISTS rebase_batches_sort_key ON rebase_batches (sort_key); diff --git a/lib/si-layer-cache/src/persister.rs b/lib/si-layer-cache/src/persister.rs index a8632c587b..3493150dd9 100644 --- a/lib/si-layer-cache/src/persister.rs +++ b/lib/si-layer-cache/src/persister.rs @@ -421,6 +421,8 @@ impl PersistEventTask { LayeredEventKind::CasInsertion | LayeredEventKind::EncryptedSecretInsertion | LayeredEventKind::Raw + | LayeredEventKind::RebaseBatchEvict + | LayeredEventKind::RebaseBatchWrite | LayeredEventKind::SnapshotEvict | LayeredEventKind::SnapshotWrite => { pg_layer diff --git a/lib/si-layer-cache/tests/integration_test/activities.rs b/lib/si-layer-cache/tests/integration_test/activities.rs index 06ee36bec2..84536f9c5f 100644 --- a/lib/si-layer-cache/tests/integration_test/activities.rs +++ b/lib/si-layer-cache/tests/integration_test/activities.rs @@ -12,7 +12,7 @@ use tokio_util::sync::CancellationToken; use crate::integration_test::{disk_cache_path, setup_nats_client, setup_pg_db}; -type TestLayerDb = LayerDb, Arc, String>; +type TestLayerDb = LayerDb, Arc, String, String>; #[tokio::test] async fn activities() { diff --git a/lib/si-layer-cache/tests/integration_test/activities/rebase.rs b/lib/si-layer-cache/tests/integration_test/activities/rebase.rs index 58d0fd92f6..6b601f6409 100644 --- a/lib/si-layer-cache/tests/integration_test/activities/rebase.rs +++ b/lib/si-layer-cache/tests/integration_test/activities/rebase.rs @@ -4,8 +4,7 @@ use std::sync::{ }; use si_events::{ - Actor, ChangeSetId, Tenancy, VectorClockActorId, VectorClockChangeSetId, VectorClockId, - WorkspacePk, WorkspaceSnapshotAddress, + rebase_batch_address::RebaseBatchAddress, Actor, ChangeSetId, Tenancy, WorkspacePk, }; use si_layer_cache::{ activities::ActivityId, event::LayeredEventMetadata, memory_cache::MemoryCacheConfig, LayerDb, @@ -15,7 +14,7 @@ use ulid::Ulid; use crate::integration_test::{disk_cache_path, setup_nats_client, setup_pg_db}; -type TestLayerDb = LayerDb, Arc, String>; +type TestLayerDb = LayerDb, Arc, String, String>; #[tokio::test] async fn subscribe_rebaser_requests_work_queue() { @@ -81,16 +80,13 @@ async fn subscribe_rebaser_requests_work_queue() { let tenancy = Tenancy::new(WorkspacePk::new(), ChangeSetId::new()); let actor = Actor::System; let metadata = LayeredEventMetadata::new(tenancy, actor); - let actor_id: Ulid = tenancy.workspace_pk.into(); - let vector_clock_id = VectorClockId::new(Ulid::new(), actor_id); let rebase_request_activity = ldb_duff .activity() .rebase() .rebase( Ulid::new(), - WorkspaceSnapshotAddress::new(b"poop"), - vector_clock_id, + RebaseBatchAddress::new(b"poop"), metadata.clone(), ) .await @@ -105,7 +101,7 @@ async fn subscribe_rebaser_requests_work_queue() { message: "poop".to_string(), }, Ulid::new(), - WorkspaceSnapshotAddress::new(b"skid row"), + RebaseBatchAddress::new(b"skid row"), metadata, ActivityId::new(), ) @@ -198,19 +194,13 @@ async fn rebase_and_wait() { let metadata = LayeredEventMetadata::new(tenancy, actor); let metadata_for_task = metadata.clone(); - let onto_vector_clock_id = VectorClockId::new( - VectorClockChangeSetId::new(Ulid::new().into()), - VectorClockActorId::new(Ulid::new().into()), - ); - let rebase_request_task = tokio::spawn(async move { ldb_slash .activity() .rebase() .rebase_and_wait( Ulid::new(), - WorkspaceSnapshotAddress::new(b"poop"), - onto_vector_clock_id, + RebaseBatchAddress::new(b"poop"), metadata_for_task, ) .await @@ -230,7 +220,7 @@ async fn rebase_and_wait() { message: "poop".to_string(), }, Ulid::new(), - WorkspaceSnapshotAddress::new(b"skid row"), + RebaseBatchAddress::new(b"skid row"), metadata, rebase_request.id, ) @@ -340,17 +330,12 @@ async fn rebase_requests_work_queue_stress() { tracker.spawn(async move { let mut count = 0; while count < rebase_activities { - let vector_clock_id = VectorClockId::new( - VectorClockChangeSetId::new(Ulid::new().into()), - VectorClockActorId::new(Ulid::new().into()), - ); let _rebase_request_activity = ldb_duff .activity() .rebase() .rebase( Ulid::new(), - WorkspaceSnapshotAddress::new(b"poop"), - vector_clock_id, + RebaseBatchAddress::new(b"poop"), send_meta.clone(), ) .await @@ -477,7 +462,7 @@ async fn rebase_and_wait_stress() { message: "poop".to_string(), }, Ulid::new(), - WorkspaceSnapshotAddress::new(b"skid row"), + RebaseBatchAddress::new(b"skid row"), mp, rebase_request.id, ) @@ -499,19 +484,10 @@ async fn rebase_and_wait_stress() { loop { SENT_REQUEST_COUNTER.fetch_add(1, Ordering::Relaxed); let mp = metadata_for_sender.clone(); - let onto_vector_clock_id = VectorClockId::new( - VectorClockChangeSetId::new(Ulid::new().into()), - VectorClockActorId::new(Ulid::new().into()), - ); let _response = ldb_slash_clone .activity() .rebase() - .rebase_and_wait( - Ulid::new(), - WorkspaceSnapshotAddress::new(b"poop"), - onto_vector_clock_id, - mp, - ) + .rebase_and_wait(Ulid::new(), RebaseBatchAddress::new(b"poop"), mp) .await; RECV_REPLY_COUNTER.fetch_add(1, Ordering::Relaxed); } diff --git a/lib/si-layer-cache/tests/integration_test/db/cas.rs b/lib/si-layer-cache/tests/integration_test/db/cas.rs index 672932848c..0f62a9c55f 100644 --- a/lib/si-layer-cache/tests/integration_test/db/cas.rs +++ b/lib/si-layer-cache/tests/integration_test/db/cas.rs @@ -9,7 +9,7 @@ use tokio_util::sync::CancellationToken; use crate::integration_test::{disk_cache_path, setup_nats_client, setup_pg_db}; -type TestLayerDb = LayerDb; +type TestLayerDb = LayerDb; #[tokio::test] async fn write_to_db() { diff --git a/lib/si-layer-cache/tests/integration_test/db/func_run.rs b/lib/si-layer-cache/tests/integration_test/db/func_run.rs index 60676dfe0e..15f305c07f 100644 --- a/lib/si-layer-cache/tests/integration_test/db/func_run.rs +++ b/lib/si-layer-cache/tests/integration_test/db/func_run.rs @@ -14,7 +14,7 @@ use tokio_util::sync::CancellationToken; use crate::integration_test::{disk_cache_path, setup_nats_client, setup_pg_db}; -type TestLayerDb = LayerDb; +type TestLayerDb = LayerDb; #[tokio::test] async fn write_to_db() { diff --git a/lib/si-layer-cache/tests/integration_test/db/func_run_log.rs b/lib/si-layer-cache/tests/integration_test/db/func_run_log.rs index 0e2d8c4fce..4d35618829 100644 --- a/lib/si-layer-cache/tests/integration_test/db/func_run_log.rs +++ b/lib/si-layer-cache/tests/integration_test/db/func_run_log.rs @@ -11,7 +11,7 @@ use tokio_util::sync::CancellationToken; use crate::integration_test::{disk_cache_path, setup_nats_client, setup_pg_db}; -type TestLayerDb = LayerDb; +type TestLayerDb = LayerDb; #[tokio::test] async fn write_to_db() { diff --git a/lib/si-layer-cache/tests/integration_test/db/workspace_snapshot.rs b/lib/si-layer-cache/tests/integration_test/db/workspace_snapshot.rs index a9ffe581f7..8b9ddb3462 100644 --- a/lib/si-layer-cache/tests/integration_test/db/workspace_snapshot.rs +++ b/lib/si-layer-cache/tests/integration_test/db/workspace_snapshot.rs @@ -9,7 +9,7 @@ use tokio_util::sync::CancellationToken; use crate::integration_test::{disk_cache_path, setup_nats_client, setup_pg_db}; -type TestLayerDb = LayerDb; +type TestLayerDb = LayerDb; #[tokio::test] async fn write_to_db() { diff --git a/lib/veritech-client/src/lib.rs b/lib/veritech-client/src/lib.rs index eabb013703..0b6d2827ac 100644 --- a/lib/veritech-client/src/lib.rs +++ b/lib/veritech-client/src/lib.rs @@ -306,7 +306,7 @@ impl Client { loop { tokio::select! { _ = keep_alive_subscriber.next() => { - info!("Heartbeat from veritech"); + debug!("Heartbeat from veritech"); continue; } // Abort if no keep-alive for too long