fix: checkin improvements

- conversion impls in tangram_client
- defer writing graph objects to output stage of checkin (#436)
- write all objects in checkin in single message to SQLite (#437)

packages/client/src/graph/data.rs

@@ -11,7 +11,7 @@ pub struct Graph {
 	pub nodes: Vec<tg::graph::data::Node>,
 }
 
-#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
+#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, derive_more::TryUnwrap)]
 #[serde(tag = "kind", rename_all = "snake_case")]
 pub enum Node {
 	Directory(Directory),

packages/client/src/object/data.rs

@@ -65,3 +65,13 @@ impl Object {
 		}
 	}
 }
+
+impl From<tg::artifact::Data> for Object {
+	fn from(value: tg::artifact::Data) -> Self {
+		match value {
+			tg::artifact::data::Artifact::Directory(data) => Object::Directory(data),
+			tg::artifact::data::Artifact::File(data) => Object::File(data),
+			tg::artifact::data::Artifact::Symlink(data) => Object::Symlink(data),
+		}
+	}
+}

packages/server/src/artifact/checkin.rs

@@ -120,7 +120,7 @@ impl Server {
 			.map_err(|source| tg::error!(!source, "failed to write objects"))?;
 
 		// Write the output to the database.
-		self.write_output_to_database(&output_graph)
+		self.write_output_to_database(output_graph.clone(), object_graph.clone())
 			.await
 			.map_err(|source| tg::error!(!source, "failed to write to the database"))?;
 

packages/server/src/artifact/checkin/lockfile.rs

@@ -116,6 +116,7 @@ impl Server {
 			data.clone()
 		} else {
 			let id: tg::artifact::Id = id.try_into().unwrap();
+
 			tg::Artifact::with_id(id.clone()).data(self).await.map_err(
 				|source| tg::error!(!source, %artifact = id, "missing artifact in object graph"),
 			)?
@@ -184,12 +185,13 @@ impl Server {
 	) -> tg::Result<tg::lockfile::Node> {
 		let id = tg::file::Id::new(&data.serialize()?);
 		let (contents, executable) = match data {
-			tg::file::Data::Graph { graph, node } => {
-				let file = tg::Graph::with_id(graph.clone()).object(self).await?.nodes[*node]
-					.clone()
-					.try_unwrap_file()
-					.unwrap();
-				let contents = file.contents.id(self).await?;
+			tg::file::Data::Graph {
+				graph: graph_id,
+				node,
+			} => {
+				let graph = &graph.graphs.get(graph_id).unwrap().0;
+				let file = graph.nodes[*node].clone().try_unwrap_file().unwrap();
+				let contents = file.contents;
 				let executable = file.executable;
 				(contents, executable)
 			},
@@ -234,17 +236,18 @@ impl Server {
 	) -> tg::Result<tg::lockfile::Node> {
 		let id = tg::symlink::Id::new(&data.serialize()?);
 		let subpath = match data {
-			tg::symlink::Data::Graph { graph, node } => {
-				let symlink = tg::Graph::with_id(graph.clone()).object(self).await?.nodes[*node]
-					.clone()
-					.try_unwrap_symlink()
-					.unwrap();
+			tg::symlink::Data::Graph {
+				graph: graph_id,
+				node,
+			} => {
+				let graph = &graph.graphs.get(graph_id).unwrap().0;
+				let symlink = graph.nodes[*node].clone().try_unwrap_symlink().unwrap();
 				match symlink {
-					tg::graph::object::Symlink::Artifact {
+					tg::graph::data::Symlink::Artifact {
 						artifact: _,
 						subpath,
 					} => subpath,
-					tg::graph::object::Symlink::Target { target } => Some(target),
+					tg::graph::data::Symlink::Target { target } => Some(target),
 				}
 			},
 			tg::symlink::Data::Target { target } => Some(PathBuf::from(target)),

packages/server/src/artifact/checkin/object.rs

@@ -5,12 +5,14 @@ use std::{
 	collections::{BTreeMap, BTreeSet},
 	os::unix::fs::PermissionsExt as _,
 	path::{Path, PathBuf},
+	sync::Arc,
 };
 use tangram_client::{self as tg, handle::Ext};
 use tangram_either::Either;
 
 #[derive(Debug)]
 pub struct Graph {
+	pub graphs: BTreeMap<tg::graph::Id, (tg::graph::Data, tg::object::Metadata)>,
 	pub indices: BTreeMap<unify::Id, usize>,
 	pub nodes: Vec<Node>,
 	pub paths: BTreeMap<PathBuf, usize>,
@@ -50,7 +52,7 @@ impl Server {
 		input: &input::Graph,
 		unify: &unify::Graph,
 		root: &unify::Id,
-	) -> tg::Result<Graph> {
+	) -> tg::Result<Arc<Graph>> {
 		let mut indices = BTreeMap::new();
 		let mut paths: BTreeMap<PathBuf, usize> = BTreeMap::new();
 		let mut nodes = Vec::with_capacity(unify.nodes.len());
@@ -60,13 +62,14 @@ impl Server {
 			.await;
 
 		let mut graph = Graph {
+			graphs: BTreeMap::new(),
 			indices,
 			nodes,
 			paths,
 			objects,
 		};
 		self.create_objects(input, &mut graph).await?;
-		Ok(graph)
+		Ok(Arc::new(graph))
 	}
 
 	async fn create_object_graph_inner(
@@ -132,7 +135,6 @@ impl Server {
 		input: &input::Graph,
 		graph: &mut Graph,
 	) -> tg::Result<()> {
-		let mut graph_metadata = BTreeMap::new();
 		let mut file_metadata = BTreeMap::new();
 
 		// Partition the graph into its strongly connected components.
@@ -193,14 +195,14 @@ impl Server {
 				graph.nodes[index].id.replace(id.clone().into());
 				graph.objects.insert(id.into(), index);
 
+				// Update the graph.
+				let bytes = data.serialize()?;
+				let id = tg::artifact::Id::new(data.kind(), &bytes);
+				graph.nodes[index].data.replace(data.clone());
+				graph.nodes[index].id.replace(id.clone().into());
+
 				// Get the metadata.
-				let metadata = self.compute_object_metadata(
-					graph,
-					index,
-					&data,
-					&file_metadata,
-					&graph_metadata,
-				);
+				let metadata = self.compute_object_metadata(graph, index, &data, &file_metadata);
 				graph.nodes[index].metadata.replace(metadata);
 			} else {
 				// Otherwise, construct an object graph.
@@ -211,10 +213,12 @@ impl Server {
 				// Store the graph.
 				let bytes = object_graph.serialize()?;
 				let id = tg::graph::Id::new(&bytes);
-				let arg = tg::object::put::Arg { bytes };
-				self.put_object(&id.clone().into(), arg).await?;
+				let metadata =
+					self.compute_graph_metadata(graph, &object_graph, &scc, &file_metadata);
+				graph
+					.graphs
+					.insert(id.clone(), (object_graph.clone(), metadata));
 
-				// Update data.
 				for old_index in scc.iter().copied() {
 					// Get the index within the object graph.
 					let new_index = indices.get(&old_index).copied().unwrap();
@@ -246,11 +250,6 @@ impl Server {
 					graph.objects.insert(id.into(), old_index);
 				}
 
-				// Update the graph metadata.
-				let metadata =
-					self.compute_graph_metadata(graph, &object_graph, &scc, &file_metadata);
-				graph_metadata.insert(id.clone(), metadata);
-
 				// Update metadata.
 				for old_index in scc {
 					// Get the metadata.
@@ -260,7 +259,6 @@ impl Server {
 						old_index,
 						data,
 						&file_metadata,
-						&graph_metadata,
 					);
 					graph.nodes[old_index].metadata.replace(metadata);
 				}
@@ -711,7 +709,6 @@ impl Server {
 		index: usize,
 		data: &tg::artifact::Data,
 		file_metadata: &BTreeMap<usize, tg::object::Metadata>,
-		graph_metadata: &BTreeMap<tg::graph::Id, tg::object::Metadata>,
 	) -> tg::object::Metadata {
 		if let Some(metadata) = graph.nodes[index].metadata.clone() {
 			return metadata;
@@ -730,10 +727,16 @@ impl Server {
 		let mut weight = weight.map(|weight| weight + data_size);
 
 		match data {
-			tg::artifact::Data::Directory(tg::directory::Data::Graph { graph, .. })
-			| tg::artifact::Data::File(tg::file::Data::Graph { graph, .. })
-			| tg::artifact::Data::Symlink(tg::symlink::Data::Graph { graph, .. }) => {
-				let metadata = graph_metadata.get(graph).unwrap();
+			tg::artifact::Data::Directory(tg::directory::Data::Graph {
+				graph: graph_id, ..
+			})
+			| tg::artifact::Data::File(tg::file::Data::Graph {
+				graph: graph_id, ..
+			})
+			| tg::artifact::Data::Symlink(tg::symlink::Data::Graph {
+				graph: graph_id, ..
+			}) => {
+				let metadata = &graph.graphs.get(graph_id).unwrap().1;
 				complete &= metadata.complete;
 				if let Some(c) = metadata.count {
 					count = count.map(|count| count + c);
@@ -761,13 +764,7 @@ impl Server {
 
 					let metadata = graph.nodes[edge.index].metadata.clone().unwrap_or_else(|| {
 						if let Some(data) = graph.nodes[edge.index].data.as_ref() {
-							self.compute_object_metadata(
-								graph,
-								edge.index,
-								data,
-								file_metadata,
-								graph_metadata,
-							)
+							self.compute_object_metadata(graph, edge.index, data, file_metadata)
 						} else {
 							tg::object::Metadata {
 								complete: false,