From 960bd3954f6b472ff4e607a087294e60bd25d1d7 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 9 Aug 2023 20:37:22 -0700 Subject: [PATCH 01/12] feat: implement serde for index_map --- Cargo.toml | 5 +++++ src/index_map.rs | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 5f81a46..e78e768 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ categories = ["data-structures", "caching"] [dependencies] stable_deref_trait = "1.1.1" indexmap = { version = "1.6", optional = true } +serde = { version = "1.0.171", features = ["derive"], optional = true } [package.metadata.docs.rs] features = ["indexmap"] @@ -21,3 +22,7 @@ features = ["indexmap"] name = "string_interner" path = "examples/string_interner.rs" required-features = ["indexmap"] + +[features] +default = [] +serde = ["dep:serde", "indexmap/serde"] diff --git a/src/index_map.rs b/src/index_map.rs index fb70b7f..7bda053 100644 --- a/src/index_map.rs +++ b/src/index_map.rs @@ -8,6 +8,9 @@ use std::ops::Index; use indexmap::IndexMap; use stable_deref_trait::StableDeref; +#[cfg(feature = "serde")] +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + /// Append-only version of `indexmap::IndexMap` where /// insertion does not require mutable access pub struct FrozenIndexMap { @@ -235,3 +238,37 @@ impl Default for FrozenIndexMap { } } } + +#[cfg(feature = "serde")] +impl Serialize for FrozenIndexMap +where + K: Serialize + Eq + Hash, + V: Serialize, + S: BuildHasher, +{ + fn serialize(&self, serializer: Ser) -> Result + where + Ser: Serializer, + { + assert!(!self.in_use.get()); + self.in_use.set(true); + let map_serialized = unsafe { self.map.get().as_ref().unwrap() }.serialize(serializer); + self.in_use.set(false); + return map_serialized; + } +} + +#[cfg(feature = "serde")] +impl<'de, K, V, S> Deserialize<'de> for FrozenIndexMap +where + K: Deserialize<'de> + Eq + Hash, + V: Deserialize<'de>, + S: BuildHasher + Default, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + IndexMap::deserialize(deserializer).map(FrozenIndexMap::from) + } +} From 079501757819cfee7df907a8c50ec0cfd51837ac Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 9 Aug 2023 20:37:33 -0700 Subject: [PATCH 02/12] feat: implement serde for index_set --- src/index_set.rs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/index_set.rs b/src/index_set.rs index 9e99632..1fb2186 100644 --- a/src/index_set.rs +++ b/src/index_set.rs @@ -8,6 +8,9 @@ use std::ops::Index; use indexmap::IndexSet; use stable_deref_trait::StableDeref; +#[cfg(feature = "serde")] +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + /// Append-only version of `indexmap::IndexSet` where /// insertion does not require mutable access pub struct FrozenIndexSet { @@ -180,3 +183,35 @@ impl Default for FrozenIndexSet { Self::from(IndexSet::default()) } } + +#[cfg(feature = "serde")] +impl Serialize for FrozenIndexSet +where + T: Eq + Hash + Serialize, + S: BuildHasher, +{ + fn serialize(&self, serializer: Ser) -> Result + where + Ser: Serializer, + { + assert!(!self.in_use.get()); + self.in_use.set(true); + let map_serialized = unsafe { self.set.get().as_ref().unwrap() }.serialize(serializer); + self.in_use.set(false); + return map_serialized; + } +} + +#[cfg(feature = "serde")] +impl<'de, K, S> Deserialize<'de> for FrozenIndexSet +where + K: Deserialize<'de> + Eq + Hash, + S: BuildHasher + Default, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + IndexSet::deserialize(deserializer).map(FrozenIndexSet::from) + } +} From 39323123723300ff4cd8d51996d325fef349f727 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 9 Aug 2023 20:37:56 -0700 Subject: [PATCH 03/12] feat: implement serde for FrozenMap --- src/map.rs | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/map.rs b/src/map.rs index 147a2ec..6d765b9 100644 --- a/src/map.rs +++ b/src/map.rs @@ -9,6 +9,9 @@ use std::ops::Index; use stable_deref_trait::StableDeref; +#[cfg(feature = "serde")] +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + /// Append-only version of `std::collections::HashMap` where /// insertion does not require mutable access pub struct FrozenMap { @@ -270,6 +273,40 @@ impl Default for FrozenMap { } } +#[cfg(feature = "serde")] +impl Serialize for FrozenMap +where + K: Serialize + Eq + Hash, + V: Serialize, + S: BuildHasher, +{ + fn serialize(&self, serializer: Ser) -> Result + where + Ser: Serializer, + { + assert!(!self.in_use.get()); + self.in_use.set(true); + let map_serialized = unsafe { self.map.get().as_ref().unwrap() }.serialize(serializer); + self.in_use.set(false); + return map_serialized; + } +} + +#[cfg(feature = "serde")] +impl<'de, K, V, S> Deserialize<'de> for FrozenMap +where + K: Deserialize<'de> + Eq + Hash, + V: Deserialize<'de>, + S: BuildHasher + Default, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + HashMap::deserialize(deserializer).map(FrozenMap::from) + } +} + /// Append-only version of `std::collections::BTreeMap` where /// insertion does not require mutable access pub struct FrozenBTreeMap { @@ -495,3 +532,4 @@ impl Default for FrozenBTreeMap { } } } +} From e8e862aad5efb9f396328b33dfad6a6f57011241 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 9 Aug 2023 20:38:10 -0700 Subject: [PATCH 04/12] feat: implement serde for FrozenBTreeMap --- src/map.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/map.rs b/src/map.rs index 6d765b9..31c6ad5 100644 --- a/src/map.rs +++ b/src/map.rs @@ -532,4 +532,35 @@ impl Default for FrozenBTreeMap { } } } + +#[cfg(feature = "serde")] +impl Serialize for FrozenBTreeMap +where + K: Serialize + Eq + Hash, + V: Serialize, +{ + fn serialize(&self, serializer: Ser) -> Result + where + Ser: Serializer, + { + assert!(!self.in_use.get()); + self.in_use.set(true); + let map_serialized = unsafe { self.map.get().as_ref().unwrap() }.serialize(serializer); + self.in_use.set(false); + return map_serialized; + } +} + +#[cfg(feature = "serde")] +impl<'de, K: Clone + Ord, V: StableDeref> Deserialize<'de> for FrozenBTreeMap +where + K: Deserialize<'de> + Eq + Hash, + V: Deserialize<'de>, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + BTreeMap::deserialize(deserializer).map(FrozenBTreeMap::from) + } } From 86b7df68655ad6d253118dc7d01adbe12c4ae204 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 9 Aug 2023 20:44:04 -0700 Subject: [PATCH 05/12] feat: implement serde for LockFreeFrozenVec --- Cargo.toml | 3 +++ src/sync.rs | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index e78e768..f298d2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,9 @@ name = "string_interner" path = "examples/string_interner.rs" required-features = ["indexmap"] +[dev-dependencies] +serde_json = "1.0.104" + [features] default = [] serde = ["dep:serde", "indexmap/serde"] diff --git a/src/sync.rs b/src/sync.rs index a551b9a..53de9fa 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -713,9 +713,52 @@ fn test_non_lockfree_unchecked() { LockFreeFrozenVec::<()>::new(); } +#[cfg(feature = "serde")] +impl Serialize for LockFreeFrozenVec { + fn serialize(&self, serializer: S) -> Result { + use serde::ser::SerializeSeq; + + let len = self.len.load(Ordering::Relaxed); + let mut seq = serializer.serialize_seq(Some(len))?; + for i in 0..len { + seq.serialize_element(&self.get(i).unwrap())?; + } + seq.end() + } +} + +#[cfg(feature = "serde")] +impl<'de, T: Copy + Deserialize<'de>> Deserialize<'de> for LockFreeFrozenVec { + fn deserialize>(deserializer: D) -> Result { + use serde::de::{SeqAccess, Visitor}; + use std::marker::PhantomData; + + struct SeqVisitor(PhantomData); + + impl<'de, T: Copy + Deserialize<'de>> Visitor<'de> for SeqVisitor { + type Value = LockFreeFrozenVec; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a sequence of elements") + } + + fn visit_seq>(self, mut seq: A) -> Result { + let ret = LockFreeFrozenVec::new(); + while let Some(elem) = seq.next_element()? { + ret.push(elem); + } + Ok(ret) + } + } + + deserializer.deserialize_seq(SeqVisitor(PhantomData)) + } +} + #[test] fn test_non_lockfree() { #[derive(Copy, Clone, Debug, PartialEq, Eq)] + #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] struct Moo(i32); let vec = LockFreeFrozenVec::new(); @@ -746,6 +789,19 @@ fn test_non_lockfree() { // Test dropping empty vecs LockFreeFrozenVec::<()>::new(); + + #[cfg(feature = "serde")] + { + let vec = LockFreeFrozenVec::new(); + vec.push(Moo(1)); + vec.push(Moo(2)); + vec.push(Moo(3)); + let json = serde_json::to_string(&vec).unwrap(); + let vec = serde_json::from_str::>(&json).unwrap(); + assert_eq!(vec.get(0), Some(Moo(1))); + assert_eq!(vec.get(1), Some(Moo(2))); + assert_eq!(vec.get(2), Some(Moo(3))); + } } // TODO: Implement IntoIterator for LockFreeFrozenVec From 8cad3ba4a2b7917b55f252bc05cbd35cf6de6e4c Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 9 Aug 2023 20:50:47 -0700 Subject: [PATCH 06/12] feat: implement serde for FrozenVec --- src/sync.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sync.rs b/src/sync.rs index 53de9fa..6b1f70a 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -21,6 +21,9 @@ use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::sync::RwLock; +#[cfg(feature = "serde")] +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + /// Append-only threadsafe version of `std::collections::HashMap` where /// insertion does not require mutable access pub struct FrozenMap { @@ -383,6 +386,7 @@ impl std::convert::AsMut> for FrozenMap { /// Append-only threadsafe version of `std::vec::Vec` where /// insertion does not require mutable access +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct FrozenVec { vec: RwLock>, } From 2da0584eec989787d9f45a95c40fa9b6418e13b5 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 9 Aug 2023 20:52:30 -0700 Subject: [PATCH 07/12] feat: implement serde for FrozenMap --- src/sync.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/sync.rs b/src/sync.rs index 6b1f70a..c2e5701 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -26,6 +26,7 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; /// Append-only threadsafe version of `std::collections::HashMap` where /// insertion does not require mutable access +#[cfg_attr(feature = "serde", derive(Serialize))] pub struct FrozenMap { map: RwLock>, } @@ -384,6 +385,20 @@ impl std::convert::AsMut> for FrozenMap { } } +#[cfg(feature = "serde")] +impl<'de, K, V> Deserialize<'de> for FrozenMap +where + K: Deserialize<'de> + Eq + Hash, + V: Deserialize<'de>, +{ + fn deserialize>(deserializer: D) -> Result { + let map = HashMap::::deserialize(deserializer)?; + Ok(Self { + map: RwLock::new(map), + }) + } +} + /// Append-only threadsafe version of `std::vec::Vec` where /// insertion does not require mutable access #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] From 1c8c9b4702eb7ae4aff137a3f44e0b8a3d485137 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Wed, 9 Aug 2023 20:53:20 -0700 Subject: [PATCH 08/12] feat: implement serde for FrozenBTreeMap --- src/sync.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/sync.rs b/src/sync.rs index c2e5701..d7b2945 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -828,6 +828,7 @@ fn test_non_lockfree() { /// Append-only threadsafe version of `std::collections::BTreeMap` where /// insertion does not require mutable access #[derive(Debug)] +#[cfg_attr(feature = "serde", derive(Serialize))] pub struct FrozenBTreeMap(RwLock>); impl FrozenBTreeMap { @@ -1008,3 +1009,11 @@ impl Default for FrozenBTreeMap { Self::new() } } + +#[cfg(feature = "serde")] +impl<'de> Deserialize<'de> for FrozenBTreeMap { + fn deserialize>(deserializer: D) -> Result { + let map = BTreeMap::::deserialize(deserializer)?; + Ok(map.into()) + } +} From 787fa06ce8f6d470ada0c8b9846d0481c007f885 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sat, 23 Sep 2023 02:03:56 -0700 Subject: [PATCH 09/12] test: serde sync_frozen_vec --- src/sync.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/sync.rs b/src/sync.rs index d7b2945..8633f75 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -401,6 +401,7 @@ where /// Append-only threadsafe version of `std::vec::Vec` where /// insertion does not require mutable access +#[derive(Debug)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct FrozenVec { vec: RwLock>, @@ -507,6 +508,36 @@ impl Default for FrozenVec { } } +impl Clone for FrozenVec { + fn clone(&self) -> Self { + Self { + vec: self.vec.read().unwrap().clone().into(), + } + } +} + +impl PartialEq for FrozenVec { + fn eq(&self, other: &Self) -> bool { + let self_ref: &Vec = &self.vec.read().unwrap(); + let other_ref: &Vec = &other.vec.read().unwrap(); + self_ref == other_ref + } +} + +#[test] +fn test_sync_frozen_vec() { + #[cfg(feature = "serde")] + { + let vec = FrozenVec::new(); + vec.push(String::from("a")); + + let vec_json = serde_json::to_string(&vec).unwrap(); + assert_eq!(vec_json, "{\"vec\":[\"a\"]}"); + let vec_serde = serde_json::from_str::>(&vec_json).unwrap(); + assert_eq!(vec, vec_serde); + } +} + // The context for these functions is that we want to have a // series of exponentially increasing buffer sizes. We want // to maximize the total size of the buffers (since this From 1be36bc76a8a147709fb35dd54c6876ca921b557 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sat, 23 Sep 2023 02:11:12 -0700 Subject: [PATCH 10/12] fix: fix deserialization for sync::FrozenMap --- src/sync.rs | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/src/sync.rs b/src/sync.rs index 8633f75..884c5ae 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -26,7 +26,12 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; /// Append-only threadsafe version of `std::collections::HashMap` where /// insertion does not require mutable access -#[cfg_attr(feature = "serde", derive(Serialize))] +#[derive(Debug)] +#[cfg_attr( + feature = "serde", + derive(Serialize, Deserialize), + serde(bound(deserialize = "K: Eq + Hash + Deserialize<'de>, V: Deserialize<'de>")) +)] pub struct FrozenMap { map: RwLock>, } @@ -385,17 +390,26 @@ impl std::convert::AsMut> for FrozenMap { } } -#[cfg(feature = "serde")] -impl<'de, K, V> Deserialize<'de> for FrozenMap -where - K: Deserialize<'de> + Eq + Hash, - V: Deserialize<'de>, -{ - fn deserialize>(deserializer: D) -> Result { - let map = HashMap::::deserialize(deserializer)?; - Ok(Self { - map: RwLock::new(map), - }) +impl PartialEq for FrozenMap { + fn eq(&self, other: &Self) -> bool { + let self_ref: &HashMap = &self.map.read().unwrap(); + let other_ref: &HashMap = &other.map.read().unwrap(); + self_ref == other_ref + } +} + +#[test] +fn test_sync_frozen_map() { + #[cfg(feature = "serde")] + { + let map = FrozenMap::new(); + map.insert(String::from("a"), String::from("b")); + + let map_json = serde_json::to_string(&map).unwrap(); + assert_eq!(map_json, "{\"map\":{\"a\":\"b\"}}"); + + let map_serde = serde_json::from_str::>(&map_json).unwrap(); + assert_eq!(map, map_serde); } } From e766ad70f9ea9c519aa692b2894ce44d4415f3ef Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sat, 23 Sep 2023 02:21:16 -0700 Subject: [PATCH 11/12] fix: fix deserialization for sync::FrozenBTreeMap --- src/sync.rs | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/sync.rs b/src/sync.rs index 884c5ae..9d78d12 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -873,7 +873,11 @@ fn test_non_lockfree() { /// Append-only threadsafe version of `std::collections::BTreeMap` where /// insertion does not require mutable access #[derive(Debug)] -#[cfg_attr(feature = "serde", derive(Serialize))] +#[cfg_attr( + feature = "serde", + derive(Serialize, Deserialize), + serde(bound(deserialize = "K: Ord + Deserialize<'de>, V: Deserialize<'de>")) +)] pub struct FrozenBTreeMap(RwLock>); impl FrozenBTreeMap { @@ -1055,10 +1059,24 @@ impl Default for FrozenBTreeMap { } } -#[cfg(feature = "serde")] -impl<'de> Deserialize<'de> for FrozenBTreeMap { - fn deserialize>(deserializer: D) -> Result { - let map = BTreeMap::::deserialize(deserializer)?; - Ok(map.into()) +impl PartialEq for FrozenBTreeMap { + fn eq(&self, other: &Self) -> bool { + let self_ref: &BTreeMap = &self.0.read().unwrap(); + let other_ref: &BTreeMap = &other.0.read().unwrap(); + self_ref == other_ref + } +} +#[test] +fn test_sync_frozen_btreemap() { + #[cfg(feature = "serde")] + { + let map = FrozenBTreeMap::new(); + map.insert(String::from("a"), String::from("b")); + + let map_json = serde_json::to_string(&map).unwrap(); + assert_eq!(map_json, "{\"a\":\"b\"}"); + + let map_serde = serde_json::from_str::>(&map_json).unwrap(); + assert_eq!(map, map_serde); } } From 3cbdd45d572c1198f18dc4b6a062676767e18997 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 24 Oct 2023 21:38:45 -0700 Subject: [PATCH 12/12] fix: fix the conflicts and add test for empty LockFreeFrozenVec --- Cargo.toml | 1 + src/sync.rs | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f298d2e..ca0ef7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,3 +29,4 @@ serde_json = "1.0.104" [features] default = [] serde = ["dep:serde", "indexmap/serde"] +indexmap = ["dep:indexmap"] diff --git a/src/sync.rs b/src/sync.rs index 718e2e4..804d428 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -30,7 +30,6 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; /// Append-only threadsafe version of `std::collections::HashMap` where /// insertion does not require mutable access -#[derive(Debug)] #[cfg_attr( feature = "serde", derive(Serialize, Deserialize), @@ -457,7 +456,6 @@ fn test_sync_frozen_map() { /// Append-only threadsafe version of `std::vec::Vec` where /// insertion does not require mutable access -#[derive(Debug)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct FrozenVec { vec: RwLock>, @@ -1069,14 +1067,18 @@ fn test_non_lockfree() { #[cfg(feature = "serde")] { let vec = LockFreeFrozenVec::new(); + let json_empty = serde_json::to_string(&vec).unwrap(); + let vec_empty = serde_json::from_str::>(&json_empty).unwrap(); + assert_eq!(vec_empty.get(0), None); + vec.push(Moo(1)); vec.push(Moo(2)); vec.push(Moo(3)); let json = serde_json::to_string(&vec).unwrap(); - let vec = serde_json::from_str::>(&json).unwrap(); - assert_eq!(vec.get(0), Some(Moo(1))); - assert_eq!(vec.get(1), Some(Moo(2))); - assert_eq!(vec.get(2), Some(Moo(3))); + let serde_vec = serde_json::from_str::>(&json).unwrap(); + assert_eq!(serde_vec.get(0), Some(Moo(1))); + assert_eq!(serde_vec.get(1), Some(Moo(2))); + assert_eq!(serde_vec.get(2), Some(Moo(3))); } }