Skip to content

Commit

Permalink
feat(blockstore): add IndexedDb blockstore (#221)
Browse files Browse the repository at this point in the history 
* feat(blockstore): Add IndexedDb blockstore

* chore(ci): build wasm32 with all features

* remove commented out key path

Co-authored-by: Mikołaj Florkiewicz <[email protected]>
Signed-off-by: Maciej Zwoliński <[email protected]>

* fmt

* use rstest instead of inhouse macro

* remove _blockstore suffix from testcase functions

* correct the function name

* fix wasm tests

* fix the documentation

* remove plurality from store's name

* remove the max multihash size

---------

Signed-off-by: Maciej Zwoliński <[email protected]>
Co-authored-by: Mikołaj Florkiewicz <[email protected]>
  • Loading branch information
@zvolin @fl0rek
zvolin and fl0rek authored Feb 19, 2024
1 parent 2b873d1 commit f3e3612
Show file tree
Hide file tree
Showing 11 changed files with 584 additions and 327 deletions.
5 changes: 4 additions & 1 deletion .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ jobs:
uses: nanasess/setup-chromedriver@v2

- name: Build (wasm32-unknown-unknown)
run: cargo build --all --target=wasm32-unknown-unknown --features=wasm-bindgen
run: cargo build --all --target=wasm32-unknown-unknown --all-features

- name: Test proto crate
run: wasm-pack test --node proto
Expand All @@ -97,6 +97,9 @@ jobs:
- name: Test node-wasm crate
run: wasm-pack test --headless --chrome node-wasm

- name: Test blockstore crate
run: wasm-pack test --headless --chrome blockstore --all-features

- name: Build and pack node-wasm
run: wasm-pack build --release --target web node-wasm && wasm-pack pack node-wasm

Expand Down
47 changes: 47 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 14 additions & 1 deletion blockstore/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,23 +19,36 @@ cid = "0.11.0"
dashmap = "5.5.3"
lru = { version = "0.12.2", optional = true }
multihash = "0.19.1"
thiserror = "1.0.40"

[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
# Upgrading this dependency invalidates existing persistent dbs.
# Those can be restored by migrating between versions:
# https://docs.rs/sled/latest/sled/struct.Db.html#examples-1
sled = { version = "0.34.7", optional = true }
tokio = { version = "1.29.0", features = ["macros", "rt"], optional = true }
thiserror = "1.0.40"

[target.'cfg(target_arch = "wasm32")'.dependencies]
js-sys = { version = "0.3.68", optional = true }
rexie = { version = "0.5.0", optional = true }
send_wrapper = { version = "0.6.0", features = ["futures"], optional = true }
wasm-bindgen = { version = "0.2.91", optional = true }

[dev-dependencies]
rstest = "0.18.2"
tokio = { version = "1.29.0", features = ["macros", "rt"] }
tempfile = "3.10"

# doc-tests
multihash-codetable = { version = "0.1.1", features = ["digest", "sha2"] }

[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
wasm-bindgen-test = "0.3.41"

[features]
lru = ["dep:lru"]
sled = ["dep:sled", "dep:tokio"]
indexeddb = ["dep:js-sys", "dep:rexie", "dep:send_wrapper", "dep:wasm-bindgen"]

[package.metadata.docs.rs]
rustdoc-args = ["--cfg", "docs_rs"]
Expand Down
207 changes: 0 additions & 207 deletions blockstore/src/in_memory_blockstore.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,210 +60,3 @@ impl<const MAX_MULTIHASH_SIZE: usize> Default for InMemoryBlockstore<MAX_MULTIHA
Self::new()
}
}

#[cfg(test)]
mod tests {
use super::*;
use crate::block::{Block, CidError};
use multihash::Multihash;
use std::iter::zip;
use std::result::Result as StdResult;

#[tokio::test]
async fn test_insert_get() {
let store = InMemoryBlockstore::<128>::new();

let cid = CidGeneric::<128>::read_bytes(
[
0x01, // CIDv1
0x01, // CID codec = 1
0x02, // code = 2
0x03, // len = 3
1, 2, 3, // hash
]
.as_ref(),
)
.unwrap();
let data = [0xCD; 512];

store.put_keyed(&cid, &data).await.unwrap();

let retrieved_data = store.get(&cid).await.unwrap().unwrap();
assert_eq!(data, retrieved_data.as_ref());

let another_cid = CidGeneric::<128>::default();
let missing_block = store.get(&another_cid).await.unwrap();
assert_eq!(missing_block, None);
}

#[tokio::test]
async fn test_duplicate_insert() {
let cid0 = CidGeneric::<128>::read_bytes(
[
0x01, // CIDv1
0x11, // CID codec
0x22, // multihash code
0x02, // len = 2
0, 0, // hash
]
.as_ref(),
)
.unwrap();
let cid1 = CidGeneric::<128>::read_bytes(
[
0x01, // CIDv1
0x33, // CID codec
0x44, // multihash code
0x02, // len = 2
0, 1, // hash
]
.as_ref(),
)
.unwrap();

let store = InMemoryBlockstore::<128>::new();

store.put_keyed(&cid0, &[0x01; 1]).await.unwrap();
store.put_keyed(&cid1, &[0x02; 1]).await.unwrap();
let insert_err = store.put_keyed(&cid1, &[0x03; 1]).await.unwrap_err();
assert!(matches!(insert_err, BlockstoreError::CidExists));
}

#[tokio::test]
async fn different_cid_size() {
let cid_bytes = [
0x01, // CIDv1
0x11, // CID codec
0x22, // multihash code
0x02, // len = 2
0, 0, // hash
];
let cid0 = CidGeneric::<6>::read_bytes(cid_bytes.as_ref()).unwrap();
let cid1 = CidGeneric::<32>::read_bytes(cid_bytes.as_ref()).unwrap();
let cid2 = CidGeneric::<64>::read_bytes(cid_bytes.as_ref()).unwrap();
let cid3 = CidGeneric::<65>::read_bytes(cid_bytes.as_ref()).unwrap();
let cid4 = CidGeneric::<128>::read_bytes(cid_bytes.as_ref()).unwrap();

let data = [0x99; 5];

let store = InMemoryBlockstore::<64>::new();
store.put_keyed(&cid0, data.as_ref()).await.unwrap();

let data0 = store.get(&cid0).await.unwrap().unwrap();
assert_eq!(data, data0.as_ref());
let data1 = store.get(&cid1).await.unwrap().unwrap();
assert_eq!(data, data1.as_ref());
let data2 = store.get(&cid2).await.unwrap().unwrap();
assert_eq!(data, data2.as_ref());
let data3 = store.get(&cid3).await.unwrap().unwrap();
assert_eq!(data, data3.as_ref());
let data4 = store.get(&cid4).await.unwrap().unwrap();
assert_eq!(data, data4.as_ref());
}

#[tokio::test]
async fn too_large_cid() {
let cid = CidGeneric::<32>::read_bytes(
[
0x01, // CIDv1
0x11, // CID codec
0x22, // multihash code
0x10, // len = 16
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
]
.as_ref(),
)
.unwrap();

let store = InMemoryBlockstore::<8>::new();
let insert_err = store.put_keyed(&cid, [0x00, 1].as_ref()).await.unwrap_err();
assert!(matches!(insert_err, BlockstoreError::CidTooLong));

let insert_err = store.get(&cid).await.unwrap_err();
assert!(matches!(insert_err, BlockstoreError::CidTooLong));
}

#[tokio::test]
async fn test_block_insert() {
let block = TestBlock([0, 1, 2, 3]);

let store = InMemoryBlockstore::<8>::new();
store.put(block).await.unwrap();
let retrieved_block = store.get(&block.cid().unwrap()).await.unwrap().unwrap();
assert_eq!(block.data(), &retrieved_block);
}

#[tokio::test]
async fn test_multiple_blocks_insert() {
let blocks = [
TestBlock([0, 0, 0, 0]),
TestBlock([0, 0, 0, 1]),
TestBlock([0, 0, 1, 0]),
TestBlock([0, 0, 1, 1]),
TestBlock([0, 1, 0, 0]),
TestBlock([0, 1, 0, 1]),
TestBlock([0, 1, 1, 0]),
TestBlock([0, 1, 1, 1]),
];
let uninserted_blocks = [
TestBlock([1, 0, 0, 0]),
TestBlock([1, 0, 0, 1]),
TestBlock([1, 0, 1, 0]),
TestBlock([1, 1, 0, 1]),
];

let store = InMemoryBlockstore::<8>::new();
store.put_many(blocks).await.unwrap();

for b in blocks {
let cid = b.cid().unwrap();
assert!(store.has(&cid).await.unwrap());
let retrieved_block = store.get(&cid).await.unwrap().unwrap();
assert_eq!(b.data(), &retrieved_block);
}

for b in uninserted_blocks {
let cid = b.cid().unwrap();
assert!(!store.has(&cid).await.unwrap());
assert!(store.get(&cid).await.unwrap().is_none());
}
}

#[tokio::test]
async fn test_multiple_keyed() {
let blocks = [[0], [1], [2], [3]];
let cids = [
// 4 different arbitrary CIDs
TestBlock([0, 0, 0, 1]).cid().unwrap(),
TestBlock([0, 0, 0, 2]).cid().unwrap(),
TestBlock([0, 0, 0, 3]).cid().unwrap(),
TestBlock([0, 0, 0, 4]).cid().unwrap(),
];
let pairs = zip(cids, blocks);

let store = InMemoryBlockstore::<8>::new();
store.put_many_keyed(pairs.clone()).await.unwrap();

for (cid, block) in pairs {
let retrieved_block = store.get(&cid).await.unwrap().unwrap();
assert_eq!(block.as_ref(), &retrieved_block);
}
}

const TEST_CODEC: u64 = 0x0A;
const TEST_MH_CODE: u64 = 0x0A;

#[derive(Debug, PartialEq, Clone, Copy)]
struct TestBlock(pub [u8; 4]);

impl Block<8> for TestBlock {
fn cid(&self) -> StdResult<CidGeneric<8>, CidError> {
let mh = Multihash::wrap(TEST_MH_CODE, &self.0).unwrap();
Ok(CidGeneric::new_v1(TEST_CODEC, mh))
}

fn data(&self) -> &[u8] {
&self.0
}
}
}
Loading

0 comments on commit f3e3612

Please sign in to comment.