From ef431755f60d57c638df5133776045c981de4916 Mon Sep 17 00:00:00 2001 From: b5 Date: Tue, 27 Jun 2023 09:38:23 -0400 Subject: [PATCH 1/7] feat: incorporate noosphere_car enhancements --- Cargo.toml | 11 +++--- src/error.rs | 2 +- src/header.rs | 21 ++++++---- src/lib.rs | 1 + src/reader.rs | 29 +++++++++++--- src/util.rs | 22 ++++++++--- src/varint.rs | 89 ++++++++++++++++++++++++++++++++++++++++++ src/writer.rs | 15 ++++--- tests/car_file_test.rs | 28 +++++++------ 9 files changed, 178 insertions(+), 40 deletions(-) create mode 100644 src/varint.rs diff --git a/Cargo.toml b/Cargo.toml index f0a950c..21c41bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,16 +9,17 @@ rust-version = "1.65" repository = "https://github.com/n0-computer/iroh-car" [dependencies] -cid = "0.9" +anyhow = "1" +cid = "0.10" futures = "0.3" -integer-encoding = { version = "3.0", features = ["tokio_async"] } -ipld = { package = "libipld", version = "0.15" } -ipld-cbor = { package = "libipld-cbor", version = "0.15" } +integer-encoding = { version = "3", features = ["tokio_async"] } +libipld = { package = "libipld", version = "0.16", features = ["dag-cbor", "derive"] } +libipld-cbor = { package = "libipld-cbor", version = "0.16" } thiserror = "1" tokio = { version = "1", features = ["io-util"] } [dev-dependencies] -multihash = "0.17" +multihash = "0.18" tokio = { version = "1", features = ["macros", "sync", "rt", "fs", "io-util"] } [features] diff --git a/src/error.rs b/src/error.rs index 3579413..f228b9e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,7 +10,7 @@ pub enum Error { #[error("Io error: {0}")] Io(#[from] std::io::Error), #[error("Cbor encoding error: {0}")] - Cbor(#[from] ipld::error::Error), + Cbor(#[from] libipld::error::Error), #[error("ld read too large {0}")] LdReadTooLarge(usize), } diff --git a/src/header.rs b/src/header.rs index c004e35..99a6709 100644 --- a/src/header.rs +++ b/src/header.rs @@ -1,6 +1,6 @@ use cid::Cid; -use ipld::codec::Codec; -use ipld_cbor::DagCborCodec; +use libipld::codec::Codec; +use libipld_cbor::DagCborCodec; use crate::error::Error; @@ -57,7 +57,7 @@ impl CarHeader { } /// CAR file header version 1. -#[derive(Debug, Clone, Default, ipld::DagCbor, PartialEq, Eq)] +#[derive(Debug, Clone, Default, libipld::DagCbor, PartialEq, Eq)] pub struct CarHeaderV1 { #[ipld] pub roots: Vec, @@ -80,15 +80,22 @@ impl From> for CarHeaderV1 { #[cfg(test)] mod tests { - use ipld::codec::{Decode, Encode}; - use ipld_cbor::DagCborCodec; + use libipld::codec::{Decode, Encode}; + use libipld_cbor::DagCborCodec; use multihash::MultihashDigest; use super::*; - #[test] + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::wasm_bindgen_test; + + #[cfg(target_arch = "wasm32")] + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), test)] fn symmetric_header_v1() { - let digest = multihash::Code::Blake2b256.digest(b"test"); + let digest = multihash::Code::Blake3_256.digest(b"test"); let cid = Cid::new_v1(DagCborCodec.into(), digest); let header = CarHeaderV1::from(vec![cid]); diff --git a/src/lib.rs b/src/lib.rs index d4e5f66..0f2d383 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ mod error; mod header; mod reader; mod util; +mod varint; mod writer; pub use crate::header::CarHeader; diff --git a/src/reader.rs b/src/reader.rs index 9917596..7e1623f 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -8,6 +8,18 @@ use crate::{ util::{ld_read, read_node}, }; +#[cfg(not(target_arch = "wasm32"))] +pub trait CarReaderSend: Send {} + +#[cfg(not(target_arch = "wasm32"))] +impl CarReaderSend for S where S: Send {} + +#[cfg(target_arch = "wasm32")] +pub trait CarReaderSend {} + +#[cfg(target_arch = "wasm32")] +impl CarReaderSend for S {} + /// Reads CAR files that are in a BufReader #[derive(Debug)] pub struct CarReader { @@ -18,7 +30,7 @@ pub struct CarReader { impl CarReader where - R: AsyncRead + Send + Unpin, + R: AsyncRead + CarReaderSend + Unpin, { /// Creates a new CarReader and parses the CarHeader pub async fn new(mut reader: R) -> Result { @@ -64,19 +76,26 @@ mod tests { use cid::Cid; use futures::TryStreamExt; - use ipld_cbor::DagCborCodec; + use libipld_cbor::DagCborCodec; use multihash::MultihashDigest; use crate::{header::CarHeaderV1, writer::CarWriter}; use super::*; - #[tokio::test] + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::wasm_bindgen_test; + + #[cfg(target_arch = "wasm32")] + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn car_write_read() { - let digest_test = multihash::Code::Blake2b256.digest(b"test"); + let digest_test = multihash::Code::Blake3_256.digest(b"test"); let cid_test = Cid::new_v1(DagCborCodec.into(), digest_test); - let digest_foo = multihash::Code::Blake2b256.digest(b"foo"); + let digest_foo = multihash::Code::Blake3_256.digest(b"foo"); let cid_foo = Cid::new_v1(DagCborCodec.into(), digest_foo); let header = CarHeader::V1(CarHeaderV1::from(vec![cid_foo])); diff --git a/src/util.rs b/src/util.rs index 25be761..a179371 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,7 +1,9 @@ +use anyhow::Result; use cid::Cid; -use integer_encoding::VarIntAsyncReader; use tokio::io::{AsyncRead, AsyncReadExt}; +use crate::{reader::CarReaderSend, varint::read_varint_async}; + use super::error::Error; /// Maximum size that is used for single node. @@ -9,9 +11,9 @@ pub(crate) const MAX_ALLOC: usize = 4 * 1024 * 1024; pub(crate) async fn ld_read(mut reader: R, buf: &mut Vec) -> Result, Error> where - R: AsyncRead + Send + Unpin, + R: AsyncRead + CarReaderSend + Unpin, { - let length: usize = match VarIntAsyncReader::read_varint_async(&mut reader).await { + let length: usize = match read_varint_async(&mut reader).await { Ok(len) => len, Err(e) => { if e.kind() == std::io::ErrorKind::UnexpectedEof { @@ -41,7 +43,7 @@ pub(crate) async fn read_node( buf: &mut Vec, ) -> Result)>, Error> where - R: AsyncRead + Send + Unpin, + R: AsyncRead + CarReaderSend + Unpin, { if let Some(buf) = ld_read(buf_reader, buf).await? { let mut cursor = std::io::Cursor::new(buf); @@ -60,6 +62,12 @@ mod tests { use super::*; + #[cfg(target_arch = "wasm32")] + use wasm_bindgen_test::wasm_bindgen_test; + + #[cfg(target_arch = "wasm32")] + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + async fn ld_write<'a, W>(writer: &mut W, bytes: &[u8]) -> Result<(), Error> where W: AsyncWrite + Send + Unpin, @@ -70,7 +78,8 @@ mod tests { Ok(()) } - #[tokio::test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn ld_read_write_good() { let mut buffer = Vec::::new(); ld_write(&mut buffer, b"test bytes").await.unwrap(); @@ -81,7 +90,8 @@ mod tests { assert_eq!(read, b"test bytes"); } - #[tokio::test] + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] + #[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn ld_read_write_fail() { let mut buffer = Vec::::new(); let size = MAX_ALLOC + 1; diff --git a/src/varint.rs b/src/varint.rs new file mode 100644 index 0000000..8f973b6 --- /dev/null +++ b/src/varint.rs @@ -0,0 +1,89 @@ +//! This module has been adapted from the [integer_encoding] crate. The +//! constructs here are mostly unchanged, except that the `Send` bound on the +//! async reader has been made optional in the case that we are compiling for +//! Wasm and deploying to a browser. + +use std::{ + io::{Error as IoError, ErrorKind as IoErrorKind}, + mem::size_of, +}; + +use integer_encoding::VarInt; +use tokio::io::{AsyncRead, AsyncReadExt}; + +use crate::reader::CarReaderSend; + +pub(crate) trait VarIntMaxSize { + fn varint_max_size() -> usize; +} + +impl VarIntMaxSize for VI { + fn varint_max_size() -> usize { + (size_of::() * 8 + 7) / 7 + } +} + +pub const MSB: u8 = 0b1000_0000; + +/// VarIntProcessor encapsulates the logic for decoding a VarInt byte-by-byte. +#[derive(Default)] +pub struct VarIntProcessor { + buf: [u8; 10], + maxsize: usize, + i: usize, +} + +impl VarIntProcessor { + fn new() -> VarIntProcessor { + VarIntProcessor { + maxsize: VI::varint_max_size(), + ..VarIntProcessor::default() + } + } + fn push(&mut self, b: u8) -> Result<(), IoError> { + if self.i >= self.maxsize { + return Err(IoError::new( + IoErrorKind::InvalidData, + "Unterminated varint", + )); + } + self.buf[self.i] = b; + self.i += 1; + Ok(()) + } + fn finished(&self) -> bool { + self.i > 0 && (self.buf[self.i - 1] & MSB == 0) + } + fn decode(&self) -> Option { + Some(VI::decode_var(&self.buf[0..self.i])?.0) + } +} + +pub async fn read_varint_async(reader: &mut R) -> Result +where + V: VarInt, + R: AsyncRead + CarReaderSend + Unpin, +{ + let mut read_buffer = [0_u8; 1]; + let mut p = VarIntProcessor::new::(); + + while !p.finished() { + let read = reader.read(&mut read_buffer).await?; + + // EOF + if read == 0 && p.i == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "Reached EOF", + )); + } + if read == 0 { + break; + } + + p.push(read_buffer[0])?; + } + + p.decode() + .ok_or_else(|| IoError::new(IoErrorKind::UnexpectedEof, "Reached EOF")) +} diff --git a/src/writer.rs b/src/writer.rs index 9f17eb9..f3a28d8 100644 --- a/src/writer.rs +++ b/src/writer.rs @@ -25,11 +25,7 @@ where } } - /// Writes header and stream of data to writer in Car format. - pub async fn write(&mut self, cid: Cid, data: T) -> Result<(), Error> - where - T: AsRef<[u8]>, - { + pub async fn write_header(&mut self) -> Result<(), Error> { if !self.is_header_written { // Write header bytes let header_bytes = self.header.encode()?; @@ -37,6 +33,15 @@ where self.writer.write_all(&header_bytes).await?; self.is_header_written = true; } + Ok(()) + } + + /// Writes header and stream of data to writer in Car format. + pub async fn write(&mut self, cid: Cid, data: T) -> Result<(), Error> + where + T: AsRef<[u8]>, + { + self.write_header().await?; // Write the given block. self.cid_buffer.clear(); diff --git a/tests/car_file_test.rs b/tests/car_file_test.rs index 28340d4..1f548ec 100644 --- a/tests/car_file_test.rs +++ b/tests/car_file_test.rs @@ -1,12 +1,20 @@ use futures::TryStreamExt; use iroh_car::*; -use tokio::fs::{self, File}; use tokio::io::BufReader; -#[tokio::test] +#[cfg(target_arch = "wasm32")] +use wasm_bindgen_test::wasm_bindgen_test; + +#[cfg(target_arch = "wasm32")] +wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + +const TEST_V1_CAR: &[u8] = include_bytes!("testv1.car"); +const CAR_V1_BASIC: &[u8] = include_bytes!("carv1_basic.car"); + +#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] +#[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn roundtrip_carv1_test_file() { - let file = File::open("tests/testv1.car").await.unwrap(); - let buf_reader = BufReader::new(file); + let buf_reader = BufReader::new(TEST_V1_CAR); let car_reader = CarReader::new(buf_reader).await.unwrap(); let header = car_reader.header().clone(); @@ -20,14 +28,13 @@ async fn roundtrip_carv1_test_file() { } writer.finish().await.unwrap(); - let file = fs::read("tests/testv1.car").await.unwrap(); - assert_eq!(file, buffer); + assert_eq!(TEST_V1_CAR, buffer.as_slice()); } -#[tokio::test] +#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] +#[cfg_attr(not(target_arch = "wasm32"), tokio::test)] async fn roundtrip_carv1_basic_fixtures_file() { - let file = File::open("tests/carv1_basic.car").await.unwrap(); - let buf_reader = BufReader::new(file); + let buf_reader = BufReader::new(CAR_V1_BASIC); let car_reader = CarReader::new(buf_reader).await.unwrap(); let header = car_reader.header().clone(); @@ -69,6 +76,5 @@ async fn roundtrip_carv1_basic_fixtures_file() { } writer.finish().await.unwrap(); - let file = fs::read("tests/carv1_basic.car").await.unwrap(); - assert_eq!(file, buffer); + assert_eq!(CAR_V1_BASIC, buffer.as_slice()); } From 2c1fc6271bfcc945f8ee4e007bbe0c470ad4deeb Mon Sep 17 00:00:00 2001 From: b5 Date: Tue, 27 Jun 2023 10:44:10 -0400 Subject: [PATCH 2/7] wasm fixups --- .cargo/config.toml | 2 ++ .github/workflows/ci.yml | 42 ++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 11 ++++++++--- 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..620dbad --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.'cfg(target_arch="wasm32")'] +runner = "wasm-bindgen-test-runner" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ce2413..fadc18e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,3 +155,45 @@ jobs: - uses: actions-rs/cargo@v1 with: command: audit + + run-test-suite-web-wasm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + - name: 'Setup Rust' + run: | + curl -sSf https://sh.rustup.rs | sh -s -- -y + rustup component add clippy + rustup component add rustfmt + - name: 'Install environment packages' + run: | + sudo apt-get update -qqy + sudo apt-get install jq protobuf-compiler cmake + - name: 'Install Rust/WASM test dependencies' + run: | + rustup target install wasm32-unknown-unknown + cargo install toml-cli + WASM_BINDGEN_VERSION=`toml get ./Cargo.lock . | jq '.package | map(select(.name == "wasm-bindgen"))[0].version' | xargs echo` + cargo install wasm-bindgen-cli --vers "$WASM_BINDGEN_VERSION" + shell: bash + # See: https://github.com/SeleniumHQ/selenium/blob/5d108f9a679634af0bbc387e7e3811bc1565912b/.github/actions/setup-chrome/action.yml + - name: 'Setup Chrome and chromedriver' + run: | + wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - + echo "deb http://dl.google.com/linux/chrome/deb/ stable main" | sudo tee -a /etc/apt/sources.list.d/google-chrome.list + sudo apt-get update -qqy + sudo apt-get -qqy install google-chrome-stable + CHROME_VERSION=$(google-chrome-stable --version) + CHROME_FULL_VERSION=${CHROME_VERSION%%.*} + CHROME_MAJOR_VERSION=${CHROME_FULL_VERSION//[!0-9]} + sudo rm /etc/apt/sources.list.d/google-chrome.list + export CHROMEDRIVER_VERSION=`curl -s https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${CHROME_MAJOR_VERSION%%.*}` + curl -L -O "https://chromedriver.storage.googleapis.com/${CHROMEDRIVER_VERSION}/chromedriver_linux64.zip" + unzip chromedriver_linux64.zip && chmod +x chromedriver && sudo mv chromedriver /usr/local/bin + chromedriver -version + shell: bash + - name: 'Run Rust headless browser tests' + working-directory: ./rust + run: CHROMEDRIVER=/usr/local/bin/chromedriver cargo test --target wasm32-unknown-unknown + shell: bash \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 21c41bd..94414f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,8 +5,10 @@ description = "Implementation the car files for iroh" version = "0.2.1" edition = "2021" license = "Apache-2.0/MIT" -rust-version = "1.65" +rust-version = "1.60.0" repository = "https://github.com/n0-computer/iroh-car" +homepage = "https://github.com/n0-computer/iroh-car" +readme="README.md" [dependencies] anyhow = "1" @@ -16,11 +18,14 @@ integer-encoding = { version = "3", features = ["tokio_async"] } libipld = { package = "libipld", version = "0.16", features = ["dag-cbor", "derive"] } libipld-cbor = { package = "libipld-cbor", version = "0.16" } thiserror = "1" -tokio = { version = "1", features = ["io-util"] } +tokio = { version = "^1", features = ["io-util"] } [dev-dependencies] multihash = "0.18" -tokio = { version = "1", features = ["macros", "sync", "rt", "fs", "io-util"] } +tokio = { version = "^1", features = ["macros", "sync", "rt", "io-util"] } [features] default = [] + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies] +wasm-bindgen-test = { version = "^0.3" } \ No newline at end of file From 712c21fa50e6dbd3c04ca1bf244ba28c0f36a2a8 Mon Sep 17 00:00:00 2001 From: b5 Date: Tue, 27 Jun 2023 10:50:39 -0400 Subject: [PATCH 3/7] add Cargo.lock to git --- .github/workflows/ci.yml | 1 + .gitignore | 1 - Cargo.lock | 950 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 951 insertions(+), 1 deletion(-) create mode 100644 Cargo.lock diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fadc18e..db530af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,6 +157,7 @@ jobs: command: audit run-test-suite-web-wasm: + name: Build & Run Browser WASM tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.gitignore b/.gitignore index 699c970..d062239 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ /target /iroh_gateway/test_files/* .env -Cargo.lock diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..ad5000d --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,950 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anyhow" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" + +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "async-trait" +version = "0.1.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.22", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "blake2b_simd" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c2f0dc9a68c6317d884f97cc36cf5a3d20ba14ce404227df55e1af708ab04bc" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "blake2s_simd" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6637f448b9e61dfadbdcbae9a885fadee1f3eaffb1f8d3c1965d3ade8bdfd44f" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "blake3" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729b71f35bd3fa1a4c86b85d32c8b9069ea7fe14f7a53cfabb65f62d4265b888" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cid" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd94671561e36e4e7de75f753f577edafb0e7c05d6e4547229fdf7938fbcd2c3" +dependencies = [ + "core2", + "multibase", + "multihash", + "serde", + "unsigned-varint", +] + +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "constant_time_eq" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6" + +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + +[[package]] +name = "cpufeatures" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" + +[[package]] +name = "data-encoding-macro" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c904b33cc60130e1aeea4956ab803d08a3f4a0ca82d64ed757afac3891f2bb99" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fdf3fce3ce863539ec1d7fd1b6dcc3c645663376b43ed376bbf887733e4f772" +dependencies = [ + "data-encoding", + "syn 1.0.109", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.22", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +dependencies = [ + "async-trait", + "tokio", +] + +[[package]] +name = "iroh-car" +version = "0.2.1" +dependencies = [ + "anyhow", + "cid", + "futures", + "integer-encoding", + "libipld", + "libipld-cbor", + "multihash", + "thiserror", + "tokio", + "wasm-bindgen-test", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "keccak" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "libipld" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1ccd6b8ffb3afee7081fcaec00e1b099fd1c7ccf35ba5729d88538fcc3b4599" +dependencies = [ + "fnv", + "libipld-cbor", + "libipld-cbor-derive", + "libipld-core", + "libipld-json", + "libipld-macro", + "libipld-pb", + "log", + "multihash", + "thiserror", +] + +[[package]] +name = "libipld-cbor" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77d98c9d1747aa5eef1cf099cd648c3fd2d235249f5fed07522aaebc348e423b" +dependencies = [ + "byteorder", + "libipld-core", + "thiserror", +] + +[[package]] +name = "libipld-cbor-derive" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5ba3a729b72973e456a1812b0afe2e176a376c1836cc1528e9fc98ae8cb838" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure", +] + +[[package]] +name = "libipld-core" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5acd707e8d8b092e967b2af978ed84709eaded82b75effe6cb6f6cc797ef8158" +dependencies = [ + "anyhow", + "cid", + "core2", + "multibase", + "multihash", + "thiserror", +] + +[[package]] +name = "libipld-json" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25856def940047b07b25c33d4e66d248597049ab0202085215dc4dca0487731c" +dependencies = [ + "libipld-core", + "multihash", + "serde", + "serde_json", +] + +[[package]] +name = "libipld-macro" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71171c54214f866ae6722f3027f81dff0931e600e5a61e6b1b6a49ca0b5ed4ae" +dependencies = [ + "libipld-core", +] + +[[package]] +name = "libipld-pb" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f2d0f866c4cd5dc9aa8068c429ba478d2882a3a4b70ab56f7e9a0eddf5d16f" +dependencies = [ + "bytes", + "libipld-core", + "quick-protobuf", + "thiserror", +] + +[[package]] +name = "log" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "multibase" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +dependencies = [ + "base-x", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd8a792c1694c6da4f68db0a9d707c72bd260994da179e6030a5dcee00bb815" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "blake3", + "core2", + "digest", + "multihash-derive", + "sha2", + "sha3", + "unsigned-varint", +] + +[[package]] +name = "multihash-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6d4752e6230d8ef7adf7bd5d8c4b1f6561c1014c5ba9a37445ccefe18aa1db" +dependencies = [ + "proc-macro-crate", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "proc-macro-crate" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" +dependencies = [ + "thiserror", + "toml", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-protobuf" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f" +dependencies = [ + "byteorder", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "serde" +version = "1.0.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.22", +] + +[[package]] +name = "serde_json" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2efbeae7acf4eabd6bcdcbd11c92f45231ddda7539edc7806bd1a04a03b24616" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.22", +] + +[[package]] +name = "tokio" +version = "1.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +dependencies = [ + "autocfg", + "bytes", + "pin-project-lite", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.22", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "unsigned-varint" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86a8dc7f45e4c1b0d30e43038c38f274e77af056aa5f74b93c2cf9eb3c1c836" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.22", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.22", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e6e302a7ea94f83a6d09e78e7dc7d9ca7b186bc2829c24a22d0753efd680671" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "scoped-tls", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecb993dd8c836930ed130e020e77d9b2e65dd0fbab1b67c790b0f5d80b11a575" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" From 9ade46271a66407e1f583257da109e09da698fab Mon Sep 17 00:00:00 2001 From: b5 Date: Tue, 27 Jun 2023 10:56:42 -0400 Subject: [PATCH 4/7] fix working dir for wasm test --- .github/workflows/ci.yml | 88 ++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db530af..ec55626 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,49 @@ on: - main jobs: + + build-and-test-wasm: + name: Build and test (WASM + Chrome) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: Swatinem/rust-cache@v2 + - name: 'Setup Rust' + run: | + curl -sSf https://sh.rustup.rs | sh -s -- -y + rustup component add clippy + rustup component add rustfmt + - name: 'Install environment packages' + run: | + sudo apt-get update -qqy + sudo apt-get install jq protobuf-compiler cmake + - name: 'Install Rust/WASM test dependencies' + run: | + rustup target install wasm32-unknown-unknown + cargo install toml-cli + WASM_BINDGEN_VERSION=`toml get ./Cargo.lock . | jq '.package | map(select(.name == "wasm-bindgen"))[0].version' | xargs echo` + cargo install wasm-bindgen-cli --vers "$WASM_BINDGEN_VERSION" + shell: bash + # See: https://github.com/SeleniumHQ/selenium/blob/5d108f9a679634af0bbc387e7e3811bc1565912b/.github/actions/setup-chrome/action.yml + - name: 'Setup Chrome and chromedriver' + run: | + wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - + echo "deb http://dl.google.com/linux/chrome/deb/ stable main" | sudo tee -a /etc/apt/sources.list.d/google-chrome.list + sudo apt-get update -qqy + sudo apt-get -qqy install google-chrome-stable + CHROME_VERSION=$(google-chrome-stable --version) + CHROME_FULL_VERSION=${CHROME_VERSION%%.*} + CHROME_MAJOR_VERSION=${CHROME_FULL_VERSION//[!0-9]} + sudo rm /etc/apt/sources.list.d/google-chrome.list + export CHROMEDRIVER_VERSION=`curl -s https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${CHROME_MAJOR_VERSION%%.*}` + curl -L -O "https://chromedriver.storage.googleapis.com/${CHROMEDRIVER_VERSION}/chromedriver_linux64.zip" + unzip chromedriver_linux64.zip && chmod +x chromedriver && sudo mv chromedriver /usr/local/bin + chromedriver -version + shell: bash + - name: 'Run Rust headless browser tests' + run: CHROMEDRIVER=/usr/local/bin/chromedriver cargo test --target wasm32-unknown-unknown + shell: bash + build_and_test: name: Build and test runs-on: ${{ matrix.os }} @@ -154,47 +197,4 @@ jobs: command: generate-lockfile - uses: actions-rs/cargo@v1 with: - command: audit - - run-test-suite-web-wasm: - name: Build & Run Browser WASM tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: 'Setup Rust' - run: | - curl -sSf https://sh.rustup.rs | sh -s -- -y - rustup component add clippy - rustup component add rustfmt - - name: 'Install environment packages' - run: | - sudo apt-get update -qqy - sudo apt-get install jq protobuf-compiler cmake - - name: 'Install Rust/WASM test dependencies' - run: | - rustup target install wasm32-unknown-unknown - cargo install toml-cli - WASM_BINDGEN_VERSION=`toml get ./Cargo.lock . | jq '.package | map(select(.name == "wasm-bindgen"))[0].version' | xargs echo` - cargo install wasm-bindgen-cli --vers "$WASM_BINDGEN_VERSION" - shell: bash - # See: https://github.com/SeleniumHQ/selenium/blob/5d108f9a679634af0bbc387e7e3811bc1565912b/.github/actions/setup-chrome/action.yml - - name: 'Setup Chrome and chromedriver' - run: | - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" | sudo tee -a /etc/apt/sources.list.d/google-chrome.list - sudo apt-get update -qqy - sudo apt-get -qqy install google-chrome-stable - CHROME_VERSION=$(google-chrome-stable --version) - CHROME_FULL_VERSION=${CHROME_VERSION%%.*} - CHROME_MAJOR_VERSION=${CHROME_FULL_VERSION//[!0-9]} - sudo rm /etc/apt/sources.list.d/google-chrome.list - export CHROMEDRIVER_VERSION=`curl -s https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${CHROME_MAJOR_VERSION%%.*}` - curl -L -O "https://chromedriver.storage.googleapis.com/${CHROMEDRIVER_VERSION}/chromedriver_linux64.zip" - unzip chromedriver_linux64.zip && chmod +x chromedriver && sudo mv chromedriver /usr/local/bin - chromedriver -version - shell: bash - - name: 'Run Rust headless browser tests' - working-directory: ./rust - run: CHROMEDRIVER=/usr/local/bin/chromedriver cargo test --target wasm32-unknown-unknown - shell: bash \ No newline at end of file + command: audit \ No newline at end of file From 2b93acb91bd9351e8aa7bf877afdb52f30a0175c Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 28 Jun 2023 11:36:38 +0200 Subject: [PATCH 5/7] refactor: apply CR --- .github/workflows/ci.yml | 4 ++-- Cargo.lock | 1 - Cargo.toml | 3 +-- src/error.rs | 2 +- src/header.rs | 10 +++++----- src/reader.rs | 16 ++-------------- src/util.rs | 6 +++--- src/varint.rs | 4 +--- 8 files changed, 15 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec55626..e15d255 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: # Force not building debuginfo to save space on disk. RUSTFLAGS: "-C debuginfo=0" RUSTV: ${{ matrix.rust }} - MSRV: "1.65" + MSRV: "1.60" steps: - uses: actions/checkout@master @@ -197,4 +197,4 @@ jobs: command: generate-lockfile - uses: actions-rs/cargo@v1 with: - command: audit \ No newline at end of file + command: audit diff --git a/Cargo.lock b/Cargo.lock index ad5000d..9edd09f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -334,7 +334,6 @@ dependencies = [ "futures", "integer-encoding", "libipld", - "libipld-cbor", "multihash", "thiserror", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 94414f2..02105e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,7 @@ anyhow = "1" cid = "0.10" futures = "0.3" integer-encoding = { version = "3", features = ["tokio_async"] } -libipld = { package = "libipld", version = "0.16", features = ["dag-cbor", "derive"] } -libipld-cbor = { package = "libipld-cbor", version = "0.16" } +ipld = { package = "libipld", version = "0.16", features = ["dag-cbor", "derive"] } thiserror = "1" tokio = { version = "^1", features = ["io-util"] } diff --git a/src/error.rs b/src/error.rs index f228b9e..3579413 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,7 +10,7 @@ pub enum Error { #[error("Io error: {0}")] Io(#[from] std::io::Error), #[error("Cbor encoding error: {0}")] - Cbor(#[from] libipld::error::Error), + Cbor(#[from] ipld::error::Error), #[error("ld read too large {0}")] LdReadTooLarge(usize), } diff --git a/src/header.rs b/src/header.rs index 99a6709..a7a434d 100644 --- a/src/header.rs +++ b/src/header.rs @@ -1,6 +1,6 @@ use cid::Cid; -use libipld::codec::Codec; -use libipld_cbor::DagCborCodec; +use ipld::cbor::DagCborCodec; +use ipld::codec::Codec; use crate::error::Error; @@ -57,7 +57,7 @@ impl CarHeader { } /// CAR file header version 1. -#[derive(Debug, Clone, Default, libipld::DagCbor, PartialEq, Eq)] +#[derive(Debug, Clone, Default, ipld::DagCbor, PartialEq, Eq)] pub struct CarHeaderV1 { #[ipld] pub roots: Vec, @@ -80,8 +80,8 @@ impl From> for CarHeaderV1 { #[cfg(test)] mod tests { - use libipld::codec::{Decode, Encode}; - use libipld_cbor::DagCborCodec; + use ipld::cbor::DagCborCodec; + use ipld::codec::{Decode, Encode}; use multihash::MultihashDigest; use super::*; diff --git a/src/reader.rs b/src/reader.rs index 7e1623f..51f7a9f 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -8,18 +8,6 @@ use crate::{ util::{ld_read, read_node}, }; -#[cfg(not(target_arch = "wasm32"))] -pub trait CarReaderSend: Send {} - -#[cfg(not(target_arch = "wasm32"))] -impl CarReaderSend for S where S: Send {} - -#[cfg(target_arch = "wasm32")] -pub trait CarReaderSend {} - -#[cfg(target_arch = "wasm32")] -impl CarReaderSend for S {} - /// Reads CAR files that are in a BufReader #[derive(Debug)] pub struct CarReader { @@ -30,7 +18,7 @@ pub struct CarReader { impl CarReader where - R: AsyncRead + CarReaderSend + Unpin, + R: AsyncRead + Unpin, { /// Creates a new CarReader and parses the CarHeader pub async fn new(mut reader: R) -> Result { @@ -76,7 +64,7 @@ mod tests { use cid::Cid; use futures::TryStreamExt; - use libipld_cbor::DagCborCodec; + use ipld::cbor::DagCborCodec; use multihash::MultihashDigest; use crate::{header::CarHeaderV1, writer::CarWriter}; diff --git a/src/util.rs b/src/util.rs index a179371..9341574 100644 --- a/src/util.rs +++ b/src/util.rs @@ -2,7 +2,7 @@ use anyhow::Result; use cid::Cid; use tokio::io::{AsyncRead, AsyncReadExt}; -use crate::{reader::CarReaderSend, varint::read_varint_async}; +use crate::varint::read_varint_async; use super::error::Error; @@ -11,7 +11,7 @@ pub(crate) const MAX_ALLOC: usize = 4 * 1024 * 1024; pub(crate) async fn ld_read(mut reader: R, buf: &mut Vec) -> Result, Error> where - R: AsyncRead + CarReaderSend + Unpin, + R: AsyncRead + Unpin, { let length: usize = match read_varint_async(&mut reader).await { Ok(len) => len, @@ -43,7 +43,7 @@ pub(crate) async fn read_node( buf: &mut Vec, ) -> Result)>, Error> where - R: AsyncRead + CarReaderSend + Unpin, + R: AsyncRead + Unpin, { if let Some(buf) = ld_read(buf_reader, buf).await? { let mut cursor = std::io::Cursor::new(buf); diff --git a/src/varint.rs b/src/varint.rs index 8f973b6..a6663bf 100644 --- a/src/varint.rs +++ b/src/varint.rs @@ -11,8 +11,6 @@ use std::{ use integer_encoding::VarInt; use tokio::io::{AsyncRead, AsyncReadExt}; -use crate::reader::CarReaderSend; - pub(crate) trait VarIntMaxSize { fn varint_max_size() -> usize; } @@ -62,7 +60,7 @@ impl VarIntProcessor { pub async fn read_varint_async(reader: &mut R) -> Result where V: VarInt, - R: AsyncRead + CarReaderSend + Unpin, + R: AsyncRead + Unpin, { let mut read_buffer = [0_u8; 1]; let mut p = VarIntProcessor::new::(); From 741e61b32d75dd0bd023d09eb288d149c266d21d Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 28 Jun 2023 11:45:58 +0200 Subject: [PATCH 6/7] fix: use patched integer-encoding-rs --- Cargo.lock | 135 ++++++++++++++++++++++++-------------------------- Cargo.toml | 5 +- src/lib.rs | 1 - src/util.rs | 5 +- src/varint.rs | 87 -------------------------------- 5 files changed, 70 insertions(+), 163 deletions(-) delete mode 100644 src/varint.rs diff --git a/Cargo.lock b/Cargo.lock index 9edd09f..2e2cf24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "anyhow" version = "1.0.71" @@ -37,6 +52,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base-x" version = "0.2.11" @@ -315,11 +345,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "integer-encoding" version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +source = "git+https://github.com/dignifiedquire/integer-encoding-rs?branch=feat-no-send-bound#6a86e902358480e61c410bc51610c34436ae2253" dependencies = [ "async-trait", "tokio", @@ -471,6 +506,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + [[package]] name = "multibase" version = "0.9.1" @@ -513,6 +557,15 @@ dependencies = [ "synstructure", ] +[[package]] +name = "object" +version = "0.30.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -592,6 +645,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "ryu" version = "1.0.13" @@ -721,15 +780,15 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.2" +version = "1.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "374442f06ee49c3a28a8fc9f01a2596fed7559c6b99b31279c3261778e77d84f" dependencies = [ "autocfg", + "backtrace", "bytes", "pin-project-lite", "tokio-macros", - "windows-sys", ] [[package]] @@ -881,69 +940,3 @@ dependencies = [ "js-sys", "wasm-bindgen", ] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/Cargo.toml b/Cargo.toml index 02105e7..7a186f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,4 +27,7 @@ tokio = { version = "^1", features = ["macros", "sync", "rt", "io-util"] } default = [] [target.'cfg(target_arch = "wasm32")'.dev-dependencies] -wasm-bindgen-test = { version = "^0.3" } \ No newline at end of file +wasm-bindgen-test = { version = "^0.3" } + +[patch.crates-io] +integer-encoding = { git = "https://github.com/dignifiedquire/integer-encoding-rs", branch = "feat-no-send-bound" } \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 0f2d383..d4e5f66 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,6 @@ mod error; mod header; mod reader; mod util; -mod varint; mod writer; pub use crate::header::CarHeader; diff --git a/src/util.rs b/src/util.rs index 9341574..d39c2b0 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,9 +1,8 @@ use anyhow::Result; use cid::Cid; +use integer_encoding::VarIntAsyncReader; use tokio::io::{AsyncRead, AsyncReadExt}; -use crate::varint::read_varint_async; - use super::error::Error; /// Maximum size that is used for single node. @@ -13,7 +12,7 @@ pub(crate) async fn ld_read(mut reader: R, buf: &mut Vec) -> Result len, Err(e) => { if e.kind() == std::io::ErrorKind::UnexpectedEof { diff --git a/src/varint.rs b/src/varint.rs deleted file mode 100644 index a6663bf..0000000 --- a/src/varint.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! This module has been adapted from the [integer_encoding] crate. The -//! constructs here are mostly unchanged, except that the `Send` bound on the -//! async reader has been made optional in the case that we are compiling for -//! Wasm and deploying to a browser. - -use std::{ - io::{Error as IoError, ErrorKind as IoErrorKind}, - mem::size_of, -}; - -use integer_encoding::VarInt; -use tokio::io::{AsyncRead, AsyncReadExt}; - -pub(crate) trait VarIntMaxSize { - fn varint_max_size() -> usize; -} - -impl VarIntMaxSize for VI { - fn varint_max_size() -> usize { - (size_of::() * 8 + 7) / 7 - } -} - -pub const MSB: u8 = 0b1000_0000; - -/// VarIntProcessor encapsulates the logic for decoding a VarInt byte-by-byte. -#[derive(Default)] -pub struct VarIntProcessor { - buf: [u8; 10], - maxsize: usize, - i: usize, -} - -impl VarIntProcessor { - fn new() -> VarIntProcessor { - VarIntProcessor { - maxsize: VI::varint_max_size(), - ..VarIntProcessor::default() - } - } - fn push(&mut self, b: u8) -> Result<(), IoError> { - if self.i >= self.maxsize { - return Err(IoError::new( - IoErrorKind::InvalidData, - "Unterminated varint", - )); - } - self.buf[self.i] = b; - self.i += 1; - Ok(()) - } - fn finished(&self) -> bool { - self.i > 0 && (self.buf[self.i - 1] & MSB == 0) - } - fn decode(&self) -> Option { - Some(VI::decode_var(&self.buf[0..self.i])?.0) - } -} - -pub async fn read_varint_async(reader: &mut R) -> Result -where - V: VarInt, - R: AsyncRead + Unpin, -{ - let mut read_buffer = [0_u8; 1]; - let mut p = VarIntProcessor::new::(); - - while !p.finished() { - let read = reader.read(&mut read_buffer).await?; - - // EOF - if read == 0 && p.i == 0 { - return Err(std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "Reached EOF", - )); - } - if read == 0 { - break; - } - - p.push(read_buffer[0])?; - } - - p.decode() - .ok_or_else(|| IoError::new(IoErrorKind::UnexpectedEof, "Reached EOF")) -} From 7db3dae7a5637ee0b0fb2fc5196e90bf1f5bd5ea Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 28 Jun 2023 18:42:48 +0200 Subject: [PATCH 7/7] refactor: switch to unsigned-varint drops integer-encoding, avoiding the dep and using the same dep that multihash already uses --- Cargo.lock | 22 +--------------------- Cargo.toml | 5 +---- src/util.rs | 46 ++++++++++++++++++++++++++++++++++++---------- src/writer.rs | 7 +++---- 4 files changed, 41 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e2cf24..1dc5cf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,17 +35,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" -[[package]] -name = "async-trait" -version = "0.1.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.22", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -351,15 +340,6 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "git+https://github.com/dignifiedquire/integer-encoding-rs?branch=feat-no-send-bound#6a86e902358480e61c410bc51610c34436ae2253" -dependencies = [ - "async-trait", - "tokio", -] - [[package]] name = "iroh-car" version = "0.2.1" @@ -367,11 +347,11 @@ dependencies = [ "anyhow", "cid", "futures", - "integer-encoding", "libipld", "multihash", "thiserror", "tokio", + "unsigned-varint", "wasm-bindgen-test", ] diff --git a/Cargo.toml b/Cargo.toml index 7a186f8..9e33832 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,10 @@ readme="README.md" anyhow = "1" cid = "0.10" futures = "0.3" -integer-encoding = { version = "3", features = ["tokio_async"] } ipld = { package = "libipld", version = "0.16", features = ["dag-cbor", "derive"] } thiserror = "1" tokio = { version = "^1", features = ["io-util"] } +unsigned-varint = "0.7.1" [dev-dependencies] multihash = "0.18" @@ -28,6 +28,3 @@ default = [] [target.'cfg(target_arch = "wasm32")'.dev-dependencies] wasm-bindgen-test = { version = "^0.3" } - -[patch.crates-io] -integer-encoding = { git = "https://github.com/dignifiedquire/integer-encoding-rs", branch = "feat-no-send-bound" } \ No newline at end of file diff --git a/src/util.rs b/src/util.rs index d39c2b0..0e65564 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,7 +1,6 @@ use anyhow::Result; use cid::Cid; -use integer_encoding::VarIntAsyncReader; -use tokio::io::{AsyncRead, AsyncReadExt}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use super::error::Error; @@ -12,16 +11,13 @@ pub(crate) async fn ld_read(mut reader: R, buf: &mut Vec) -> Result len, + let length: usize = match read_varint_usize(&mut reader).await { + Ok(Some(len)) => len, + Ok(None) => return Ok(None), Err(e) => { - if e.kind() == std::io::ErrorKind::UnexpectedEof { - return Ok(None); - } return Err(Error::Parsing(e.to_string())); } }; - if length > MAX_ALLOC { return Err(Error::LdReadTooLarge(length)); } @@ -37,6 +33,37 @@ where Ok(Some(&buf[..length])) } +/// Read a varint from the provided reader. Returns `Ok(None)` on unexpected `EOF`. +pub async fn read_varint_usize( + mut reader: R, +) -> Result, unsigned_varint::io::ReadError> { + let mut b = unsigned_varint::encode::usize_buffer(); + for i in 0..b.len() { + let n = reader.read(&mut b[i..i + 1]).await?; + if n == 0 { + return Ok(None); + } + if unsigned_varint::decode::is_last(b[i]) { + let slice = &b[..=i]; + let (num, _) = unsigned_varint::decode::usize(slice)?; + return Ok(Some(num)); + } + } + Err(unsigned_varint::decode::Error::Overflow)? +} + +/// Write the given number as varint to the provided writer. +pub async fn write_varint_usize( + num: usize, + mut writer: W, +) -> std::io::Result<()> { + let mut buffer = unsigned_varint::encode::usize_buffer(); + let to_write = unsigned_varint::encode::usize(num, &mut buffer); + writer.write_all(to_write).await?; + + Ok(()) +} + pub(crate) async fn read_node( buf_reader: &mut R, buf: &mut Vec, @@ -56,7 +83,6 @@ where #[cfg(test)] mod tests { - use integer_encoding::VarIntAsyncWriter; use tokio::io::{AsyncWrite, AsyncWriteExt}; use super::*; @@ -71,7 +97,7 @@ mod tests { where W: AsyncWrite + Send + Unpin, { - writer.write_varint_async(bytes.len()).await?; + write_varint_usize(bytes.len(), &mut *writer).await?; writer.write_all(bytes).await?; writer.flush().await?; Ok(()) diff --git a/src/writer.rs b/src/writer.rs index f3a28d8..28a378e 100644 --- a/src/writer.rs +++ b/src/writer.rs @@ -1,8 +1,7 @@ use cid::Cid; -use integer_encoding::VarIntAsyncWriter; use tokio::io::{AsyncWrite, AsyncWriteExt}; -use crate::{error::Error, header::CarHeader}; +use crate::{error::Error, header::CarHeader, util::write_varint_usize}; #[derive(Debug)] pub struct CarWriter { @@ -29,7 +28,7 @@ where if !self.is_header_written { // Write header bytes let header_bytes = self.header.encode()?; - self.writer.write_varint_async(header_bytes.len()).await?; + write_varint_usize(header_bytes.len(), &mut self.writer).await?; self.writer.write_all(&header_bytes).await?; self.is_header_written = true; } @@ -50,7 +49,7 @@ where let data = data.as_ref(); let len = self.cid_buffer.len() + data.len(); - self.writer.write_varint_async(len).await?; + write_varint_usize(len, &mut self.writer).await?; self.writer.write_all(&self.cid_buffer).await?; self.writer.write_all(data).await?;