Add the Zstandard codec

juntyr · Aug 8, 2024 · 81b2a10 · 81b2a10
1 parent b387190
commit 81b2a10
Show file tree

Hide file tree

Showing 7 changed files with 410 additions and 11 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -8,6 +8,7 @@ members = [
     "codecs/identity",
     "codecs/uniform-noise",
     "codecs/zlib",
+    "codecs/zstd",
 ]
 
 [workspace.package]
@@ -27,6 +28,7 @@ numcodecs-bit-round = { version = "0.1", path = "codecs/bit-round", default-feat
 numcodecs-identity = { version = "0.1", path = "codecs/identity", default-features = false }
 numcodecs-uniform-noise = { version = "0.1", path = "codecs/uniform-noise", default-features = false }
 numcodecs-zlib = { version = "0.1", path = "codecs/zlib", default-features = false }
+numcodecs-zstd = { version = "0.1", path = "codecs/zstd", default-features = false }
 
 # crates.io third-party dependencies
 convert_case = { version = "0.6", default-features = false }
@@ -43,6 +45,8 @@ serde_json = { version = "1.0", default-features = false }
 serde_repr = { version = "0.1", default-features = false }
 thiserror = { version = "1.0", default-features = false }
 wyhash = { version = "0.5", default-features = false }
+zstd = { version = "0.13", default-features = false }
+zstd-sys = { version = "2.0.12", default-features = false }
 
 [workspace.lints.rust]
 unsafe_code = "deny"

diff --git a/codecs/zlib/src/lib.rs b/codecs/zlib/src/lib.rs
@@ -143,10 +143,10 @@ pub enum ZlibCodecError {
     /// [`ZlibCodec`] decode consumed less encoded data, which contains trailing
     /// junk
     #[error("Zlib decode consumed less encoded data, which contains trailing junk")]
-    ZlibDecodeExcessiveEncodedData,
+    DecodeExcessiveEncodedData,
     /// [`ZlibCodec`] produced less decoded data than expected
     #[error("Zlib produced less decoded data than expected")]
-    ZlibDecodeProducedLess,
+    DecodeProducedLess,
     /// [`ZlibCodec`] failed to decode the encoded data
     #[error("Zlib failed to decode the encoded data")]
     ZlibDecodeFailed {
@@ -267,10 +267,10 @@ pub fn compress(array: AnyArrayView, level: ZlibLevel) -> Result<Vec<u8>, ZlibCo
 ///
 /// Errors with
 /// - [`ZlibCodecError::HeaderDecodeFailed`] if decoding the header failed.
-/// - [`ZlibCodecError::ZlibDecodeExcessiveEncodedData`] if the encoded data
+/// - [`ZlibCodecError::DecodeExcessiveEncodedData`] if the encoded data
 ///   contains excessive trailing data junk
-/// - [`ZlibCodecError::ZlibDecodeProducedLess`] if decoding produced less data
-///   than expected
+/// - [`ZlibCodecError::DecodeProducedLess`] if decoding produced less data than
+///   expected
 /// - [`ZlibCodecError::ZlibDecodeFailed`] if an opaque decoding error occurred
 pub fn decompress(encoded: &[u8]) -> Result<AnyArray, ZlibCodecError> {
     let (header, encoded) =
@@ -298,10 +298,10 @@ pub fn decompress(encoded: &[u8]) -> Result<AnyArray, ZlibCodecError> {
 /// - [`ZlibCodecError::MismatchedDecodeIntoShape`] if the `decoded` array is of
 ///   the wrong shape.
 /// - [`ZlibCodecError::HeaderDecodeFailed`] if decoding the header failed.
-/// - [`ZlibCodecError::ZlibDecodeExcessiveEncodedData`] if the encoded data
+/// - [`ZlibCodecError::DecodeExcessiveEncodedData`] if the encoded data
 ///   contains excessive trailing data junk
-/// - [`ZlibCodecError::ZlibDecodeProducedLess`] if decoding produced less data
-///   than expected
+/// - [`ZlibCodecError::DecodeProducedLess`] if decoding produced less data than
+///   expected
 /// - [`ZlibCodecError::ZlibDecodeFailed`] if an opaque decoding error occurred
 pub fn decompress_into(encoded: &[u8], mut decoded: AnyArrayViewMut) -> Result<(), ZlibCodecError> {
     let (header, encoded) =
@@ -340,11 +340,11 @@ fn decompress_into_bytes(encoded: &[u8], decoded: &mut [u8]) -> Result<(), ZlibC
     match status {
         miniz_oxide::inflate::TINFLStatus::Done => {
             if in_consumed != encoded.len() {
-                Err(ZlibCodecError::ZlibDecodeExcessiveEncodedData)
+                Err(ZlibCodecError::DecodeExcessiveEncodedData)
             } else if out_consumed == decoded.len() {
                 Ok(())
             } else {
-                Err(ZlibCodecError::ZlibDecodeProducedLess)
+                Err(ZlibCodecError::DecodeProducedLess)
             }
         }
         status => Err(ZlibCodecError::ZlibDecodeFailed {

diff --git a/codecs/zstd/Cargo.toml b/codecs/zstd/Cargo.toml
@@ -0,0 +1,28 @@
+[package]
+name = "numcodecs-zstd"
+version = "0.1.0"
+edition = { workspace = true }
+authors = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+rust-version = { workspace = true }
+
+description = "Zstandard codec implementation for the numcodecs API"
+readme = "README.md"
+categories = ["compression", "encoding"]
+keywords = ["zstd", "numcodecs", "compression", "encoding"]
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+ndarray = { workspace = true }
+numcodecs = { workspace = true }
+postcard = { workspace = true }
+serde = { workspace = true, features = ["std", "derive"] }
+thiserror = { workspace = true }
+zstd = { workspace = true }
+# Explicitly enable the `no_wasm_shim` feature in zstd/zstd-sys
+zstd-sys = { workspace = true, features = ["no_wasm_shim"] }
+
+[lints]
+workspace = true
diff --git a/codecs/zstd/LICENSE b/codecs/zstd/LICENSE
@@ -0,0 +1 @@
+../../LICENSE
diff --git a/codecs/zstd/README.md b/codecs/zstd/README.md
@@ -0,0 +1,32 @@
+[![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs]
+
+[CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main
+[workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain
+
+[MSRV]: https://img.shields.io/badge/MSRV-1.64.0-blue
+[repo]: https://github.com/juntyr/numcodecs-rs
+
+[Latest Version]: https://img.shields.io/crates/v/numcodecs-zstd
+[crates.io]: https://crates.io/crates/numcodecs-zstd
+
+[Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-zstd
+[docs.rs]: https://docs.rs/numcodecs-zstd/
+
+[Rust Doc Main]: https://img.shields.io/badge/docs-main-blue
+[docs]: https://juntyr.github.io/numcodecs-rs/numcodecs-zstd
+
+# numcodecs-zstd
+
+Zstandard codec implementation for the [`numcodecs`] API.
+
+[`numcodecs`]: https://docs.rs/numcodecs/0.1/numcodecs/
+
+## License
+
+Licensed under the Mozilla Public License, Version 2.0 ([LICENSE](LICENSE) or https://www.mozilla.org/en-US/MPL/2.0/).
+
+## Funding
+
+The `numcodecs-zstd` crate has been developed as part of [ESiWACE3](https://www.esiwace.eu), the third phase of the Centre of Excellence in Simulation of Weather and Climate in Europe.
+
+Funded by the European Union. This work has received funding from the European High Performance Computing Joint Undertaking (JU) under grant agreement No 101093054.