feat(serde): zstd compression & decompression

Tests are broken since the generated ZStd file by the CLI does not match the one generated by serde. Not really sure why.

crates/lune-std-serde/Cargo.toml

@@ -21,6 +21,7 @@ async-compression = { version = "0.4", features = [
     "deflate",
     "gzip",
     "zlib",
+    "zstd",
 ] }
 bstr = "1.9"
 lz4 = "1.24"

crates/lune-std-serde/src/compress_decompress.rs

@@ -11,8 +11,10 @@ use tokio::{
 use async_compression::{
     tokio::bufread::{
         BrotliDecoder, BrotliEncoder, GzipDecoder, GzipEncoder, ZlibDecoder, ZlibEncoder,
+        ZstdDecoder, ZstdEncoder,
     },
     Level::Best as CompressionQuality,
+    Level::Precise as ZstdCompressionQuality,
 };
 
 /**
@@ -24,6 +26,7 @@ pub enum CompressDecompressFormat {
     GZip,
     LZ4,
     ZLib,
+    ZStd,
 }
 
 #[allow(dead_code)]
@@ -63,6 +66,12 @@ impl CompressDecompressFormat {
             {
                 Some(Self::ZLib)
             }
+
+            b if b.len() >= 4
+                && matches!(u32::from_le_bytes(b[..4].try_into().unwrap()), 0xFD2FB528) =>
+            {
+                Some(Self::ZStd)
+            }
             _ => None,
         }
     }
@@ -91,6 +100,7 @@ impl<'lua> FromLua<'lua> for CompressDecompressFormat {
                 "gzip" => Ok(Self::GZip),
                 "lz4" => Ok(Self::LZ4),
                 "zlib" => Ok(Self::ZLib),
+                "zstd" => Ok(Self::ZStd),
                 kind => Err(LuaError::FromLuaConversionError {
                     from: value.type_name(),
                     to: "CompressDecompressFormat",
@@ -144,6 +154,10 @@ pub async fn compress<'lua>(
             let mut encoder = ZlibEncoder::with_quality(reader, CompressionQuality);
             copy(&mut encoder, &mut bytes).await?;
         }
+        CompressDecompressFormat::ZStd => {
+            let mut encoder = ZstdEncoder::with_quality(reader, ZstdCompressionQuality(22));
+            copy(&mut encoder, &mut bytes).await?;
+        }
         CompressDecompressFormat::LZ4 => unreachable!(),
     }
 
@@ -185,6 +199,10 @@ pub async fn decompress<'lua>(
             let mut decoder = ZlibDecoder::new(reader);
             copy(&mut decoder, &mut bytes).await?;
         }
+        CompressDecompressFormat::ZStd => {
+            let mut decoder = ZstdDecoder::new(reader);
+            copy(&mut decoder, &mut bytes).await?;
+        }
         CompressDecompressFormat::LZ4 => unreachable!(),
     }
 

scripts/generate_compression_test_files.luau

@@ -105,6 +105,7 @@ local BIN_BROTLI = if process.os == "macos" then "/opt/homebrew/bin/brotli" else
 local BIN_GZIP = if process.os == "macos" then "/opt/homebrew/bin/gzip" else "gzip"
 local BIN_LZ4 = if process.os == "macos" then "/opt/homebrew/bin/lz4" else "lz4"
 local BIN_ZLIB = if process.os == "macos" then "/opt/homebrew/bin/pigz" else "pigz"
+local BIN_ZSTD = if process.os == "macos" then "/opt/homebrew/bin/zstd" else "zstd"
 
 local function checkInstalled(program: string, args: { string }?)
 	print("Checking if", program, "is installed")
@@ -119,6 +120,7 @@ checkInstalled(BIN_BROTLI, { "--version" })
 checkInstalled(BIN_GZIP, { "--version" })
 checkInstalled(BIN_LZ4, { "--version" })
 checkInstalled(BIN_ZLIB, { "--version" })
+checkInstalled(BIN_ZSTD, { "--version" })
 
 -- Run them to generate files
 
@@ -180,6 +182,14 @@ local OUTPUT_FILES = {
 		process = processNoop,
 		final = INPUT_FILE .. ".z",
 	},
+	{
+		command = BIN_ZSTD,
+		format = "zstd" :: serde.CompressDecompressFormat,
+		args = { "--compress", "--ultra", "-22", "--format=zstd", TEMP_FILE },
+		output = TEMP_FILE .. ".zst",
+		process = processNoop,
+		final = INPUT_FILE .. ".zst",
+	},
 }
 
 for _, spec in OUTPUT_FILES do
@@ -219,31 +229,33 @@ for _, spec in OUTPUT_FILES do
 		)
 	end
 
-	-- If the newly compressed contents do not match the existing contents,
-	-- warn the user about this and ask if they want to overwrite the file
-	local existingContents = fs.readFile(spec.final)
-	if compressedContents ~= existingContents then
-		stdio.ewrite("\nCompressed file does not match existing contents!")
-		stdio.ewrite("\n\nExisting:\n")
-		stdio.ewrite(stringAsHex(existingContents))
-		stdio.ewrite("\n\nCompressed:\n")
-		stdio.ewrite(hexDiff(existingContents, compressedContents))
-		stdio.ewrite("\n\n")
-		local confirm = stdio.prompt("confirm", "Do you want to continue?")
-		if confirm == true then
-			print("Overwriting file!")
-		else
-			stdio.ewrite("\n\nAborting...\n")
-			process.exit(1)
-			return
+	if fs.isFile(spec.final) then
+		-- If the newly compressed contents do not match the existing contents,
+		-- warn the user about this and ask if they want to overwrite the file
+		local existingContents = fs.readFile(spec.final)
+		if compressedContents ~= existingContents then
+			stdio.ewrite("\nCompressed file does not match existing contents!")
+			stdio.ewrite("\n\nExisting:\n")
+			stdio.ewrite(stringAsHex(existingContents))
+			stdio.ewrite("\n\nCompressed:\n")
+			stdio.ewrite(hexDiff(existingContents, compressedContents))
+			stdio.ewrite("\n\n")
+			local confirm = stdio.prompt("confirm", "Do you want to continue?")
+			if confirm == true then
+				print("Overwriting file!")
+			else
+				stdio.ewrite("\n\nAborting...\n")
+				process.exit(1)
+				return
+			end
 		end
 	end
 
 	-- Check if the compressed contents can be decompressed using serde
 	local decompressSuccess, decompressedContents =
 		pcall(serde.decompress, spec.format, compressedContents)
 	if not decompressSuccess then
-		stdio.ewrite("\nCompressed contents could not be decompressed using serde!")
+		stdio.ewrite(`\nCompressed contents for format {spec.format} could not be decompressed using serde!`)
 		stdio.ewrite("\n\nCompressed:\n")
 		stdio.ewrite(stringAsHex(compressedContents))
 		stdio.ewrite("\n\nError:\n")

tests/serde/compression/files.luau

@@ -30,6 +30,11 @@ local TESTS: { Test } = {
 		Source = "tests/serde/test-files/loremipsum.txt",
 		Target = "tests/serde/test-files/loremipsum.txt.z",
 	},
+	{
+		Format = "zstd",
+		Source = "tests/serde/test-files/loremipsum.txt",
+		Target = "tests/serde/test-files/loremipsum.txt.zst",
+	},
 }
 
 local failed = false

tests/serde/compression/roundtrip.luau

@@ -3,7 +3,7 @@ local process = require("@lune/process")
 local serde = require("@lune/serde")
 local stdio = require("@lune/stdio")
 
-local FORMATS: { serde.CompressDecompressFormat } = { "brotli", "gzip", "lz4", "zlib" }
+local FORMATS: { serde.CompressDecompressFormat } = { "brotli", "gzip", "lz4", "zlib", "zstd" }
 local FILES: { string } = {
 	"tests/serde/test-files/loremipsum.txt",
 	"tests/serde/test-files/uncompressed.csv",

tests/serde/test-files/uncompressed.json

@@ -1,17 +1,30 @@
 {
     "name": "John",
     "age": 30,
-    "hobbies": ["reading", "writing", "coding", "👽"],
+    "hobbies": [
+        "reading",
+        "writing",
+        "coding",
+        "👽"
+    ],
     "friends": [
         {
             "name": "Ξθής",
             "age": 28,
-            "hobbies": ["painting", "hiking", "🦛"]
+            "hobbies": [
+                "painting",
+                "hiking",
+                "🦛"
+            ]
         },
         {
             "name": "Bob",
             "age": 35,
-            "hobbies": ["fishing", "gardening", "🌿"]
+            "hobbies": [
+                "fishing",
+                "gardening",
+                "🌿"
+            ]
         }
     ]
-}
+}

types/serde.luau

@@ -1,6 +1,6 @@
 export type EncodeDecodeFormat = "json" | "yaml" | "toml"
 
-export type CompressDecompressFormat = "brotli" | "gzip" | "lz4" | "zlib"
+export type CompressDecompressFormat = "brotli" | "gzip" | "lz4" | "zlib" | "zstd"
 
 --[=[
 	@class Serde