From 0de9083abf32ffdbdf307ae37d9e3408e1e0e74d Mon Sep 17 00:00:00 2001 From: Erica Marigold Date: Tue, 28 May 2024 13:24:19 +0530 Subject: [PATCH] feat(serde): zstd compression & decompression Tests are broken since the generated ZStd file by the CLI does not match the one generated by serde. Not really sure why. --- Cargo.lock | 283 ++++++++++-------- crates/lune-std-serde/Cargo.toml | 1 + .../lune-std-serde/src/compress_decompress.rs | 18 ++ scripts/generate_compression_test_files.luau | 48 +-- tests/serde/compression/files.luau | 5 + tests/serde/compression/roundtrip.luau | 2 +- tests/serde/test-files/loremipsum.txt.zst | Bin 0 -> 128 bytes tests/serde/test-files/uncompressed.json | 21 +- types/serde.luau | 2 +- 9 files changed, 225 insertions(+), 155 deletions(-) create mode 100644 tests/serde/test-files/loremipsum.txt.zst diff --git a/Cargo.lock b/Cargo.lock index cf0e82dd..80f1b773 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,9 +118,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.83" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "arbitrary" @@ -151,13 +151,12 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "async-channel" -version = "2.2.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d4d23bcc79e27423727b36823d86233aad06dfea531837b038394d11e9928" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" dependencies = [ "concurrent-queue", - "event-listener 5.3.0", - "event-listener-strategy 0.5.2", + "event-listener-strategy", "futures-core", "pin-project-lite", ] @@ -174,13 +173,15 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", + "zstd", + "zstd-safe", ] [[package]] name = "async-executor" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b10202063978b3351199d68f8b22c4e47e4b1b822f8d43fd862d5ea8c006b29a" +checksum = "c8828ec6e544c02b0d6691d21ed9f9218d0384a82542855073c2a3f58304aaf0" dependencies = [ "async-task", "concurrent-queue", @@ -189,17 +190,6 @@ dependencies = [ "slab", ] -[[package]] -name = "async-lock" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" -dependencies = [ - "event-listener 4.0.3", - "event-listener-strategy 0.4.0", - "pin-project-lite", -] - [[package]] name = "async-task" version = "4.7.1" @@ -298,12 +288,11 @@ dependencies = [ [[package]] name = "blocking" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "495f7104e962b7356f0aeb34247aca1fe7d2e783b346582db7f2904cb5717e88" +checksum = "703f41c54fc768e63e091340b424302bb1c29ef4aa0c7f10fe849dfb114d29ea" dependencies = [ "async-channel", - "async-lock", "async-task", "futures-io", "futures-lite", @@ -323,9 +312,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -389,9 +378,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" dependencies = [ "jobserver", "libc", @@ -426,9 +415,9 @@ dependencies = [ [[package]] name = "chrono_lc" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa1812634894df89eb9d5075eba0b97d42b4affe477bde7d0db3d2cf8454a800" +checksum = "568e485d6ad62f607516ebb0820ae8cc46361b0870aeb46eb8232440a00b2eb6" dependencies = [ "chrono", "lazy_static", @@ -480,7 +469,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -611,18 +600,18 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crypto-common" @@ -663,7 +652,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -749,7 +738,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -787,11 +776,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "erased-serde" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b73807008a3c7f171cc40312f37d95ef0396e048b5848d775f54b1a4dd4a0d3" +checksum = "24e2389d65ab4fab27dc2a5de7b191e1f6617d1f1c8855c0dc569c94a4cbb18d" dependencies = [ "serde", + "typeid", ] [[package]] @@ -832,16 +822,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "event-listener-strategy" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" -dependencies = [ - "event-listener 4.0.3", - "pin-project-lite", -] - [[package]] name = "event-listener-strategy" version = "0.5.2" @@ -937,7 +917,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -1038,15 +1018,15 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", "http 1.1.0", "indexmap", "slab", @@ -1192,7 +1172,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.4", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -1235,9 +1215,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +checksum = "3d8d52be92d09acc2e01dddb7fde3ad983fc6489c7db4837e605bc3fca4cb63e" dependencies = [ "bytes", "futures-channel", @@ -1369,9 +1349,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.154" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libloading" @@ -1411,9 +1391,9 @@ checksum = "dd1bc4d24ad230d21fb898d1116b1801d7adfc449d42026475862ab48b11e70e" [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" @@ -1425,6 +1405,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.21" @@ -1433,9 +1419,9 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "luau0-src" -version = "0.8.6+luau622" +version = "0.9.1+luau625" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07758c1f5908f7f9dd9109efaf8c66907cc38acf312db03287e7ad2a64b5de1c" +checksum = "39df4ee5bd067cb2363b9f9e5eed0f1cf326fc80508af8ac037eed5b15459c4a" dependencies = [ "cc", ] @@ -1696,9 +1682,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" dependencies = [ "adler", ] @@ -1716,9 +1702,9 @@ dependencies = [ [[package]] name = "mlua" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9bed6bce296397a9d6a86f995dd10a547a4e6949825d45225906bdcbfe7367" +checksum = "e340c022072f3208a4105458286f4985ba5355bfe243c3073afe45cbe9ecf491" dependencies = [ "bstr", "erased-serde", @@ -1751,9 +1737,9 @@ dependencies = [ [[package]] name = "mlua-sys" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16a9ba1dd2c6ac971b204262d434c24d65067038598f0638b64e5dca28d52b8" +checksum = "5552e7e4e22ada0463dfdeee6caf6dc057a189fdc83136408a8f950a5e5c5540" dependencies = [ "cc", "cfg-if", @@ -1826,6 +1812,27 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "object" version = "0.32.2" @@ -1879,9 +1886,9 @@ checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "parking_lot" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -1951,7 +1958,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -1968,9 +1975,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "piper" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +checksum = "464db0c665917b13ebb5d453ccdec4add5658ee1adc7affc7677615356a8afaf" dependencies = [ "atomic-waker", "fastrand", @@ -2009,6 +2016,15 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "proc-macro-crate" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +dependencies = [ + "toml_edit 0.21.1", +] + [[package]] name = "proc-macro-hack" version = "0.5.20+deprecated" @@ -2017,9 +2033,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.82" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" +checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" dependencies = [ "unicode-ident", ] @@ -2040,7 +2056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8021cf59c8ec9c432cfc2526ac6b8aa508ecaf29cd415f271b8406c1b851c3fd" dependencies = [ "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -2428,7 +2444,7 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki 0.102.3", + "rustls-webpki 0.102.4", "subtle", "zeroize", ] @@ -2460,9 +2476,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.3" +version = "0.102.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ "ring", "rustls-pki-types", @@ -2551,9 +2567,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.201" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780f1cebed1629e4753a1a38a3c72d30b97ec044f0aef68cb26650a3c5cf363c" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] @@ -2570,13 +2586,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.201" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -2593,9 +2609,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" dependencies = [ "serde", ] @@ -2795,9 +2811,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.63" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf5be731623ca1a1fb7d8be6f261a3be6d3e2337b8a1f97be944d020c8fcb704" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", @@ -2845,22 +2861,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -2984,7 +3000,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -3039,37 +3055,48 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.12" +version = "0.8.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +checksum = "a4e43f8cc456c9704c851ae29c67e17ef65d2c30017c17a9765b89c382dc8bba" dependencies = [ "indexmap", "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.22.13", ] [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.12" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow 0.5.40", +] + +[[package]] +name = "toml_edit" +version = "0.22.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef" +checksum = "c127785850e8c20836d49732ae6abfa47616e60bf9d9f57c43c250361a9db96c" dependencies = [ "indexmap", "serde", "serde_spanned", "toml_datetime", - "winnow", + "winnow 0.6.8", ] [[package]] @@ -3085,7 +3112,6 @@ dependencies = [ "tokio", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -3106,7 +3132,6 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3120,7 +3145,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -3190,10 +3215,10 @@ dependencies = [ ] [[package]] -name = "typed-arena" -version = "2.0.2" +name = "typeid" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +checksum = "059d83cc991e7a42fc37bd50941885db0888e34209f8cfd9aab07ddec03bc9cf" [[package]] name = "typenum" @@ -3339,7 +3364,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", "wasm-bindgen-shared", ] @@ -3373,7 +3398,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3588,6 +3613,15 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "0.6.8" @@ -3624,29 +3658,15 @@ checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.63", -] +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zip" -version = "1.2.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c700ea425e148de30c29c580c1f9508b93ca57ad31c9f4e96b83c194c37a7a8f" +checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" dependencies = [ "aes", "arbitrary", @@ -3660,12 +3680,11 @@ dependencies = [ "hmac", "indexmap", "lzma-rs", + "num_enum", "pbkdf2", - "rand", "sha1 0.10.6", "thiserror", "time 0.3.36", - "zeroize", "zopfli", "zstd", ] @@ -3681,14 +3700,16 @@ dependencies = [ [[package]] name = "zopfli" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1f48f3508a3a3f2faee01629564400bc12260f6214a056d06a3aaaa6ef0736" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" dependencies = [ + "bumpalo", "crc32fast", + "lockfree-object-pool", "log", + "once_cell", "simd-adler32", - "typed-arena", ] [[package]] diff --git a/crates/lune-std-serde/Cargo.toml b/crates/lune-std-serde/Cargo.toml index 91786ffd..a8110ddf 100644 --- a/crates/lune-std-serde/Cargo.toml +++ b/crates/lune-std-serde/Cargo.toml @@ -21,6 +21,7 @@ async-compression = { version = "0.4", features = [ "deflate", "gzip", "zlib", + "zstd", ] } bstr = "1.9" lz4 = "1.24" diff --git a/crates/lune-std-serde/src/compress_decompress.rs b/crates/lune-std-serde/src/compress_decompress.rs index 3e5aaa02..574e7423 100644 --- a/crates/lune-std-serde/src/compress_decompress.rs +++ b/crates/lune-std-serde/src/compress_decompress.rs @@ -11,8 +11,10 @@ use tokio::{ use async_compression::{ tokio::bufread::{ BrotliDecoder, BrotliEncoder, GzipDecoder, GzipEncoder, ZlibDecoder, ZlibEncoder, + ZstdDecoder, ZstdEncoder, }, Level::Best as CompressionQuality, + Level::Precise as ZstdCompressionQuality, }; /** @@ -24,6 +26,7 @@ pub enum CompressDecompressFormat { GZip, LZ4, ZLib, + ZStd, } #[allow(dead_code)] @@ -63,6 +66,12 @@ impl CompressDecompressFormat { { Some(Self::ZLib) } + + b if b.len() >= 4 + && matches!(u32::from_le_bytes(b[..4].try_into().unwrap()), 0xFD2FB528) => + { + Some(Self::ZStd) + } _ => None, } } @@ -91,6 +100,7 @@ impl<'lua> FromLua<'lua> for CompressDecompressFormat { "gzip" => Ok(Self::GZip), "lz4" => Ok(Self::LZ4), "zlib" => Ok(Self::ZLib), + "zstd" => Ok(Self::ZStd), kind => Err(LuaError::FromLuaConversionError { from: value.type_name(), to: "CompressDecompressFormat", @@ -144,6 +154,10 @@ pub async fn compress<'lua>( let mut encoder = ZlibEncoder::with_quality(reader, CompressionQuality); copy(&mut encoder, &mut bytes).await?; } + CompressDecompressFormat::ZStd => { + let mut encoder = ZstdEncoder::with_quality(reader, ZstdCompressionQuality(22)); + copy(&mut encoder, &mut bytes).await?; + } CompressDecompressFormat::LZ4 => unreachable!(), } @@ -185,6 +199,10 @@ pub async fn decompress<'lua>( let mut decoder = ZlibDecoder::new(reader); copy(&mut decoder, &mut bytes).await?; } + CompressDecompressFormat::ZStd => { + let mut decoder = ZstdDecoder::new(reader); + copy(&mut decoder, &mut bytes).await?; + } CompressDecompressFormat::LZ4 => unreachable!(), } diff --git a/scripts/generate_compression_test_files.luau b/scripts/generate_compression_test_files.luau index 954929a1..d6b590dd 100644 --- a/scripts/generate_compression_test_files.luau +++ b/scripts/generate_compression_test_files.luau @@ -105,6 +105,7 @@ local BIN_BROTLI = if process.os == "macos" then "/opt/homebrew/bin/brotli" else local BIN_GZIP = if process.os == "macos" then "/opt/homebrew/bin/gzip" else "gzip" local BIN_LZ4 = if process.os == "macos" then "/opt/homebrew/bin/lz4" else "lz4" local BIN_ZLIB = if process.os == "macos" then "/opt/homebrew/bin/pigz" else "pigz" +local BIN_ZSTD = if process.os == "macos" then "/opt/homebrew/bin/zstd" else "zstd" local function checkInstalled(program: string, args: { string }?) print("Checking if", program, "is installed") @@ -119,6 +120,7 @@ checkInstalled(BIN_BROTLI, { "--version" }) checkInstalled(BIN_GZIP, { "--version" }) checkInstalled(BIN_LZ4, { "--version" }) checkInstalled(BIN_ZLIB, { "--version" }) +checkInstalled(BIN_ZSTD, { "--version" }) -- Run them to generate files @@ -180,6 +182,14 @@ local OUTPUT_FILES = { process = processNoop, final = INPUT_FILE .. ".z", }, + { + command = BIN_ZSTD, + format = "zstd" :: serde.CompressDecompressFormat, + args = { "--compress", "--ultra", "-22", "--format=zstd", TEMP_FILE }, + output = TEMP_FILE .. ".zst", + process = processNoop, + final = INPUT_FILE .. ".zst", + }, } for _, spec in OUTPUT_FILES do @@ -219,23 +229,25 @@ for _, spec in OUTPUT_FILES do ) end - -- If the newly compressed contents do not match the existing contents, - -- warn the user about this and ask if they want to overwrite the file - local existingContents = fs.readFile(spec.final) - if compressedContents ~= existingContents then - stdio.ewrite("\nCompressed file does not match existing contents!") - stdio.ewrite("\n\nExisting:\n") - stdio.ewrite(stringAsHex(existingContents)) - stdio.ewrite("\n\nCompressed:\n") - stdio.ewrite(hexDiff(existingContents, compressedContents)) - stdio.ewrite("\n\n") - local confirm = stdio.prompt("confirm", "Do you want to continue?") - if confirm == true then - print("Overwriting file!") - else - stdio.ewrite("\n\nAborting...\n") - process.exit(1) - return + if fs.isFile(spec.final) then + -- If the newly compressed contents do not match the existing contents, + -- warn the user about this and ask if they want to overwrite the file + local existingContents = fs.readFile(spec.final) + if compressedContents ~= existingContents then + stdio.ewrite("\nCompressed file does not match existing contents!") + stdio.ewrite("\n\nExisting:\n") + stdio.ewrite(stringAsHex(existingContents)) + stdio.ewrite("\n\nCompressed:\n") + stdio.ewrite(hexDiff(existingContents, compressedContents)) + stdio.ewrite("\n\n") + local confirm = stdio.prompt("confirm", "Do you want to continue?") + if confirm == true then + print("Overwriting file!") + else + stdio.ewrite("\n\nAborting...\n") + process.exit(1) + return + end end end @@ -243,7 +255,7 @@ for _, spec in OUTPUT_FILES do local decompressSuccess, decompressedContents = pcall(serde.decompress, spec.format, compressedContents) if not decompressSuccess then - stdio.ewrite("\nCompressed contents could not be decompressed using serde!") + stdio.ewrite(`\nCompressed contents for format {spec.format} could not be decompressed using serde!`) stdio.ewrite("\n\nCompressed:\n") stdio.ewrite(stringAsHex(compressedContents)) stdio.ewrite("\n\nError:\n") diff --git a/tests/serde/compression/files.luau b/tests/serde/compression/files.luau index 6f32ed01..7a2a8fbc 100644 --- a/tests/serde/compression/files.luau +++ b/tests/serde/compression/files.luau @@ -30,6 +30,11 @@ local TESTS: { Test } = { Source = "tests/serde/test-files/loremipsum.txt", Target = "tests/serde/test-files/loremipsum.txt.z", }, + { + Format = "zstd", + Source = "tests/serde/test-files/loremipsum.txt", + Target = "tests/serde/test-files/loremipsum.txt.zst", + }, } local failed = false diff --git a/tests/serde/compression/roundtrip.luau b/tests/serde/compression/roundtrip.luau index ee78167b..c68d3e02 100644 --- a/tests/serde/compression/roundtrip.luau +++ b/tests/serde/compression/roundtrip.luau @@ -3,7 +3,7 @@ local process = require("@lune/process") local serde = require("@lune/serde") local stdio = require("@lune/stdio") -local FORMATS: { serde.CompressDecompressFormat } = { "brotli", "gzip", "lz4", "zlib" } +local FORMATS: { serde.CompressDecompressFormat } = { "brotli", "gzip", "lz4", "zlib", "zstd" } local FILES: { string } = { "tests/serde/test-files/loremipsum.txt", "tests/serde/test-files/uncompressed.csv", diff --git a/tests/serde/test-files/loremipsum.txt.zst b/tests/serde/test-files/loremipsum.txt.zst new file mode 100644 index 0000000000000000000000000000000000000000..7dd129bae8aba2b949165683c09052e33d91b1eb GIT binary patch literal 128 zcmV-`0Du1|wJ-f-hyj%Y05XXf5TNbo&1#mK)q+ESjTTjjBLfI-*zTsp!vqmkAnzB~62e^)9JnX1PQ_9DFOLc`G&@$G}qa+tmS6($etWHrm?3nQ+w>=Lr iS$R}G+_mVe+38ur9RULX;w4jO1Ji$gSug{K0=ai&w>xqG literal 0 HcmV?d00001 diff --git a/tests/serde/test-files/uncompressed.json b/tests/serde/test-files/uncompressed.json index ef02f7ae..c98a6ee8 100644 --- a/tests/serde/test-files/uncompressed.json +++ b/tests/serde/test-files/uncompressed.json @@ -1,17 +1,30 @@ { "name": "John", "age": 30, - "hobbies": ["reading", "writing", "coding", "👽"], + "hobbies": [ + "reading", + "writing", + "coding", + "👽" + ], "friends": [ { "name": "Ξθής", "age": 28, - "hobbies": ["painting", "hiking", "🦛"] + "hobbies": [ + "painting", + "hiking", + "🦛" + ] }, { "name": "Bob", "age": 35, - "hobbies": ["fishing", "gardening", "🌿"] + "hobbies": [ + "fishing", + "gardening", + "🌿" + ] } ] -} +} \ No newline at end of file diff --git a/types/serde.luau b/types/serde.luau index c4a21d89..ac4ec491 100644 --- a/types/serde.luau +++ b/types/serde.luau @@ -1,6 +1,6 @@ export type EncodeDecodeFormat = "json" | "yaml" | "toml" -export type CompressDecompressFormat = "brotli" | "gzip" | "lz4" | "zlib" +export type CompressDecompressFormat = "brotli" | "gzip" | "lz4" | "zlib" | "zstd" --[=[ @class Serde