From 4f8231530c9949afcda8b410bdef87bd01102907 Mon Sep 17 00:00:00 2001 From: wllenyj Date: Thu, 4 May 2023 15:47:49 +0800 Subject: [PATCH] fork versionize from https://github.com/firecracker-microvm/versionize Since our PR (https://github.com/firecracker-microvm/versionize_derive/pull/23) can't be merged into upstream for a short time, here we fork it first. After the upstream is merged, switch to the upstream dependency. Signed-off-by: wllenyj --- crates/dbs-versionize/CHANGELOG.md | 43 + crates/dbs-versionize/CONTRIBUTING.md | 53 + crates/dbs-versionize/Cargo.toml | 26 + crates/dbs-versionize/LICENSE | 201 ++ crates/dbs-versionize/README.md | 78 + crates/dbs-versionize/SECURITY-POLICY.md | 22 + .../coverage_config_aarch64.json | 1 + .../coverage_config_x86_64.json | 1 + crates/dbs-versionize/src/crc.rs | 154 ++ crates/dbs-versionize/src/lib.rs | 182 ++ crates/dbs-versionize/src/primitives.rs | 833 ++++++++ crates/dbs-versionize/src/version_map.rs | 309 +++ crates/dbs-versionize/tests/test.rs | 1699 +++++++++++++++++ 13 files changed, 3602 insertions(+) create mode 100644 crates/dbs-versionize/CHANGELOG.md create mode 100644 crates/dbs-versionize/CONTRIBUTING.md create mode 100644 crates/dbs-versionize/Cargo.toml create mode 100644 crates/dbs-versionize/LICENSE create mode 100644 crates/dbs-versionize/README.md create mode 100644 crates/dbs-versionize/SECURITY-POLICY.md create mode 100644 crates/dbs-versionize/coverage_config_aarch64.json create mode 100644 crates/dbs-versionize/coverage_config_x86_64.json create mode 100644 crates/dbs-versionize/src/crc.rs create mode 100644 crates/dbs-versionize/src/lib.rs create mode 100644 crates/dbs-versionize/src/primitives.rs create mode 100644 crates/dbs-versionize/src/version_map.rs create mode 100644 crates/dbs-versionize/tests/test.rs diff --git a/crates/dbs-versionize/CHANGELOG.md b/crates/dbs-versionize/CHANGELOG.md new file mode 100644 index 00000000..0a35b253 --- /dev/null +++ b/crates/dbs-versionize/CHANGELOG.md @@ -0,0 +1,43 @@ +# v0.1.10 + +- Fixed a possible out of bounds memory access in FamStructWrapper::deserialize + +# v0.1.9 + +- Implement Versionize for i128 and u128 + +# v0.1.8 + +- Fixed VersionMap not implementing Sync + Send in 0.1.7 + +# v0.1.7 [yanked] + +- Use caret requirements instead of comparison requirements + for specifying dependencies +- Update vmm-sys-utils to 0.11.0 + +# v0.1.6 + +- Upgraded vmm-sys-utils to v0.8.0 + +# v0.1.5 + +- Added more documentation and examples. + +# v0.1.4 + +- Removed Versionize proc macro support for unions. Serializing unions can lead to undefined behaviour especially when no +layout guarantees are provided. The Versionize trait can still be implemented but only for repr(C) unions and extensive care +and testing is required from the implementer. + +# v0.1.3 + +- Added extra validations in VersionMap::get_type_version(). + +# v0.1.2 + +- Improved edge cases handling for Vec serialization and deserialization. + +# v0.1.0 + +- "versionize" v0.1.0 first release. diff --git a/crates/dbs-versionize/CONTRIBUTING.md b/crates/dbs-versionize/CONTRIBUTING.md new file mode 100644 index 00000000..8fe8968a --- /dev/null +++ b/crates/dbs-versionize/CONTRIBUTING.md @@ -0,0 +1,53 @@ +# Contributing to versionize + +## Contribution Workflow + +The versionize repository uses the “fork-and-pull” development model. Follow +these steps if you want to merge your changes: + +1. Within your fork of + [versionize](https://github.com/firecracker-microvm/versionize), create a + branch for your contribution. Use a meaningful name. +1. Create your contribution, meeting all + [contribution quality standards](#contribution-quality-standards) +1. [Create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) + against the master branch of the versionize repository. +1. Work with your reviewers to address any comments and obtain a + minimum of 2 approvals, at least one of which must be provided by + [a maintainer](MAINTAINERS.md). + To update your pull request amend existing commits whenever applicable and + then push the new changes to your pull request branch. +1. Once the pull request is approved, one of the maintainers will merge it. + +## Request for Comments + +If you just want to receive feedback for a contribution proposal, open an “RFC” +(“Request for Comments”) pull request: + +1. On your fork of + [versionize](https://github.com/firecracker-microvm/versionize), create a + branch for the contribution you want feedback on. Use a meaningful name. +1. Create your proposal based on the existing codebase. +1. [Create a draft pull request](https://github.blog/2019-02-14-introducing-draft-pull-requests/) + against the master branch of the versionize repository. +1. Discuss your proposal with the community on the pull request page (or on any + other channel). Add the conclusion(s) of this discussion to the pull request + page. + +## Contribution Quality Standards + +Most quality and style standards are enforced automatically during integration +testing. Your contribution needs to meet the following standards: + +- Separate each **logical change** into its own commit. +- Each commit must pass all unit & code style tests, and the full pull request + must pass all integration tests. +- Unit test coverage must _increase_ the overall project code coverage. +- Document all your public functions. +- Add a descriptive message for each commit. Follow + [commit message best practices](https://github.com/erlang/otp/wiki/writing-good-commit-messages). +- Document your pull requests. Include the reasoning behind each change. +- Acknowledge versionize's [Apache 2.0 license](LICENSE) and certify that no + part of your contribution contravenes this license by signing off on all your + commits with `git -s`. Ensure that every file in your pull request has a + header referring to the repository license file. diff --git a/crates/dbs-versionize/Cargo.toml b/crates/dbs-versionize/Cargo.toml new file mode 100644 index 00000000..647176ce --- /dev/null +++ b/crates/dbs-versionize/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "versionize" +version = "0.1.10" +license = "Apache-2.0" +authors = ["Amazon Firecracker team "] +description = "A version tolerant serialization/deserialization framework." +readme = "README.md" +repository = "https://github.com/firecracker-microvm/versionize" +keywords = ["serialization", "version"] +categories = ["encoding"] + +[dependencies] +serde = "1.0.27" +serde_derive = "1.0.27" +bincode = "1.2.1" +versionize_derive = "0.1.4" +crc64 = "2.0.0" +vmm-sys-util = "0.11.0" + +[build-dependencies] +proc-macro2 = "1.0" +quote = "1.0" +syn = { version = "1.0.13", features=["default","full"]} + +[badges] +maintenance = { status = "experimental" } diff --git a/crates/dbs-versionize/LICENSE b/crates/dbs-versionize/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/crates/dbs-versionize/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/dbs-versionize/README.md b/crates/dbs-versionize/README.md new file mode 100644 index 00000000..31425a15 --- /dev/null +++ b/crates/dbs-versionize/README.md @@ -0,0 +1,78 @@ +**Versionize is a framework for version tolerant serializion/deserialization of +Rust data structures, designed for usecases that need fast deserialization +times and minimal size overhead. It does not aim to be a generic serialization +framework and only the [bincode](https://crates.io/crates/bincode) backend is +supported.** + +## Important note + +This crate is currently used for cross-version serialization with the +[Firecracker snapshot-restore dev preview][1], but has not been tested for +other use cases. It should be considered **experimental software** outside the +Firecracker context. It’s likely that this crate will see both interface and +implementation changes in the future. + +## Versionize in action + +```rust +extern crate versionize; +extern crate versionize_derive; + +use versionize::{VersionMap, Versionize, VersionizeResult}; +use versionize_derive::Versionize; + +// The test structure is at version 3. +#[derive(Debug, PartialEq, Versionize)] +pub struct Test { + a: u32, + #[version(start = 2, end = 3)] + b: u8, + #[version(start = 3, default_fn = "default_c")] + c: String, +} + +impl Test { + // Default value for field `c`. + // The callback is invoked when deserializing an older version + // where the field did not exist. + fn default_c(_source_version: u16) -> String { + "test_string".to_owned() + } +} + +// Memory to hold the serialization output. +let mut mem = vec![0u8; 512]; +// Create a new version map - it will start at app version 1. +let mut version_map = VersionMap::new(); +// Map structure versions to app version. +version_map + .new_version() // App version 2. + .set_type_version(Test::type_id(), 2) // Struct(2) -> App(2). + .new_version() // App version 3. + .set_type_version(Test::type_id(), 3); // Struct(3) -> App(3). + +let test_struct = Test { + a: 1337, + b: 0xFF, + c: "c_value".to_owned(), +}; + +// Serialize to app version 2 - field c will not be serialized. +test_struct + .serialize(&mut mem.as_mut_slice(), &version_map, 2) + .unwrap(); + +// Deserialize from app version 2 - c should contain the default_fn() return value. +let restored_test_struct = Test::deserialize(&mut mem.as_slice(), &version_map, 2).unwrap(); + +assert_eq!( + restored_test_struct, + Test { + a: 1337, + b: 255, + c: "test_string".to_owned() + } +); +``` + +[1]: https://github.com/firecracker-microvm/firecracker/tree/v0.24.0 \ No newline at end of file diff --git a/crates/dbs-versionize/SECURITY-POLICY.md b/crates/dbs-versionize/SECURITY-POLICY.md new file mode 100644 index 00000000..3d7b9745 --- /dev/null +++ b/crates/dbs-versionize/SECURITY-POLICY.md @@ -0,0 +1,22 @@ +# Security Issue Policy + +If you uncover a security issue with versionize, please write to us on +. + +Once the Firecracker [maintainers](MAINTAINERS.md) become aware (or are made +aware) of a security issue, they will immediately assess it. Based on impact +and complexity, they will determine an embargo period (if externally reported, +the period will be agreed upon with the external party). + +During the embargo period, maintainers will prioritize developing a fix over +other activities. Within this period, maintainers may also notify a limited +number of trusted parties via a pre-disclosure list, providing them with +technical information, a risk assessment, and early access to a fix. + +The external customers are included in this group based on the scale of their +versionize usage in production. The pre-disclosure list may also contain +significant external security contributors that can join the effort to fix the +issue during the embargo period. + +At the end of the embargo period, maintainers will publicly release information +about the security issue together with the versionize patches that mitigate it. diff --git a/crates/dbs-versionize/coverage_config_aarch64.json b/crates/dbs-versionize/coverage_config_aarch64.json new file mode 100644 index 00000000..bfdac81e --- /dev/null +++ b/crates/dbs-versionize/coverage_config_aarch64.json @@ -0,0 +1 @@ +{"coverage_score": 92.4, "exclude_path": "test", "crate_features": ""} diff --git a/crates/dbs-versionize/coverage_config_x86_64.json b/crates/dbs-versionize/coverage_config_x86_64.json new file mode 100644 index 00000000..bfdac81e --- /dev/null +++ b/crates/dbs-versionize/coverage_config_x86_64.json @@ -0,0 +1 @@ +{"coverage_score": 92.4, "exclude_path": "test", "crate_features": ""} diff --git a/crates/dbs-versionize/src/crc.rs b/crates/dbs-versionize/src/crc.rs new file mode 100644 index 00000000..f9d77df7 --- /dev/null +++ b/crates/dbs-versionize/src/crc.rs @@ -0,0 +1,154 @@ +// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Implements readers and writers that compute the CRC64 checksum of the bytes +//! read/written. + +use crc64::crc64; +use std::io::{Read, Write}; + +/// Computes the CRC64 checksum of the read bytes. +/// +/// ``` +/// use std::io::Read; +/// use versionize::crc::CRC64Reader; +/// +/// let buf = vec![1, 2, 3, 4, 5]; +/// let mut read_buf = Vec::new(); +/// let mut slice = buf.as_slice(); +/// +/// // Create a reader from a slice. +/// let mut crc_reader = CRC64Reader::new(&mut slice); +/// +/// let count = crc_reader.read_to_end(&mut read_buf).unwrap(); +/// assert_eq!(crc_reader.checksum(), 0xFB04_60DE_0638_3654); +/// assert_eq!(read_buf, buf); +/// ``` +pub struct CRC64Reader { + reader: T, + crc64: u64, +} + +impl CRC64Reader +where + T: Read, +{ + /// Create a new reader. + pub fn new(reader: T) -> Self { + CRC64Reader { crc64: 0, reader } + } + /// Returns the current checksum value. + pub fn checksum(&self) -> u64 { + self.crc64 + } +} + +impl Read for CRC64Reader +where + T: Read, +{ + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let bytes_read = self.reader.read(buf)?; + self.crc64 = crc64(self.crc64, &buf[..bytes_read]); + Ok(bytes_read) + } +} + +/// Computes the CRC64 checksum of the written bytes. +/// +/// ``` +/// use std::io::Write; +/// use versionize::crc::CRC64Writer; +/// +/// let mut buf = vec![0; 16]; +/// let write_buf = vec![123; 16]; +/// let mut slice = buf.as_mut_slice(); +/// +/// // Create a new writer from slice. +/// let mut crc_writer = CRC64Writer::new(&mut slice); +/// +/// crc_writer.write_all(&write_buf.as_slice()).unwrap(); +/// assert_eq!(crc_writer.checksum(), 0x29D5_3572_1632_6566); +/// assert_eq!(write_buf, buf); +/// ``` +pub struct CRC64Writer { + writer: T, + crc64: u64, +} + +impl CRC64Writer +where + T: Write, +{ + /// Create a new writer. + pub fn new(writer: T) -> Self { + CRC64Writer { crc64: 0, writer } + } + + /// Returns the current checksum value. + pub fn checksum(&self) -> u64 { + self.crc64 + } +} + +impl Write for CRC64Writer +where + T: Write, +{ + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let bytes_written = self.writer.write(buf)?; + self.crc64 = crc64(self.crc64, &buf[..bytes_written]); + Ok(bytes_written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.writer.flush() + } +} + +#[cfg(test)] +mod tests { + use super::{CRC64Reader, CRC64Writer, Read, Write}; + + #[test] + fn test_crc_new() { + let buf = vec![1; 5]; + let mut slice = buf.as_slice(); + let crc_reader = CRC64Reader::new(&mut slice); + assert_eq!(crc_reader.crc64, 0); + assert_eq!(crc_reader.reader, &[1; 5]); + assert_eq!(crc_reader.checksum(), 0); + + let mut buf = vec![0; 5]; + let mut slice = buf.as_mut_slice(); + let crc_writer = CRC64Writer::new(&mut slice); + assert_eq!(crc_writer.crc64, 0); + assert_eq!(crc_writer.writer, &[0; 5]); + assert_eq!(crc_writer.checksum(), 0); + } + + #[test] + fn test_crc_read() { + let buf = vec![1, 2, 3, 4, 5]; + let mut read_buf = vec![0; 16]; + + let mut slice = buf.as_slice(); + let mut crc_reader = CRC64Reader::new(&mut slice); + crc_reader.read_to_end(&mut read_buf).unwrap(); + assert_eq!(crc_reader.checksum(), 0xFB04_60DE_0638_3654); + assert_eq!(crc_reader.checksum(), crc_reader.crc64); + } + + #[test] + fn test_crc_write() { + let mut buf = vec![0; 16]; + let write_buf = vec![123; 16]; + + let mut slice = buf.as_mut_slice(); + let mut crc_writer = CRC64Writer::new(&mut slice); + crc_writer.write_all(write_buf.as_slice()).unwrap(); + crc_writer.flush().unwrap(); + assert_eq!(crc_writer.checksum(), 0x29D5_3572_1632_6566); + assert_eq!(crc_writer.checksum(), crc_writer.crc64); + } +} diff --git a/crates/dbs-versionize/src/lib.rs b/crates/dbs-versionize/src/lib.rs new file mode 100644 index 00000000..e9460da6 --- /dev/null +++ b/crates/dbs-versionize/src/lib.rs @@ -0,0 +1,182 @@ +// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +#![deny(missing_docs)] + +//! Defines a generic interface for version tolerant serialization and +//! implements it for primitive data types using `bincode` as backend. +//! +//! The interface has two components: +//! - `Versionize` trait +//! - `VersionMap` helper +//! +//! `VersionMap` maps individual structure/enum versions to a root version +//! (app version). This mapping is required both when serializing or +//! deserializing structures as it needs to know which version of structure +//! to serialize for a given target app version. +//! +//! `Versionize` trait is implemented for the following primitives: +//! u8, u16, u32, u64, usize, i8, i16, i32, i64, isize, char, f32, f64, +//! String, Vec, Arrays up to 32 elements, Box, Wrapping, Option, +//! FamStructWrapper, and (T, U). +//! +//! Known issues and limitations: +//! - Union serialization is not supported via the `Versionize` proc macro. +//! - Implementing `Versionize` for non-repr(C) unions can result in undefined +//! behaviour and MUST be avoided. +//! - Versionize trait implementations for repr(C) unions must be backed by +//! extensive testing. +//! - Semantic serialization and deserialization is available only for +//! structures. +extern crate bincode; +extern crate crc64; +extern crate serde; +extern crate serde_derive; +extern crate versionize_derive; +extern crate vmm_sys_util; + +pub mod crc; +pub mod primitives; +pub mod version_map; + +use std::any::TypeId; +use std::io::{Read, Write}; +pub use version_map::VersionMap; +use versionize_derive::Versionize; + +/// Versioned serialization/deserialization error definitions. +#[allow(clippy::derive_partial_eq_without_eq)] // FIXME: next major release +#[derive(Debug, PartialEq)] +pub enum VersionizeError { + /// An IO error occured. + Io(i32), + /// Generic serialization error. + Serialize(String), + /// Generic deserialization error. + Deserialize(String), + /// Semantic translation/validation error. + Semantic(String), + /// String length exceeded. + StringLength(usize), + /// Vector length exceeded. + VecLength(usize), +} + +impl std::fmt::Display for VersionizeError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::result::Result<(), std::fmt::Error> { + use VersionizeError::*; + + match self { + Io(e) => write!(f, "An IO error occured: {}", e), + Serialize(e) => write!(f, "A serialization error occured: {}", e), + Deserialize(e) => write!(f, "A deserialization error occured: {}", e), + Semantic(e) => write!(f, "A user generated semantic error occured: {}", e), + StringLength(bad_len) => write!( + f, + "String length exceeded {} > {} bytes", + bad_len, + primitives::MAX_STRING_LEN + ), + VecLength(bad_len) => write!( + f, + "Vec of length {} exceeded maximum size of {} bytes", + bad_len, + primitives::MAX_VEC_SIZE + ), + } + } +} + +/// Versioned serialization/deserialization result. +pub type VersionizeResult = std::result::Result; + +/// Trait that provides an interface for version aware serialization and +/// deserialization. +/// The [Versionize proc macro][1] can generate an implementation for a given +/// type if generics are not used, otherwise a manual implementation is +/// required. +/// +/// Example implementation +/// ``` +/// extern crate versionize; +/// extern crate versionize_derive; +/// use versionize::{VersionMap, Versionize, VersionizeResult}; +/// use versionize_derive::Versionize; +/// +/// struct MyType(T); +/// +/// impl Versionize for MyType +/// where +/// T: Versionize, +/// { +/// #[inline] +/// fn serialize( +/// &self, +/// writer: &mut W, +/// version_map: &VersionMap, +/// app_version: u16, +/// ) -> VersionizeResult<()> { +/// self.0.serialize(writer, version_map, app_version) +/// } +/// +/// #[inline] +/// fn deserialize( +/// reader: &mut R, +/// version_map: &VersionMap, +/// app_version: u16, +/// ) -> VersionizeResult { +/// Ok(MyType(T::deserialize(reader, version_map, app_version)?)) +/// } +/// +/// fn version() -> u16 { +/// 1 +/// } +/// } +/// ``` +/// [1]: https://docs.rs/versionize_derive/latest/versionize_derive/derive.Versionize.html +pub trait Versionize { + /// Serializes `self` to `target_verion` using the specficifed `writer` and + /// `version_map`. + fn serialize( + &self, + writer: &mut W, + version_map: &VersionMap, + target_version: u16, + ) -> VersionizeResult<()>; + + /// Returns a new instance of `Self` by deserializing from `source_version` + /// using the specficifed `reader` and `version_map`. + fn deserialize( + reader: &mut R, + version_map: &VersionMap, + source_version: u16, + ) -> VersionizeResult + where + Self: Sized; + + /// Returns the `Self` type id. + /// The returned ID represents a globally unique identifier for a type. + /// It is required by the `VersionMap` implementation. + fn type_id() -> std::any::TypeId + where + Self: 'static, + { + TypeId::of::() + } + + /// Returns latest `Self` version number. + fn version() -> u16; +} + +#[cfg(test)] +mod tests { + #[test] + fn test_error_debug_display() { + // Validates Debug and Display are implemented. + use VersionizeError::*; + let str = String::from("test"); + format!("{:?}{}", Io(0), Io(0)); + format!("{:?}{}", Serialize(str.clone()), Serialize(str.clone())); + format!("{:?}{}", Deserialize(str.clone()), Deserialize(str.clone())); + format!("{:?}{}", Semantic(str.clone()), Semantic(str)); + } +} diff --git a/crates/dbs-versionize/src/primitives.rs b/crates/dbs-versionize/src/primitives.rs new file mode 100644 index 00000000..f388b0da --- /dev/null +++ b/crates/dbs-versionize/src/primitives.rs @@ -0,0 +1,833 @@ +// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +//! Serialization support for primitive data types. +#![allow(clippy::float_cmp)] + +use self::super::{VersionMap, Versionize, VersionizeError, VersionizeResult}; +use vmm_sys_util::fam::{FamStruct, FamStructWrapper}; + +/// Maximum allowed string len in bytes (16KB). +/// Calling `serialize()` or `deserialiaze()` will fail beyond this limit. +pub const MAX_STRING_LEN: usize = 16384; +/// Maximum allowed vec size in bytes (10MB). +/// Calling `serialize()` or `deserialiaze()` will fail beyond this limit. +pub const MAX_VEC_SIZE: usize = 10_485_760; + +/// A macro that implements the Versionize trait for primitive types using the +/// serde bincode backed. +macro_rules! impl_versionize { + ($ty:ident) => { + impl Versionize for $ty { + #[inline] + fn serialize( + &self, + writer: &mut W, + _version_map: &VersionMap, + _version: u16, + ) -> VersionizeResult<()> { + bincode::serialize_into(writer, &self) + .map_err(|ref err| VersionizeError::Serialize(format!("{:?}", err)))?; + Ok(()) + } + + #[inline] + fn deserialize( + mut reader: &mut R, + _version_map: &VersionMap, + _version: u16, + ) -> VersionizeResult + where + Self: Sized, + { + bincode::deserialize_from(&mut reader) + .map_err(|ref err| VersionizeError::Deserialize(format!("{:?}", err))) + } + + // Not used. + fn version() -> u16 { + 1 + } + } + }; +} + +impl_versionize!(bool); +impl_versionize!(isize); +impl_versionize!(i8); +impl_versionize!(i16); +impl_versionize!(i32); +impl_versionize!(i64); +impl_versionize!(i128); +impl_versionize!(usize); +impl_versionize!(u8); +impl_versionize!(u16); +impl_versionize!(u32); +impl_versionize!(u64); +impl_versionize!(u128); +impl_versionize!(f32); +impl_versionize!(f64); +impl_versionize!(char); + +impl Versionize for String { + #[inline] + fn serialize( + &self, + writer: &mut W, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult<()> { + // It is better to fail early at serialization time. + if self.len() > MAX_STRING_LEN { + return Err(VersionizeError::StringLength(self.len())); + } + + self.len().serialize(writer, version_map, app_version)?; + writer + .write_all(self.as_bytes()) + .map_err(|e| VersionizeError::Io(e.raw_os_error().unwrap_or(0)))?; + Ok(()) + } + + #[inline] + fn deserialize( + mut reader: &mut R, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult { + let len = usize::deserialize(&mut reader, version_map, app_version)?; + // Even if we fail in serialize, we still need to enforce this on the hot path + // in case the len is corrupted. + if len > MAX_STRING_LEN { + return Err(VersionizeError::StringLength(len)); + } + + let mut v = vec![0u8; len]; + reader + .read_exact(v.as_mut_slice()) + .map_err(|e| VersionizeError::Io(e.raw_os_error().unwrap_or(0)))?; + String::from_utf8(v) + .map_err(|err| VersionizeError::Deserialize(format!("Utf8 error: {:?}", err))) + } + + // Not used yet. + fn version() -> u16 { + 1 + } +} + +macro_rules! impl_versionize_array_with_size { + ($ty:literal) => { + impl Versionize for [T; $ty] + where + T: Copy + Default + Versionize, + { + #[inline] + fn serialize( + &self, + writer: &mut W, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult<()> { + for element in self { + element.serialize(writer, version_map, app_version)?; + } + + Ok(()) + } + + #[inline] + fn deserialize( + reader: &mut R, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult { + let mut array = [T::default(); $ty]; + for i in 0..$ty { + array[i] = T::deserialize(reader, version_map, app_version)?; + } + Ok(array) + } + + // Not used yet. + fn version() -> u16 { + 1 + } + } + }; +} + +// Conventionally, traits are available for primitive arrays only up to size 32 +// until the const generics feature is implemented. +// [https://doc.rust-lang.org/std/primitive.array.html] +// [https://github.com/rust-lang/rust/issues/44580] +macro_rules! impl_versionize_arrays { + ($($N:literal)+) => { + $( + impl_versionize_array_with_size!($N); + )+ + } +} + +impl_versionize_arrays! { + 1 2 3 4 5 6 7 8 9 10 + 11 12 13 14 15 16 17 18 19 20 + 21 22 23 24 25 26 27 28 29 30 + 31 32 +} + +impl Versionize for Box +where + T: Versionize, +{ + #[inline] + fn serialize( + &self, + writer: &mut W, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult<()> { + self.as_ref().serialize(writer, version_map, app_version) + } + + #[inline] + fn deserialize( + reader: &mut R, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult { + Ok(Box::new(T::deserialize(reader, version_map, app_version)?)) + } + + // Not used yet. + fn version() -> u16 { + 1 + } +} + +impl Versionize for std::num::Wrapping +where + T: Versionize, +{ + #[inline] + fn serialize( + &self, + writer: &mut W, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult<()> { + self.0.serialize(writer, version_map, app_version) + } + + #[inline] + fn deserialize( + reader: &mut R, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult { + Ok(std::num::Wrapping(T::deserialize( + reader, + version_map, + app_version, + )?)) + } + + // Not used yet. + fn version() -> u16 { + 1 + } +} + +impl Versionize for Option +where + T: Versionize, +{ + #[inline] + fn serialize( + &self, + writer: &mut W, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult<()> { + // Serialize an Option just like bincode does: u8, T. + match self { + Some(value) => { + 1u8.serialize(writer, version_map, app_version)?; + value.serialize(writer, version_map, app_version) + } + None => 0u8.serialize(writer, version_map, app_version), + } + } + + #[inline] + fn deserialize( + reader: &mut R, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult { + let option = u8::deserialize(reader, version_map, app_version)?; + match option { + 0u8 => Ok(None), + 1u8 => Ok(Some(T::deserialize(reader, version_map, app_version)?)), + value => Err(VersionizeError::Deserialize(format!( + "Invalid option value {}", + value + ))), + } + } + + // Not used yet. + fn version() -> u16 { + 1 + } +} + +impl Versionize for Vec +where + T: Versionize, +{ + #[inline] + fn serialize( + &self, + mut writer: &mut W, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult<()> { + if self.len() > MAX_VEC_SIZE / std::mem::size_of::() { + return Err(VersionizeError::VecLength(self.len())); + } + // Serialize in the same fashion as bincode: + // Write len. + bincode::serialize_into(&mut writer, &self.len()) + .map_err(|ref err| VersionizeError::Serialize(format!("{:?}", err)))?; + // Walk the vec and write each element. + for element in self { + element.serialize(writer, version_map, app_version)?; + } + Ok(()) + } + + #[inline] + fn deserialize( + mut reader: &mut R, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult { + let mut v = Vec::new(); + let len: usize = bincode::deserialize_from(&mut reader) + .map_err(|ref err| VersionizeError::Deserialize(format!("{:?}", err)))?; + + if len > MAX_VEC_SIZE / std::mem::size_of::() { + return Err(VersionizeError::VecLength(len)); + } + + for _ in 0..len { + let element: T = T::deserialize(reader, version_map, app_version) + .map_err(|ref err| VersionizeError::Deserialize(format!("{:?}", err)))?; + v.push(element); + } + Ok(v) + } + + // Not used yet. + fn version() -> u16 { + 1 + } +} + +// Implement versioning for FAM structures by using the FamStructWrapper interface. +impl Versionize for FamStructWrapper +where + ::Entry: Versionize, + T: std::fmt::Debug, +{ + #[inline] + fn serialize( + &self, + mut writer: &mut W, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult<()> { + // Write the fixed size header. + self.as_fam_struct_ref() + .serialize(&mut writer, version_map, app_version)?; + // Write the array. + self.as_slice() + .to_vec() + .serialize(&mut writer, version_map, app_version)?; + + Ok(()) + } + + #[inline] + fn deserialize( + reader: &mut R, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult { + let header = T::deserialize(reader, version_map, app_version) + .map_err(|ref err| VersionizeError::Deserialize(format!("{:?}", err)))?; + let entries: Vec<::Entry> = + Vec::deserialize(reader, version_map, app_version) + .map_err(|ref err| VersionizeError::Deserialize(format!("{:?}", err)))?; + + if header.len() != entries.len() { + let msg = format!( + "Mismatch between length of FAM specified in FamStruct header ({}) \ + and actual size of FAM ({})", + header.len(), + entries.len() + ); + + return Err(VersionizeError::Deserialize(msg)); + } + + // Construct the object from the array items. + // Header(T) fields will be initialized by Default trait impl. + let mut object = FamStructWrapper::from_entries(&entries) + .map_err(|ref err| VersionizeError::Deserialize(format!("{:?}", err)))?; + // Update Default T with the deserialized header. + *object.as_mut_fam_struct() = header; + Ok(object) + } + + // Not used. + fn version() -> u16 { + 1 + } +} + +// Manual implementation for tuple of 2 elems. +impl Versionize for (T, U) { + #[inline] + fn serialize( + &self, + writer: &mut W, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult<()> { + self.0.serialize(writer, version_map, app_version)?; + self.1.serialize(writer, version_map, app_version)?; + Ok(()) + } + + #[inline] + fn deserialize( + reader: &mut R, + version_map: &VersionMap, + app_version: u16, + ) -> VersionizeResult { + Ok(( + T::deserialize(reader, version_map, app_version)?, + U::deserialize(reader, version_map, app_version)?, + )) + } + + // Not used yet. + fn version() -> u16 { + 1 + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::undocumented_unsafe_blocks)] + + use super::*; + use super::{VersionMap, Versionize, VersionizeResult}; + + // Generate primitive tests using this macro. + macro_rules! primitive_int_test { + ($ty:ident, $fn_name:ident) => { + #[test] + fn $fn_name() { + let vm = VersionMap::new(); + let mut snapshot_mem = vec![0u8; 64]; + + let store: $ty = std::$ty::MAX; + store + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let restore = + <$ty as Versionize>::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + + assert_eq!(store, restore); + } + }; + } + + primitive_int_test!(usize, test_ser_de_usize); + primitive_int_test!(isize, test_ser_de_isize); + primitive_int_test!(u8, test_ser_de_u8); + primitive_int_test!(u16, test_ser_de_u16); + primitive_int_test!(u32, test_ser_de_u32); + primitive_int_test!(u64, test_ser_de_u64); + primitive_int_test!(u128, test_ser_de_u128); + primitive_int_test!(i8, test_ser_de_i8); + primitive_int_test!(i16, test_ser_de_i16); + primitive_int_test!(i32, test_ser_de_i32); + primitive_int_test!(i64, test_ser_de_i64); + primitive_int_test!(i128, test_ser_de_i128); + primitive_int_test!(f32, test_ser_de_f32); + primitive_int_test!(f64, test_ser_de_f64); + primitive_int_test!(char, test_ser_de_char); + + #[test] + fn test_corrupted_string_len() { + let vm = VersionMap::new(); + let mut buffer = vec![0u8; 1024]; + + let string = String::from("Test string1"); + string + .serialize(&mut buffer.as_mut_slice(), &vm, 1) + .unwrap(); + + // Test corrupt length field. + assert_eq!( + ::deserialize( + &mut buffer.as_slice().split_first().unwrap().1, + &vm, + 1 + ) + .unwrap_err(), + VersionizeError::StringLength(6052837899185946624) + ); + + // Test incomplete string. + assert_eq!( + ::deserialize(&mut buffer.as_slice().split_at(6).0, &vm, 1) + .unwrap_err(), + VersionizeError::Deserialize( + "Io(Error { kind: UnexpectedEof, message: \"failed to fill whole buffer\" })" + .to_owned() + ) + ); + + // Test NULL string len. + buffer[0] = 0; + assert_eq!( + ::deserialize(&mut buffer.as_slice(), &vm, 1).unwrap(), + String::new() + ); + } + + #[test] + fn test_corrupted_vec_len() { + let vm = VersionMap::new(); + let mut buffer = vec![0u8; 1024]; + + let mut string = String::from("Test string1"); + let vec = unsafe { string.as_mut_vec() }; + vec.serialize(&mut buffer.as_mut_slice(), &vm, 1).unwrap(); + + // Test corrupt length field. + assert_eq!( + as Versionize>::deserialize( + &mut buffer.as_slice().split_first().unwrap().1, + &vm, + 1 + ) + .unwrap_err(), + VersionizeError::VecLength(6052837899185946624) + ); + + // Test incomplete Vec. + assert_eq!( + as Versionize>::deserialize(&mut buffer.as_slice().split_at(6).0, &vm, 1) + .unwrap_err(), + VersionizeError::Deserialize( + "Io(Error { kind: UnexpectedEof, message: \"failed to fill whole buffer\" })" + .to_owned() + ) + ); + + // Test NULL Vec len. + buffer[0] = 0; + assert_eq!( + as Versionize>::deserialize(&mut buffer.as_slice(), &vm, 1).unwrap(), + Vec::new() + ); + } + + #[test] + fn test_ser_de_u32_tuple() { + let vm = VersionMap::new(); + let mut snapshot_mem = vec![0u8; 64]; + + let store: (u32, u32) = (std::u32::MIN, std::u32::MAX); + store + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let restore = + <(u32, u32) as Versionize>::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + + assert_eq!(store, restore); + } + + #[derive(Debug, serde_derive::Deserialize, PartialEq, serde_derive::Serialize, Versionize)] + enum CompatibleEnum { + A, + B(String), + C(u64, u64, char), + } + + #[derive(Debug, serde_derive::Deserialize, PartialEq, serde_derive::Serialize, Versionize)] + struct TestCompatibility { + _string: String, + _array: [u8; 32], + _u8: u8, + _u16: u16, + _u32: u32, + _u64: u64, + _u128: u128, + _i8: i8, + _i16: i16, + _i32: i32, + _i64: i64, + _i128: i128, + _f32: f32, + _f64: f64, + _usize: usize, + _isize: isize, + _vec: Vec, + _option: Option, + _enums: Vec, + #[allow(clippy::box_collection)] // we want to test boxes explicitly + _box: Box, + } + + #[test] + fn test_bincode_deserialize_from_versionize() { + let mut snapshot_mem = vec![0u8; 4096]; + let vm = VersionMap::new(); + + let test_struct = TestCompatibility { + _string: "String".to_owned(), + _array: [128u8; 32], + _u8: 1, + _u16: 32000, + _u32: 0x1234_5678, + _u64: 0x1234_5678_9875_4321, + _u128: 0x1234_5678_1234_5678_1234_5678_1234_5678, + _i8: -1, + _i16: -32000, + _i32: -0x1234_5678, + _i64: -0x1234_5678_9875_4321, + _i128: -0x1234_5678_9098_7654_3212_3456_7890_9876, + _usize: 0x1234_5678_9875_4321, + _isize: -0x1234_5678_9875_4321, + _f32: 0.123, + _f64: 0.123_456_789_000_000, + _vec: vec![33; 32], + _option: Some(true), + _enums: vec![ + CompatibleEnum::A, + CompatibleEnum::B("abcd".to_owned()), + CompatibleEnum::C(1, 2, 'a'), + ], + _box: Box::new("Box".to_owned()), + }; + + Versionize::serialize(&test_struct, &mut snapshot_mem.as_mut_slice(), &vm, 1).unwrap(); + + let restored_state: TestCompatibility = + bincode::deserialize_from(snapshot_mem.as_slice()).unwrap(); + assert_eq!(test_struct, restored_state); + } + + #[test] + fn test_bincode_serialize_to_versionize() { + let mut snapshot_mem = vec![0u8; 4096]; + let vm = VersionMap::new(); + + let test_struct = TestCompatibility { + _string: "String".to_owned(), + _array: [128u8; 32], + _u8: 1, + _u16: 32000, + _u32: 0x1234_5678, + _u64: 0x1234_5678_9875_4321, + _u128: 0x1234_1234_1234_1234_1234_1234_1234_1234, + _i8: -1, + _i16: -32000, + _i32: -0x1234_5678, + _i64: -0x1234_5678_9875_4321, + _i128: -0x1234_1234_1234_1234_1234_1234_1234_1234, + _usize: 0x1234_5678_9875_4321, + _isize: -0x1234_5678_9875_4321, + _f32: 0.123, + _f64: 0.123_456_789_000_000, + _vec: vec![33; 32], + _option: Some(true), + _enums: vec![ + CompatibleEnum::A, + CompatibleEnum::B("abcd".to_owned()), + CompatibleEnum::C(1, 2, 'a'), + ], + _box: Box::new("Box".to_owned()), + }; + + bincode::serialize_into(snapshot_mem.as_mut_slice(), &test_struct).unwrap(); + + let restored_state: TestCompatibility = + Versionize::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + assert_eq!(test_struct, restored_state); + } + + #[test] + fn test_ser_de_bool() { + let vm = VersionMap::new(); + let mut snapshot_mem = vec![0u8; 64]; + + let store = true; + store + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let restore = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + + assert_eq!(store, restore); + } + + #[test] + fn test_ser_de_string() { + let vm = VersionMap::new(); + let mut snapshot_mem = vec![0u8; 64]; + + let store = String::from("test string"); + store + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let restore = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + + assert_eq!(store, restore); + } + + #[test] + fn test_ser_de_vec() { + let vm = VersionMap::new(); + let mut snapshot_mem = vec![0u8; 64]; + + let store = vec![ + "test 1".to_owned(), + "test 2".to_owned(), + "test 3".to_owned(), + ]; + + store + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let restore = + as Versionize>::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + + assert_eq!(store, restore); + } + + #[test] + fn test_ser_de_option() { + let vm = VersionMap::new(); + let mut snapshot_mem = vec![0u8; 64]; + let mut store = Some("test".to_owned()); + + store + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let mut restore = + as Versionize>::deserialize(&mut snapshot_mem.as_slice(), &vm, 1) + .unwrap(); + assert_eq!(store, restore); + + // Check that ser_de also works for `None` variant. + store = None; + store + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + restore = as Versionize>::deserialize(&mut snapshot_mem.as_slice(), &vm, 1) + .unwrap(); + assert_eq!(store, restore); + + store = Some("test".to_owned()); + store + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + // Corrupt `snapshot_mem` by changing the most significant bit to a value different than 0 or 1. + snapshot_mem[0] = 2; + let restore_err = + as Versionize>::deserialize(&mut snapshot_mem.as_slice(), &vm, 1) + .unwrap_err(); + assert_eq!( + restore_err, + VersionizeError::Deserialize("Invalid option value 2".to_string()) + ); + // Corrupt `snapshot_mem` by changing the most significant bit from 1 (`Some(type)`) to 0 (`None`). + snapshot_mem[0] = 0; + restore = as Versionize>::deserialize(&mut snapshot_mem.as_slice(), &vm, 1) + .unwrap(); + assert_ne!(store, restore); + } + + #[test] + fn test_ser_de_box() { + let vm = VersionMap::new(); + let mut snapshot_mem = vec![0u8; 64]; + + let store = Box::new("test".to_owned()); + + store + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let restore = + as Versionize>::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + + assert_eq!(store, restore); + } + + #[test] + fn test_ser_de_wrapping() { + let vm = VersionMap::new(); + let mut snapshot_mem = vec![0u8; 64]; + + let store = std::num::Wrapping(1337u32); + + store + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let restore = as Versionize>::deserialize( + &mut snapshot_mem.as_slice(), + &vm, + 1, + ) + .unwrap(); + + assert_eq!(store, restore); + } + + #[test] + fn test_vec_limit() { + // We need extra 8 bytes for vector len. + let mut snapshot_mem = vec![0u8; MAX_VEC_SIZE + 8]; + let err = vec![123u8; MAX_VEC_SIZE + 1] + .serialize(&mut snapshot_mem.as_mut_slice(), &VersionMap::new(), 1) + .unwrap_err(); + assert_eq!(err, VersionizeError::VecLength(MAX_VEC_SIZE + 1)); + assert_eq!( + format!("{}", err), + "Vec of length 10485761 exceeded maximum size of 10485760 bytes" + ); + } + + #[test] + fn test_string_limit() { + // We need extra 8 bytes for string len. + let mut snapshot_mem = vec![0u8; MAX_STRING_LEN + 8]; + let err = String::from_utf8(vec![123u8; MAX_STRING_LEN + 1]) + .unwrap() + .serialize(&mut snapshot_mem.as_mut_slice(), &VersionMap::new(), 1) + .unwrap_err(); + assert_eq!(err, VersionizeError::StringLength(MAX_STRING_LEN + 1)); + assert_eq!( + format!("{}", err), + "String length exceeded 16385 > 16384 bytes" + ); + } +} diff --git a/crates/dbs-versionize/src/version_map.rs b/crates/dbs-versionize/src/version_map.rs new file mode 100644 index 00000000..7fbc56e2 --- /dev/null +++ b/crates/dbs-versionize/src/version_map.rs @@ -0,0 +1,309 @@ +// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A helper to map struct and enum versions to a sequence of root versions. +//! This helper is required to support the versioning of a hierarchy of +//! structures composed of individually versioned structures or enums. +//! +//! ```rust +//! extern crate versionize; +//! extern crate versionize_derive; +//! +//! use versionize::{VersionMap, Versionize, VersionizeResult}; +//! use versionize_derive::Versionize; +//! +//! #[derive(Versionize)] +//! pub struct Struct1 { +//! a: u32, +//! #[version(start = 2)] +//! b: u8, +//! } +//! +//! #[derive(Versionize)] +//! pub struct Struct2 { +//! x: u32, +//! #[version(start = 2)] +//! y: u8, +//! } +//! +//! #[derive(Versionize)] +//! pub struct State { +//! struct1: Struct1, +//! struct2: Struct2, +//! } +//! +//! let mut version_map = VersionMap::new(); // +//! version_map +//! .new_version() +//! .set_type_version(Struct1::type_id(), 2) +//! .new_version() +//! .set_type_version(Struct2::type_id(), 2); +//! +//! // Check that there are 3 root versions. +//! assert_eq!(version_map.latest_version(), 3); +//! +//! // Check that root version 1 has all structs at version 1. +//! assert_eq!(version_map.get_type_version(1, Struct1::type_id()), 1); +//! assert_eq!(version_map.get_type_version(1, Struct2::type_id()), 1); +//! assert_eq!(version_map.get_type_version(1, State::type_id()), 1); +//! +//! // Check that root version 2 has Struct1 at version 2 and Struct2 +//! // at version 1. +//! assert_eq!(version_map.get_type_version(2, Struct1::type_id()), 2); +//! assert_eq!(version_map.get_type_version(2, Struct2::type_id()), 1); +//! assert_eq!(version_map.get_type_version(2, State::type_id()), 1); +//! +//! // Check that root version 3 has Struct1 and Struct2 at version 2. +//! assert_eq!(version_map.get_type_version(3, Struct1::type_id()), 2); +//! assert_eq!(version_map.get_type_version(3, Struct2::type_id()), 2); +//! assert_eq!(version_map.get_type_version(3, State::type_id()), 1); +//! ``` + +use std::any::TypeId; +use std::collections::hash_map::HashMap; +use std::fmt::Debug; +use std::sync::Arc; + +const BASE_VERSION: u16 = 1; + +/// Trait to check whether is specific `version` is supported by a `VersionMap`. +pub trait VersionFilter: Debug { + /// Check whether the `version` is supported or not. + fn is_supported(&self, version: u16) -> bool; +} + +impl VersionFilter for () { + fn is_supported(&self, _version: u16) -> bool { + true + } +} +/// +/// The VersionMap API provides functionality to define the version for each +/// type and attach them to specific root versions. +#[derive(Clone, Debug)] +pub struct VersionMap { + versions: Vec>, + filter: Arc, +} + +impl Default for VersionMap { + fn default() -> Self { + VersionMap { + versions: vec![HashMap::new(); 1], + filter: Arc::new(()), + } + } +} + +impl VersionMap { + /// Create a new version map initialized at version 1. + pub fn new() -> Self { + Default::default() + } + + /// Create a new version map with specified version filter. + pub fn with_filter(filter: Arc) -> Self { + VersionMap { + versions: vec![HashMap::new(); 1], + filter, + } + } + + /// Bumps root version by 1 to create a new root version. + pub fn new_version(&mut self) -> &mut Self { + self.versions.push(HashMap::new()); + self + } + + /// Define a mapping between a specific type version and the latest root version. + pub fn set_type_version(&mut self, type_id: TypeId, type_version: u16) -> &mut Self { + // It is safe to unwrap since `self.versions` always has at least 1 element. + self.versions + .last_mut() + .unwrap() + .insert(type_id, type_version); + self + } + + /// Returns the version of `type_id` corresponding to the specified `root_version`. + /// If `root_version` is out of range returns the version of `type_id` at latest version. + pub fn get_type_version(&self, root_version: u16, type_id: TypeId) -> u16 { + let version_space = if root_version > self.latest_version() || root_version == 0 { + self.versions.as_slice() + } else { + self.versions.split_at(root_version as usize).0 + }; + + for i in (0..version_space.len()).rev() { + if let Some(version) = version_space[i].get(&type_id) { + return *version; + } + } + + BASE_VERSION + } + + /// Returns the latest version. + pub fn latest_version(&self) -> u16 { + self.versions.len() as u16 + } + + /// Check whether the `version` is supported by the version map. + pub fn is_supported(&self, version: u16) -> bool { + if version == 0 || version > self.latest_version() { + false + } else { + self.filter.is_supported(version) + } + } +} + +#[cfg(test)] +mod tests { + use super::{TypeId, VersionMap, BASE_VERSION}; + use std::sync::Arc; + use version_map::VersionFilter; + + pub struct MyType; + pub struct MySecondType; + pub struct MyThirdType; + + #[derive(Debug)] + struct MyFilter; + + impl VersionFilter for MyFilter { + fn is_supported(&self, version: u16) -> bool { + version < 5 + } + } + + #[test] + fn test_default_version() { + let vm = VersionMap::new(); + assert_eq!(vm.latest_version(), 1); + } + + #[test] + fn test_new_versions() { + let mut vm = VersionMap::new(); + vm.new_version().new_version(); + assert_eq!(vm.latest_version(), 3); + } + + #[test] + fn test_1_app_version() { + let mut vm = VersionMap::new(); + vm.set_type_version(TypeId::of::(), 1); + vm.set_type_version(TypeId::of::(), 2); + vm.set_type_version(TypeId::of::(), 3); + + assert_eq!(vm.get_type_version(1, TypeId::of::()), 1); + assert_eq!(vm.get_type_version(1, TypeId::of::()), 2); + assert_eq!(vm.get_type_version(1, TypeId::of::()), 3); + } + + #[test] + fn test_100_app_version_full() { + let mut vm = VersionMap::new(); + + for i in 1..=100 { + vm.set_type_version(TypeId::of::(), i) + .set_type_version(TypeId::of::(), i + 1) + .set_type_version(TypeId::of::(), i + 2) + .new_version(); + } + + for i in 1..=100 { + assert_eq!(vm.get_type_version(i, TypeId::of::()), i); + assert_eq!(vm.get_type_version(i, TypeId::of::()), i + 1); + assert_eq!(vm.get_type_version(i, TypeId::of::()), i + 2); + } + } + + #[test] + fn test_version_map_is_send_and_sync() { + fn assert_send_sync() {} + + assert_send_sync::(); + } + + #[test] + fn test_app_versions_with_gap() { + let my_type_id = TypeId::of::(); + let my_second_type_id = TypeId::of::(); + let my_third_type_id = TypeId::of::(); + + let mut vm = VersionMap::new(); + vm.set_type_version(my_type_id, 1); + vm.set_type_version(my_second_type_id, 1); + vm.set_type_version(my_third_type_id, 1); + vm.new_version(); + vm.set_type_version(my_type_id, 2); + vm.new_version(); + vm.set_type_version(my_third_type_id, 2); + vm.new_version(); + vm.set_type_version(my_second_type_id, 2); + + assert_eq!(vm.get_type_version(1, my_type_id), 1); + assert_eq!(vm.get_type_version(1, my_second_type_id), 1); + assert_eq!(vm.get_type_version(1, my_third_type_id), 1); + + assert_eq!(vm.get_type_version(2, my_type_id), 2); + assert_eq!(vm.get_type_version(2, my_second_type_id), 1); + assert_eq!(vm.get_type_version(2, my_third_type_id), 1); + + assert_eq!(vm.get_type_version(3, my_type_id), 2); + assert_eq!(vm.get_type_version(3, my_second_type_id), 1); + assert_eq!(vm.get_type_version(3, my_third_type_id), 2); + + assert_eq!(vm.get_type_version(4, my_type_id), 2); + assert_eq!(vm.get_type_version(4, my_second_type_id), 2); + assert_eq!(vm.get_type_version(4, my_third_type_id), 2); + } + + #[test] + fn test_unset_type() { + let vm = VersionMap::new(); + assert_eq!(vm.get_type_version(1, TypeId::of::()), BASE_VERSION); + } + + #[test] + fn test_invalid_root_version() { + let mut vm = VersionMap::new(); + vm.new_version().set_type_version(TypeId::of::(), 2); + + assert_eq!(vm.get_type_version(0, TypeId::of::()), 2); + + assert_eq!(vm.latest_version(), 2); + assert_eq!(vm.get_type_version(129, TypeId::of::()), 2); + assert_eq!(vm.get_type_version(1, TypeId::of::()), BASE_VERSION); + } + + #[test] + fn test_version_filter() { + let mut vm = VersionMap::default(); + vm.new_version(); + + assert!(!vm.is_supported(0)); + assert!(vm.is_supported(1)); + assert!(vm.is_supported(2)); + assert!(!vm.is_supported(3)); + + let mut vm = VersionMap::with_filter(Arc::new(MyFilter)); + vm.new_version(); + vm.new_version(); + vm.new_version(); + vm.new_version(); + vm.new_version(); + + let vm1 = vm.clone(); + assert!(!vm1.is_supported(0)); + assert!(vm1.is_supported(1)); + assert!(vm1.is_supported(2)); + assert!(vm1.is_supported(3)); + assert!(vm1.is_supported(4)); + assert!(!vm1.is_supported(5)); + assert!(!vm1.is_supported(6)); + assert_eq!(vm.latest_version(), 6); + } +} diff --git a/crates/dbs-versionize/tests/test.rs b/crates/dbs-versionize/tests/test.rs new file mode 100644 index 00000000..75e3cd11 --- /dev/null +++ b/crates/dbs-versionize/tests/test.rs @@ -0,0 +1,1699 @@ +// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![allow(clippy::missing_safety_doc)] + +extern crate versionize; +extern crate versionize_derive; +extern crate vmm_sys_util; + +use std::fmt::{Debug, Formatter, Result}; +use std::num::Wrapping; + +use vmm_sys_util::fam::{FamStruct, FamStructWrapper}; +use vmm_sys_util::generate_fam_struct_impl; + +use versionize::{VersionMap, Versionize, VersionizeError, VersionizeResult}; +use versionize_derive::Versionize; + +#[derive(Debug, PartialEq, Versionize, Eq)] +pub enum TestState { + Zero, + One(u32), + #[version(start = 2, default_fn = "default_state_two")] + Two(u64), +} + +impl TestState { + fn default_state_two(&self, target_version: u16) -> VersionizeResult { + match target_version { + 1 => Ok(TestState::One(2)), + i => Err(VersionizeError::Serialize(format!( + "Unknown target version: {}", + i + ))), + } + } +} + +#[test] +fn test_hardcoded_struct_deserialization() { + // We are testing representation compatibility between versions, at the `versionize` + // crate level, by checking that only the newly added/removed fields changes between + // versions are reflected in the hardcoded snapshot. + + #[rustfmt::skip] + let v1_hardcoded_snapshot: &[u8] = &[ + // usize field (8 bytes), u16 field (2 bytes) + + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, + // u64 (8 bytes), i8 (1 byte), i32 (4 bytes) + + 0xCD, 0xAB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0x20, 0x00, 0x00, 0x00, + // f32 (4 bytes), f64 (8 bytes), char (1 bytes) + + 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x50, 0x40, 0x61, + // String len (8 bytes) + + 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // actual String (11 bytes in our case) + + 0x73, 0x6F, 0x6D, 0x65, 0x5F, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67, + // enum variant number (4 bytes) + value of that variant (in this case it is + // of u32 type -> 4 bytes) + + 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + // Option variant (1 byte) + value of variant (u8 -> 1 byte) + + 0x01, 0x81, + // Box: String len (8 bytes) + actual String (17 bytes in this case). + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x6F, 0x6D, 0x65, 0x5F, + 0x6F, 0x74, 0x68, 0x65, 0x72, 0x5F, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67, + ]; + + // At version 2 isize (8 bytes), i64 (8 bytes) and bool (1 byte) fields will be also + // present. At v2 there is also a new variant available for enum, so we can store that in + // memory and it occupies 4 more bytes than the one stored at v1. + #[rustfmt::skip] + let v2_hardcoded_snapshot: &[u8] = &[ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // New isize field. + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, + 0xCD, 0xAB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0x20, 0x00, 0x00, 0x00, + // New i64 field. + 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x50, 0x40, 0x61, + // New bool field. + 0x01, + 0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x73, 0x6F, 0x6D, 0x65, 0x5F, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67, + // New available enum variant. + 0x02, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x81, + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x6F, 0x6D, 0x65, 0x5F, + 0x6F, 0x74, 0x68, 0x65, 0x72, 0x5F, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67, + ]; + + // At version 3, u64 and i64 disappear (16 bytes) and Vec (8 + 4 = 12 bytes) and Wrapping + // (4 bytes) fields are available. + #[rustfmt::skip] + let v3_hardcoded_snapshot: &[u8] = &[ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, + 0xFF, 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x50, 0x40, 0x61, + 0x01, + 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x73, 0x6F, 0x6D, 0x65, 0x5F, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67, + 0x02, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x81, + // Vec len (8 bytes) + actual Vec (4 bytes). + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x61, 0x61, 0x61, + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x6F, 0x6D, 0x65, 0x5F, + 0x6F, 0x74, 0x68, 0x65, 0x72, 0x5F, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67, + // Wrapping over an u32 (4 bytes). + 0xFF, 0x00, 0x00, 0x00, + ]; + + // At version 4, isize and Vec disappear (20 bytes): 0x6F - 0x14 = 0x5B. + #[rustfmt::skip] + let v4_hardcoded_snapshot: &[u8] = &[ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, + 0xFF, 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x50, 0x40, 0x61, + 0x01, + 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x73, 0x6F, 0x6D, 0x65, 0x5F, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67, + 0x02, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x81, + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x6F, 0x6D, 0x65, 0x5F, + 0x6F, 0x74, 0x68, 0x65, 0x72, 0x5F, 0x73, 0x74, 0x72, 0x69, 0x6E, 0x67, + 0xFF, 0x00, 0x00, 0x00, + ]; + + #[derive(Debug, PartialEq, Versionize)] + pub struct TestStruct { + usize_1: usize, + #[version(start = 2, end = 4, default_fn = "default_isize")] + isize_1: isize, + u16_1: u16, + #[version(end = 3, default_fn = "default_u64")] + u64_1: u64, + i8_1: i8, + #[version(start = 2, end = 2)] + i16_1: i16, + i32_1: i32, + #[version(start = 2, end = 3, default_fn = "default_i64")] + i64_1: i64, + f32_1: f32, + f64_1: f64, + char_1: char, + #[version(start = 2, default_fn = "default_bool")] + bool_1: bool, + string_1: String, + enum_1: TestState, + option_1: Option, + #[version(start = 3, end = 4, default_fn = "default_vec")] + vec_1: Vec, + #[allow(clippy::box_collection)] // we want to explicitly test Box + box_1: Box, + #[version(start = 3)] + wrapping_1: Wrapping, + } + + impl TestStruct { + fn default_isize(_source_version: u16) -> isize { + 12isize + } + + fn default_u64(_source_version: u16) -> u64 { + 0x0Du64 + } + + fn default_i64(_source_version: u16) -> i64 { + 0x0Ei64 + } + + fn default_bool(_source_version: u16) -> bool { + false + } + + fn default_vec(_source_version: u16) -> Vec { + vec!['v'; 8] + } + } + + let mut vm = VersionMap::new(); + vm.new_version() + .set_type_version(TestStruct::type_id(), 2) + .set_type_version(TestState::type_id(), 2) + .new_version() + .set_type_version(TestStruct::type_id(), 3) + .new_version() + .set_type_version(TestStruct::type_id(), 4); + + let mut snapshot_blob = v1_hardcoded_snapshot; + + let mut restored_state = + ::deserialize(&mut snapshot_blob, &vm, 1).unwrap(); + + // We expect isize, i16, i64, bool, Vec and Wrapping fields to have the default values at v1. + let mut expected_state = TestStruct { + usize_1: 1, + isize_1: 12, + u16_1: 4, + u64_1: 0xABCDu64, + i8_1: -1, + i16_1: 0, + i32_1: 32, + i64_1: 0x0Ei64, + f32_1: 0.5, + f64_1: 64.5, + char_1: 'a', + bool_1: false, + string_1: "some_string".to_owned(), + enum_1: TestState::One(2), + option_1: Some(129), + vec_1: vec!['v'; 8], + box_1: Box::new("some_other_string".to_owned()), + wrapping_1: Wrapping(0u32), + }; + assert_eq!(restored_state, expected_state); + + snapshot_blob = v2_hardcoded_snapshot; + + restored_state = ::deserialize(&mut snapshot_blob, &vm, 2).unwrap(); + + // We expect only i16, Vec and Wrapping fields to have the default values at v2. + expected_state = TestStruct { + usize_1: 1, + isize_1: 2, + u16_1: 4, + u64_1: 0xABCDu64, + i8_1: -1, + i16_1: 0, + i32_1: 32, + i64_1: 0xFFFFi64, + f32_1: 0.5, + f64_1: 64.5, + char_1: 'a', + bool_1: true, + string_1: "some_string".to_owned(), + enum_1: TestState::Two(14), + option_1: Some(129), + vec_1: vec!['v'; 8], + box_1: Box::new("some_other_string".to_owned()), + wrapping_1: Wrapping(0u32), + }; + assert_eq!(restored_state, expected_state); + + snapshot_blob = v3_hardcoded_snapshot; + + restored_state = ::deserialize(&mut snapshot_blob, &vm, 3).unwrap(); + + // We expect u64, i16 and i64 fields to have the default values at v3. + expected_state = TestStruct { + usize_1: 1, + isize_1: 2, + u16_1: 4, + u64_1: 0x0Du64, + i8_1: -1, + i16_1: 0, + i32_1: 32, + i64_1: 0x0Ei64, + f32_1: 0.5, + f64_1: 64.5, + char_1: 'a', + bool_1: true, + string_1: "some_string".to_owned(), + enum_1: TestState::Two(14), + option_1: Some(129), + vec_1: vec!['a'; 4], + box_1: Box::new("some_other_string".to_owned()), + wrapping_1: Wrapping(255u32), + }; + assert_eq!(restored_state, expected_state); + + snapshot_blob = v4_hardcoded_snapshot; + + restored_state = ::deserialize(&mut snapshot_blob, &vm, 4).unwrap(); + + // We expect isize, u64, i16, i64 and Vec fields to have the default values at v4. + expected_state = TestStruct { + usize_1: 1, + isize_1: 12, + u16_1: 4, + u64_1: 0x0Du64, + i8_1: -1, + i16_1: 0, + i32_1: 32, + i64_1: 0x0Ei64, + f32_1: 0.5, + f64_1: 64.5, + char_1: 'a', + bool_1: true, + string_1: "some_string".to_owned(), + enum_1: TestState::Two(14), + option_1: Some(129), + vec_1: vec!['v'; 8], + box_1: Box::new("some_other_string".to_owned()), + wrapping_1: Wrapping(255u32), + }; + assert_eq!(restored_state, expected_state); +} + +#[test] +fn test_hardcoded_enum_deserialization() { + // We are testing separately also hardcoded snapshot deserialization for enums + // as these have a different behavior in terms of serialization/deserialization. + #[rustfmt::skip] + let v1_hardcoded_snapshot: &[u8] = &[ + // Variant number (4 bytes), the first variant lacks a value. + 0x00, 0x00, 0x00, 0x00, + ]; + + #[rustfmt::skip] + let v2_hardcoded_snapshot: &[u8] = &[ + 0x00, 0x00, 0x00, 0x00, + ]; + + #[rustfmt::skip] + let unexpected_v1_hardcoded_snapshot: &[u8] = &[ + // Second variant (4 bytes) + value of that variant (8 bytes). + 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + + #[rustfmt::skip] + let invalid_v1_hardcoded_snapshot: &[u8] = &[ + // Invalid enum variant number. + 0x03, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + + let mut vm = VersionMap::new(); + vm.new_version().set_type_version(TestState::type_id(), 2); + + let mut snapshot_blob = v1_hardcoded_snapshot; + + let mut restored_state = + ::deserialize(&mut snapshot_blob, &vm, 1).unwrap(); + assert_eq!(restored_state, TestState::Zero); + + snapshot_blob = v2_hardcoded_snapshot; + + restored_state = ::deserialize(&mut snapshot_blob, &vm, 2).unwrap(); + assert_eq!(restored_state, TestState::Zero); + + snapshot_blob = unexpected_v1_hardcoded_snapshot; + + // Versioned deserialization is not implemented for enums, so even though we do not have + // `Two` state available at version 2, restoring the data won't fail :(. + // TODO: This must be fixed. + restored_state = ::deserialize(&mut snapshot_blob, &vm, 1).unwrap(); + assert_eq!(restored_state, TestState::Two(5)); + + // This snapshot contains a non-existent enum variant. + snapshot_blob = invalid_v1_hardcoded_snapshot; + + assert_eq!( + ::deserialize(&mut snapshot_blob, &vm, 1).unwrap_err(), + VersionizeError::Deserialize("Unknown variant_index 3".to_owned()) + ); +} + +#[derive(Debug, PartialEq, Eq, Versionize)] +pub struct A { + a: u32, + #[version(start = 1, end = 2)] + b: Option, + #[version(start = 2, default_fn = "default_c")] + c: String, +} + +#[derive(Debug, PartialEq, Eq, Versionize)] +pub struct X { + x: bool, + a_1: A, + #[version(end = 3, default_fn = "default_y")] + y: Box, + #[version(start = 3, default_fn = "default_z")] + z: Vec, +} + +impl A { + fn default_c(_source_version: u16) -> String { + "some_string".to_owned() + } +} + +impl X { + fn default_y(_source_version: u16) -> Box { + Box::from(4) + } + + fn default_z(_source_version: u16) -> Vec { + vec![16, 4] + } +} + +#[test] +fn test_nested_structs_deserialization() { + #[rustfmt::skip] + let v1_hardcoded_snapshot: &[u8] = &[ + // Bool field (1 byte) from X, `a` field from A (4 bytes) + + 0x00, 0x10, 0x00, 0x00, 0x00, + // `b` field from A: Option type (1 byte), inner enum variant number (4 bytes) + + // + value of that variant (4 bytes) + + 0x01, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + // `y` field from A (8 bytes). + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + + #[rustfmt::skip] + let v2_hardcoded_snapshot: &[u8] = &[ + // Bool field (1 byte) from X, `a` field from A (4 bytes) + + 0x00, 0x10, 0x00, 0x00, 0x00, + // `c` field from X: String len (8 bytes) + actual String; + // the Option field is not available at v2. + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x72, 0x61, 0x6E, 0x64, 0x6F, 0x6D, + // `y` field from A (8 bytes). + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + + #[rustfmt::skip] + let v3_hardcoded_snapshot: &[u8] = &[ + 0x00, 0x10, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x72, 0x61, 0x6E, 0x64, 0x6F, 0x6D, + // `z` field from A (8 bytes). + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, + ]; + + let mut vm = VersionMap::new(); + vm.new_version() + .set_type_version(A::type_id(), 2) + .set_type_version(X::type_id(), 2) + .set_type_version(TestState::type_id(), 2) + .new_version() + .set_type_version(X::type_id(), 3); + + let mut snapshot_blob = v1_hardcoded_snapshot; + + let mut restored_state = ::deserialize(&mut snapshot_blob, &vm, 1).unwrap(); + // We expect `z` and `c` fields to have the default values. + let mut expected_state = X { + x: false, + a_1: A { + a: 16u32, + b: Some(TestState::One(4)), + c: "some_string".to_owned(), + }, + y: Box::from(2), + z: vec![16, 4], + }; + assert_eq!(restored_state, expected_state); + + snapshot_blob = v2_hardcoded_snapshot; + + restored_state = ::deserialize(&mut snapshot_blob, &vm, 2).unwrap(); + + // We expect `b` and `z` fields to have the default values. + expected_state = X { + x: false, + a_1: A { + a: 16u32, + b: None, + c: "random".to_owned(), + }, + y: Box::from(2), + z: vec![16, 4], + }; + assert_eq!(restored_state, expected_state); + + snapshot_blob = v3_hardcoded_snapshot; + + restored_state = ::deserialize(&mut snapshot_blob, &vm, 3).unwrap(); + + // We expect `b` and `y` fields to have the default values. + expected_state = X { + x: false, + a_1: A { + a: 16u32, + b: None, + c: "random".to_owned(), + }, + y: Box::from(4), + z: vec![24; 4], + }; + assert_eq!(restored_state, expected_state); +} + +pub const SIZE: usize = 10; + +pub mod dummy_mod { + pub const SIZE: usize = 20; +} + +#[test] +fn test_versionize_struct_with_array() { + #[derive(Debug, PartialEq, Versionize)] + struct TestStruct { + a: [u32; SIZE], + b: [u8; dummy_mod::SIZE], + c: Option<[i16; SIZE]>, + } + + let test_struct = TestStruct { + a: [1; SIZE], + b: [2; dummy_mod::SIZE], + c: Some([3; SIZE]), + }; + + let mut mem = vec![0; 4096]; + let version_map = VersionMap::new(); + + test_struct + .serialize(&mut mem.as_mut_slice(), &version_map, 1) + .unwrap(); + let restored_test_struct = + TestStruct::deserialize(&mut mem.as_slice(), &version_map, 1).unwrap(); + + assert_eq!(restored_test_struct, test_struct); +} + +#[derive(Clone, Debug, PartialEq, Eq, Versionize)] +pub enum DeviceStatus { + Inactive, + Active, + #[version(start = 2, default_fn = "default_is_activating")] + IsActivating(u32), +} + +impl Default for DeviceStatus { + fn default() -> Self { + Self::Inactive + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Versionize)] +pub enum OperationSupported { + Add, + Remove, + RemoveAndAdd(bool), + #[version(start = 2, default_fn = "default_update")] + Update(String), +} + +impl Default for OperationSupported { + fn default() -> Self { + Self::Add + } +} + +impl DeviceStatus { + fn default_is_activating(&self, target_version: u16) -> VersionizeResult { + match target_version { + 1 => Ok(DeviceStatus::Inactive), + i => Err(VersionizeError::Serialize(format!( + "Unknown target version: {}", + i + ))), + } + } +} + +impl OperationSupported { + fn default_update(&self, target_version: u16) -> VersionizeResult { + match target_version { + 1 => Ok(OperationSupported::RemoveAndAdd(true)), + i => Err(VersionizeError::Serialize(format!( + "Unknown target version: {}", + i + ))), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Versionize)] +pub struct Device { + name: String, + id: Wrapping, + #[version(start = 2, ser_fn = "ser_is_activated")] + is_activated: bool, + some_params: Vec, + #[version( + start = 2, + default_fn = "default_ops", + ser_fn = "ser_ops", + de_fn = "de_ops" + )] + operations: Vec, + status: DeviceStatus, + #[version( + start = 2, + default_fn = "default_queues_limit", + ser_fn = "ser_queues_limit" + )] + no_queues_limit: usize, + queues: Vec, + features: u32, + #[version(start = 3, ser_fn = "ser_extra", de_fn = "de_extra")] + extra_features: u64, +} + +impl Device { + fn default_ops(_target_version: u16) -> Vec { + vec![OperationSupported::Add, OperationSupported::Remove] + } + + fn default_queues_limit(_target_version: u16) -> usize { + 2 + } + + fn ser_ops(&mut self, target_version: u16) -> VersionizeResult<()> { + // Fail if semantic serialization is called for a version >= 2. + assert!(target_version < 2); + self.features |= 1; + Ok(()) + } + + fn de_ops(&mut self, target_version: u16) -> VersionizeResult<()> { + // Fail if semantic deserialization is called for a version >= 2. + assert!(target_version < 2); + if self.some_params.contains(&"active".to_owned()) { + self.status = DeviceStatus::Active; + } + Ok(()) + } + + fn ser_queues_limit(&mut self, target_version: u16) -> VersionizeResult<()> { + // Fail if semantic serialization is called for a version >= 2. + assert!(target_version < 2); + if self.queues.len() > 2 { + return Err(VersionizeError::Semantic("Too many queues.".to_owned())); + } + Ok(()) + } + + fn ser_is_activated(&mut self, target_version: u16) -> VersionizeResult<()> { + // Fail if semantic serialization is called for a version >= 2. + assert!(target_version < 2); + self.some_params.push("active".to_owned()); + self.some_params.retain(|x| x.clone() != *"inactive"); + Ok(()) + } + + fn ser_extra(&mut self, target_version: u16) -> VersionizeResult<()> { + // Fail if semantic serialization is called for the latest version. + assert!(target_version < 3); + self.some_params.push("extra_features".to_owned()); + Ok(()) + } + + fn de_extra(&mut self, target_version: u16) -> VersionizeResult<()> { + // Fail if semantic deserialization is called for the latest version. + assert!(target_version < 3); + if self.queues.len() > self.no_queues_limit { + return Err(VersionizeError::Semantic("Too many queues.".to_owned())); + } + self.features |= 1u32 << 31; + Ok(()) + } +} + +#[test] +fn test_versionize_struct_with_enums() { + let mut vm = VersionMap::new(); + vm.new_version() + .set_type_version(Device::type_id(), 2) + .set_type_version(DeviceStatus::type_id(), 2) + .new_version() + .set_type_version(Device::type_id(), 3) + .set_type_version(OperationSupported::type_id(), 2); + + let mut state = Device { + name: "block".to_owned(), + id: Wrapping(1u32), + is_activated: true, + some_params: vec!["inactive".to_owned()], + operations: vec![ + OperationSupported::Add, + OperationSupported::Update("random".to_owned()), + ], + status: DeviceStatus::Inactive, + no_queues_limit: 3, + queues: vec![1u8, 2u8], + features: 6u32, + extra_features: 0u64, + }; + + let mut snapshot_mem = vec![0u8; 1024]; + + // Serialize as v1. + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let mut restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + + // At v1, all of the semantic functions should be called. + // `operations` and `no_queues_limit` will take the default values (set by `default_fn`s), + // `features` will be modified by `ser_ops` and `de_extra`, `status` will be changed to + // `Active` by `de_ops`, `is_activated` will take the default bool value, `some_params` + // will be also modified and the other fields will take the original values. + let mut expected_state = Device { + name: "block".to_owned(), + id: Wrapping(1u32), + is_activated: false, + some_params: vec!["active".to_owned(), "extra_features".to_owned()], + operations: vec![OperationSupported::Add, OperationSupported::Remove], + status: DeviceStatus::Active, + no_queues_limit: 2, + queues: vec![1u8, 2u8], + features: 0x8000_0007u32, + extra_features: 0u64, + }; + assert_eq!(expected_state, restored_state); + + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 2) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 2).unwrap(); + + // At v2, we expect that only the semantic functions from `extra_features` to be called, + // this means that `features` and `some_params` will take different values than the ones + // at v1. `status` won't be modified anymore, `is_activated` and `no_queues_limit` will + // take this time the original values. `operations` field will contain only the first + // original element, the second one will be modified by `default_update` because at v2, + // `Update` is not available. + expected_state = Device { + name: "block".to_owned(), + id: Wrapping(1u32), + is_activated: true, + some_params: vec!["inactive".to_owned(), "extra_features".to_owned()], + operations: vec![ + OperationSupported::Add, + OperationSupported::RemoveAndAdd(true), + ], + status: DeviceStatus::Inactive, + no_queues_limit: 3, + queues: vec![1u8, 2u8], + features: 0x8000_0006u32, + extra_features: 0u64, + }; + assert_eq!(expected_state, restored_state); + + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 3) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 3).unwrap(); + + // At v3, `Update` variant is available, so it will be deserialized to its original value. + // We expect no semantic function to be called, so `features` and `some_params` will also + // take the original values. + expected_state = Device { + name: "block".to_owned(), + id: Wrapping(1u32), + is_activated: true, + some_params: vec!["inactive".to_owned()], + operations: vec![ + OperationSupported::Add, + OperationSupported::Update("random".to_owned()), + ], + status: DeviceStatus::Inactive, + no_queues_limit: 3, + queues: vec![1u8, 2u8], + features: 6u32, + extra_features: 0u64, + }; + assert_eq!(expected_state, restored_state); + + // Test semantic errors. + state.queues = vec![1u8, 2u8, 3u8, 4u8]; + assert_eq!( + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap_err(), + VersionizeError::Semantic("Too many queues.".to_owned()) + ); + + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 2) + .unwrap(); + assert_eq!( + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 2).unwrap_err(), + VersionizeError::Semantic("Too many queues.".to_owned()) + ); +} + +#[derive(Clone, Debug, PartialEq, Eq, Versionize)] +pub enum State { + Zero, + One(bool), + #[version(start = 2, default_fn = "default_state_two")] + Two(Vec), + #[version(start = 2, default_fn = "default_state_three")] + Three(String), + #[version(start = 3, default_fn = "default_state_four")] + Four(Option), +} + +impl Default for State { + fn default() -> Self { + Self::One(false) + } +} + +impl State { + fn default_state_two(&self, target_version: u16) -> VersionizeResult { + match target_version { + 1 => Ok(State::One(true)), + i => Err(VersionizeError::Serialize(format!( + "Unknown target version: {}", + i + ))), + } + } + + fn default_state_three(&self, target_version: u16) -> VersionizeResult { + match target_version { + 1 => Ok(State::One(false)), + i => Err(VersionizeError::Serialize(format!( + "Unknown target version: {}", + i + ))), + } + } + + fn default_state_four(&self, target_version: u16) -> VersionizeResult { + match target_version { + 2 => Ok(State::Three("abc".to_owned())), + 1 => Ok(State::Zero), + i => Err(VersionizeError::Serialize(format!( + "Unknown target version: {}", + i + ))), + } + } +} + +#[test] +fn test_versionize_enum() { + let mut vm = VersionMap::new(); + vm.new_version() + .set_type_version(State::type_id(), 2) + .new_version() + .set_type_version(State::type_id(), 3); + + let mut snapshot_mem = vec![0u8; 1024]; + + // First we test that serializing and deserializing an enum variant available at the + // target version results in the same variant. + let mut state = State::One(true); + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let mut restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + assert_eq!(state, restored_state); + + // Now we test `default_fn`s for serialization of enum variants that don't exist in + // previous versions. + state = State::Four(Some(0x1234_5678_8765_4321u64)); + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + assert_eq!(restored_state, State::Zero); + + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 2) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 2).unwrap(); + assert_eq!(restored_state, State::Three("abc".to_owned())); + + state = State::Three("some_string".to_owned()); + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 2).unwrap(); + assert_eq!(restored_state, State::One(false)); +} + +#[derive(Clone, Debug, PartialEq, Versionize)] +pub struct S { + a: f64, + b: i64, +} + +#[derive(Clone, Debug, PartialEq, Versionize)] +pub struct Test { + usize_1: usize, + #[version(start = 2, end = 3, ser_fn = "ser_isize", de_fn = "de_isize")] + isize_1: isize, + #[version(start = 2)] + u8_1: u8, + #[version(end = 4, default_fn = "default_vec")] + vec_1: Vec, + #[version(start = 3)] + wrapping_1: Wrapping, + #[version( + end = 3, + default_fn = "default_u64", + ser_fn = "ser_u64", + de_fn = "de_u64" + )] + u64_1: u64, + #[version(start = 2, ser_fn = "ser_bool")] + bool_1: bool, + enum_1: State, + i8_1: i8, + i16_1: i16, + #[version(start = 3, end = 4)] + i32_1: i32, + #[version(start = 2, default_fn = "default_box", de_fn = "de_box")] + box_1: Box, + #[version(start = 2, end = 3, default_fn = "default_f32")] + f32_1: f32, + char_1: char, + #[version( + end = 3, + default_fn = "default_option", + ser_fn = "ser_option", + de_fn = "de_option" + )] + option_1: Option, +} + +impl Test { + fn default_vec(_target_version: u16) -> Vec { + vec![0x0102u16; 4] + } + + fn default_u64(_target_version: u16) -> u64 { + 0x0102_0102_0102_0102u64 + } + + fn default_f32(_target_version: u16) -> f32 { + 0.5 + } + + fn default_box(_target_version: u16) -> Box { + Box::new(S { a: 1.5, b: 2 }) + } + + fn default_option(_target_version: u16) -> Option { + Some("something".to_owned()) + } + + fn ser_isize(&mut self, target_version: u16) -> VersionizeResult<()> { + assert_ne!(target_version, 2); + self.vec_1.push(0x0304u16); + if self.i8_1 == -1 { + return Err(VersionizeError::Semantic( + "Unexpected value for `i8` field.".to_owned(), + )); + } + Ok(()) + } + fn ser_u64(&mut self, target_version: u16) -> VersionizeResult<()> { + assert!(target_version >= 3); + self.vec_1.pop(); + if self.u8_1 == 4 { + self.bool_1 = false; + } + Ok(()) + } + + fn ser_bool(&mut self, target_version: u16) -> VersionizeResult<()> { + assert!(target_version < 2); + self.vec_1.push(0x0506u16); + self.vec_1.push(0x0708u16); + Ok(()) + } + + fn ser_option(&mut self, target_version: u16) -> VersionizeResult<()> { + assert!(target_version >= 3); + self.u8_1 += 2; + if self.vec_1.len() == 10 { + return Err(VersionizeError::Semantic("Vec is full.".to_owned())); + } + Ok(()) + } + + fn de_isize(&mut self, target_version: u16) -> VersionizeResult<()> { + assert_ne!(target_version, 2); + self.u8_1 += 3; + Ok(()) + } + + fn de_u64(&mut self, target_version: u16) -> VersionizeResult<()> { + assert!(target_version >= 3); + self.vec_1.push(0x0101u16); + Ok(()) + } + + fn de_box(&mut self, target_version: u16) -> VersionizeResult<()> { + assert!(target_version < 2); + self.option_1 = Some("box_change".to_owned()); + if self.vec_1.len() == 3 { + return Err(VersionizeError::Semantic( + "Vec len is too small.".to_owned(), + )); + } + Ok(()) + } + + fn de_option(&mut self, target_version: u16) -> VersionizeResult<()> { + assert!(target_version >= 3); + self.enum_1 = State::Two(vec![1; 4]); + Ok(()) + } +} + +#[test] +fn test_versionize_struct() { + let mut vm = VersionMap::new(); + vm.new_version() + .set_type_version(Test::type_id(), 2) + .set_type_version(State::type_id(), 2) + .new_version() + .set_type_version(Test::type_id(), 3) + .set_type_version(State::type_id(), 3); + + let mut state = Test { + usize_1: 0x0102_0304_0506_0708usize, + isize_1: -0x1122_3344_5566_7788isize, + u8_1: 4, + vec_1: vec![0x1122u16; 5], + wrapping_1: Wrapping(4u32), + u64_1: 0x0102_0304_0506_0708u64, + bool_1: false, + enum_1: State::Four(Some(0x0102_0304_0506_0708u64)), + i8_1: 8, + i16_1: -12, + i32_1: -0x1234_5678, + box_1: Box::new(S { a: 4.5, b: 4 }), + f32_1: 1.25, + char_1: 'c', + option_1: None, + }; + + let mut snapshot_mem = vec![0u8; 1024]; + + // Serialize as v1. + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let mut restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + + let mut expected_state = Test { + // usize field exists at all versions, will take the original value. + usize_1: 0x0102_0304_0506_0708usize, + // isize field will take the default value as it is not available at v1. + isize_1: 0isize, + // u8 field doesn't exist at v1, it wll take the default value and then it will be + // modified by `de_isize`: 0 + 3 = 3. + u8_1: 3, + // Vec field will be modified by the semantic fns of the fields that don't exist + // at v1: `isize_1`, `bool_1`; there will be 3 new elements added in it. + vec_1: vec![ + 0x1122u16, 0x1122u16, 0x1122u16, 0x1122u16, 0x1122u16, 0x0304u16, 0x0506u16, 0x0708u16, + ], + // We expect here to have the default value. + wrapping_1: Wrapping(0u32), + // We expect here to have the original value. + u64_1: 0x0102_0304_0506_0708u64, + // We expect here to have the default value. + bool_1: false, + // This will take the default value for state `Four` and v1. + enum_1: State::Zero, + // i8, i16 fields take the original values. + i8_1: 8, + i16_1: -12, + // i32 field takes the default value. + i32_1: 0, + // Box and f32 fields will take the default values set by `default_fn`s. + box_1: Box::new(S { a: 1.5, b: 2 }), + f32_1: 0.5, + // We expect this field to take the original value. + char_1: 'c', + // This field will be modified by `de_box`. + option_1: Some("box_change".to_owned()), + }; + assert_eq!(expected_state, restored_state); + + // Serialize as v2. + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 2) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 2).unwrap(); + + // At v2 isize, u8, bool, box and f32 fields will be available, their semantic fns won't + // be called. + expected_state = Test { + usize_1: 0x0102_0304_0506_0708usize, + isize_1: -0x1122_3344_5566_7788isize, + u8_1: 4, + // This should take the original value this time. + vec_1: vec![0x1122u16, 0x1122u16, 0x1122u16, 0x1122u16, 0x1122u16], + wrapping_1: Wrapping(0u32), + u64_1: 0x0102_0304_0506_0708u64, + bool_1: false, + // This will take the default value for state `Four` and v2. + enum_1: State::Three("abc".to_owned()), + i8_1: 8, + i16_1: -12, + i32_1: 0, + box_1: Box::new(S { a: 4.5, b: 4 }), + f32_1: 1.25, + char_1: 'c', + option_1: None, + }; + assert_eq!(expected_state, restored_state); + + // Serialize as v3. + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 3) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 3).unwrap(); + + expected_state = Test { + usize_1: 0x0102_0304_0506_0708usize, + isize_1: 0isize, + // This field will be modified by `de_isize` and `ser_option`: 4 + 2 + 3 = 9. + u8_1: 9, + // Vec field will be modified by `ser_isize` (add one elem), `ser_u64` (remove one elem) + // and `de_64` (add one elem). + vec_1: vec![ + 0x1122u16, 0x1122u16, 0x1122u16, 0x1122u16, 0x1122u16, 0x0101u16, + ], + wrapping_1: Wrapping(4u32), + u64_1: 0x0102_0102_0102_0102u64, + bool_1: false, + enum_1: State::Two(vec![1; 4]), + i8_1: 8, + i16_1: -12, + i32_1: -0x1234_5678, + box_1: Box::new(S { a: 4.5, b: 4 }), + f32_1: 0.5, + char_1: 'c', + // We expect this field to take the default value set by its `default_fn`. + option_1: Some("something".to_owned()), + }; + assert_eq!(expected_state, restored_state); + + // Test semantic errors. + state.vec_1 = Vec::new(); + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + assert_eq!( + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap_err(), + VersionizeError::Semantic("Vec len is too small.".to_owned()) + ); + + state.vec_1 = vec![0x1122u16; 10]; + assert_eq!( + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 3) + .unwrap_err(), + VersionizeError::Semantic("Vec is full.".to_owned()) + ); + + state.i8_1 = -1; + assert_eq!( + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap_err(), + VersionizeError::Semantic("Unexpected value for `i8` field.".to_owned()) + ); + state.i8_1 = 0; + + // Test serialize and deserialize errors. + snapshot_mem = vec![0u8; 8]; + // Serializing `state` will fail due to the small size of `snapshot_mem`. + assert_eq!( + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap_err(), + VersionizeError::Serialize( + "Io(Error { kind: WriteZero, message: \"failed to write whole buffer\" })".to_owned() + ) + ); + snapshot_mem = vec![0u8; 256]; + + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + snapshot_mem.truncate(10); + // Deserialization will fail if we don't use the whole `snapshot_mem` resulted from + // serialization. + assert_eq!( + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap_err(), + VersionizeError::Deserialize( + "Io(Error { kind: UnexpectedEof, message: \"failed to fill whole buffer\" })" + .to_owned() + ) + ); +} + +#[repr(C)] +#[derive(Debug, Default, Versionize)] +struct Message { + pub len: u32, + #[version(end = 4)] + pub padding: u32, + pub value: u32, + #[version(start = 2, default_fn = "default_extra_value")] + pub extra_value: u16, + #[version(start = 3, end = 4, default_fn = "default_status")] + pub status: Wrapping, + pub entries: __IncompleteArrayField, +} + +impl Message { + fn default_extra_value(_source_version: u16) -> u16 { + 4 + } + + fn default_status(_source_version: u16) -> Wrapping { + Wrapping(false) + } +} + +#[repr(C)] +#[derive(Debug, Default, Versionize)] +struct Message2 { + pub len: u32, + #[version(end = 4)] + pub padding: u32, + pub value: u32, + #[version(start = 2, default_fn = "default_extra_value")] + pub extra_value: u16, + #[version(start = 3, end = 4, default_fn = "default_status")] + pub status: Wrapping, + pub entries: __IncompleteArrayField, +} + +impl Message2 { + fn default_extra_value(_source_version: u16) -> u16 { + 4 + } + + fn default_status(_source_version: u16) -> Wrapping { + Wrapping(false) + } +} + +generate_fam_struct_impl!(Message, u32, entries, u32, len, 100); +// Duplicated structure used but with max_len 1 - for negative testing. +generate_fam_struct_impl!(Message2, u32, entries, u32, len, 1); + +#[repr(C)] +#[derive(Default)] +pub struct __IncompleteArrayField(::std::marker::PhantomData, [T; 0]); + +impl __IncompleteArrayField { + #[inline] + pub fn new() -> Self { + __IncompleteArrayField(::std::marker::PhantomData, []) + } + #[inline] + pub unsafe fn as_ptr(&self) -> *const T { + self as *const __IncompleteArrayField as *const T + } + #[inline] + pub unsafe fn as_mut_ptr(&mut self) -> *mut T { + self as *mut __IncompleteArrayField as *mut T + } + #[inline] + pub unsafe fn as_slice(&self, len: usize) -> &[T] { + ::std::slice::from_raw_parts(self.as_ptr(), len) + } + #[inline] + pub unsafe fn as_mut_slice(&mut self, len: usize) -> &mut [T] { + ::std::slice::from_raw_parts_mut(self.as_mut_ptr(), len) + } +} + +impl Debug for __IncompleteArrayField { + fn fmt(&self, fmt: &mut Formatter<'_>) -> Result { + fmt.write_str("__IncompleteArrayField") + } +} + +impl ::std::clone::Clone for __IncompleteArrayField { + #[inline] + fn clone(&self) -> Self { + Self::new() + } +} + +impl Versionize for __IncompleteArrayField { + #[inline] + fn serialize( + &self, + _writer: &mut W, + _version_map: &VersionMap, + _app_version: u16, + ) -> VersionizeResult<()> { + Ok(()) + } + + #[inline] + fn deserialize( + _reader: &mut R, + _version_map: &VersionMap, + _app_version: u16, + ) -> VersionizeResult { + Ok(Self::new()) + } + + // Not used. + fn version() -> u16 { + 1 + } +} + +type MessageFamStructWrapper = FamStructWrapper; +type Message2FamStructWrapper = FamStructWrapper; + +#[test] +fn test_deserialize_famstructwrapper_invalid_len() { + let mut vm = VersionMap::new(); + vm.new_version() + .set_type_version(Message::type_id(), 2) + .new_version() + .set_type_version(Message::type_id(), 3) + .new_version() + .set_type_version(Message::type_id(), 4); + + // Create FamStructWrapper with len 2 + let state = MessageFamStructWrapper::new(0).unwrap(); + let mut buffer = [0; 256]; + + state.serialize(&mut buffer.as_mut_slice(), &vm, 2).unwrap(); + + // the `len` field of the header is the first serialized field. + // Let's corrupt it by making it bigger than the actual number of serialized elements + buffer[0] = 255; + + assert_eq!( + MessageFamStructWrapper::deserialize(&mut buffer.as_slice(), &vm, 2).unwrap_err(), + VersionizeError::Deserialize("Mismatch between length of FAM specified in FamStruct header (255) and actual size of FAM (0)".to_string()) + ); +} + +#[test] +fn test_versionize_famstructwrapper() { + let mut vm = VersionMap::new(); + vm.new_version() + .set_type_version(Message::type_id(), 2) + .new_version() + .set_type_version(Message::type_id(), 3) + .new_version() + .set_type_version(Message::type_id(), 4); + + let mut state = MessageFamStructWrapper::new(0).unwrap(); + state.as_mut_fam_struct().padding = 8; + state.as_mut_fam_struct().extra_value = 16; + state.as_mut_fam_struct().status = Wrapping(true); + + state.push(1).unwrap(); + state.push(2).unwrap(); + + let mut snapshot_mem = vec![0u8; 256]; + + // Serialize as v1. + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let mut restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 1) + .unwrap(); + + let mut original_values = state.as_slice(); + let mut restored_values = restored_state.as_slice(); + assert_eq!(original_values, restored_values); + assert_eq!( + restored_values.len(), + state.as_fam_struct_ref().len as usize + ); + + assert_eq!( + state.as_fam_struct_ref().padding, + restored_state.as_fam_struct_ref().padding + ); + // `extra_value` and `status` should take the default values set by their corresponding `default_fn`s. + assert_eq!(4, restored_state.as_fam_struct_ref().extra_value); + assert_eq!(Wrapping(false), restored_state.as_fam_struct_ref().status); + + // Serialize as v2. + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 2) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 2) + .unwrap(); + + original_values = state.as_slice(); + restored_values = restored_state.as_slice(); + assert_eq!(original_values, restored_values); + + assert_eq!( + state.as_fam_struct_ref().padding, + restored_state.as_fam_struct_ref().padding + ); + // `extra_value` is available at v2, so it will take its original value. + assert_eq!( + state.as_fam_struct_ref().extra_value, + restored_state.as_fam_struct_ref().extra_value + ); + assert_eq!(Wrapping(false), restored_state.as_fam_struct_ref().status); + + // Serialize as v3. + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 3) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 3) + .unwrap(); + + assert_eq!( + state.as_fam_struct_ref().padding, + restored_state.as_fam_struct_ref().padding + ); + assert_eq!( + state.as_fam_struct_ref().extra_value, + restored_state.as_fam_struct_ref().extra_value + ); + // At v3, `status` field exists, so it will take its original value. + assert_eq!(Wrapping(true), restored_state.as_fam_struct_ref().status); + + // Serialize as v4. + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 4) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 4) + .unwrap(); + + // At v4, `padding` field will take the default u32 value. + assert_eq!(0, restored_state.as_fam_struct_ref().padding); + assert_eq!( + state.as_fam_struct_ref().extra_value, + restored_state.as_fam_struct_ref().extra_value + ); + // `status` is not available anymore, so it will take the default value. + assert_eq!(Wrapping(false), restored_state.as_fam_struct_ref().status); + + snapshot_mem = vec![0u8; 16]; + + assert_eq!( + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap_err(), + VersionizeError::Serialize( + "Io(Error { kind: WriteZero, message: \"failed to write whole buffer\" })".to_owned() + ) + ); +} + +#[derive(Versionize)] +pub struct FamStructTest { + some_u8: u8, + message_box: Box, + #[version(start = 2, default_fn = "default_option", de_fn = "de_option")] + some_option: Option, + #[version(start = 3)] + some_string: String, + #[version(end = 3, default_fn = "default_message", de_fn = "de_message")] + messages: Vec, +} + +impl FamStructTest { + fn default_message(_target_version: u16) -> Vec { + let mut f = MessageFamStructWrapper::new(0).unwrap(); + f.as_mut_fam_struct().padding = 1; + f.as_mut_fam_struct().extra_value = 2; + + f.push(10).unwrap(); + f.push(20).unwrap(); + + vec![f] + } + + fn default_option(_target_version: u16) -> Option { + Some(S { a: 0.5, b: 0 }) + } + + fn de_message(&mut self, target_version: u16) -> VersionizeResult<()> { + // Fail if semantic deserialization is called for v2. + assert_ne!(target_version, 2); + self.some_option = None; + self.some_string = "some_new_string".to_owned(); + Ok(()) + } + + fn de_option(&mut self, target_version: u16) -> VersionizeResult<()> { + // Fail if semantic deserialization is called for a version >= 2. + assert!(target_version < 2); + + let mut f = MessageFamStructWrapper::new(0).unwrap(); + f.as_mut_fam_struct().padding = 3; + f.as_mut_fam_struct().extra_value = 4; + + f.push(10).unwrap(); + f.push(20).unwrap(); + + self.messages.push(f); + Ok(()) + } +} + +#[test] +fn test_versionize_struct_with_famstructs() { + let mut vm = VersionMap::new(); + vm.new_version() + .set_type_version(FamStructTest::type_id(), 2) + .set_type_version(Message::type_id(), 2) + .new_version() + .set_type_version(FamStructTest::type_id(), 3) + .set_type_version(Message::type_id(), 3); + + let mut snapshot_mem = vec![0u8; 1024]; + + let mut f = MessageFamStructWrapper::new(0).unwrap(); + f.as_mut_fam_struct().padding = 5; + f.as_mut_fam_struct().extra_value = 6; + f.push(10).unwrap(); + + let mut f2 = MessageFamStructWrapper::new(0).unwrap(); + f2.as_mut_fam_struct().padding = 7; + f2.as_mut_fam_struct().extra_value = 8; + f2.push(20).unwrap(); + + let state = FamStructTest { + some_u8: 1, + messages: vec![f], + some_string: "some_string".to_owned(), + message_box: Box::new(f2), + some_option: None, + }; + + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let mut restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + + // At version 1, we expect `de_option` and `de_message` to be called. + // `some_string` and `some_option` will take the default values. + assert_eq!(restored_state.some_string, String::default()); + assert_eq!(restored_state.some_option, Some(S { a: 0.5, b: 0 })); + let messages = restored_state.messages; + + // We expect to have 2 elements in the messages Vec (the one with which it was initialized and + // the one inserted by `de_option`). + assert_eq!(messages.len(), 2); + for message in messages.iter() { + assert_eq!(message.as_fam_struct_ref().extra_value, 4); + assert_eq!(message.as_fam_struct_ref().status, Wrapping(false)); + } + assert_eq!(messages[0].as_fam_struct_ref().padding, 5); + assert_eq!(messages[1].as_fam_struct_ref().padding, 3); + + // Serialize as v2. + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 2) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 2).unwrap(); + + assert_eq!(restored_state.some_string, String::default()); + // `some_option` is available at v2, so it will take the original value. + assert_eq!(restored_state.some_option, None); + let messages = restored_state.messages; + // We expect to have only one element in `messages` as `de_option` shouldn't be called + // this time. + assert_eq!(messages.len(), 1); + + // Serialize as v3. + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 3) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 3).unwrap(); + + // `some_string` is also available at v3. + assert_eq!(restored_state.some_string, "some_new_string".to_owned()); + assert_eq!(restored_state.some_option, None); + let messages = restored_state.messages; + // `messages` field is not available anymore at v3, it will take the default value, + // set by the corresponding `default_fn`. + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].as_fam_struct_ref().padding, 1); +} + +#[derive(Clone, Versionize)] +pub struct SomeStruct { + message: MessageFamStructWrapper, + #[version(start = 2, ser_fn = "ser_u16")] + some_u16: u16, +} + +impl SomeStruct { + fn ser_u16(&mut self, target_version: u16) -> VersionizeResult<()> { + // Fail if semantic serialization is called for the latest version. + assert!(target_version < 2); + self.message.as_mut_fam_struct().padding += 2; + + Ok(()) + } +} + +#[derive(Clone, Versionize)] +pub struct SomeStruct2 { + message: Message2FamStructWrapper, + #[version(start = 2, ser_fn = "ser_u16")] + some_u16: u16, +} + +impl SomeStruct2 { + fn ser_u16(&mut self, target_version: u16) -> VersionizeResult<()> { + // Fail if semantic serialization is called for the latest version. + assert!(target_version < 2); + self.message.as_mut_fam_struct().padding += 2; + + Ok(()) + } +} + +// `Clone` issue fixed: https://github.com/rust-vmm/vmm-sys-util/issues/85. +// We are keeping this as regression test. +#[test] +fn test_famstructwrapper_clone() { + // Test that having a `FamStructWrapper` in a structure that implements + // Clone will result in keeping with their original values, only the number + // of entries and the entries array when serializing. + let mut vm = VersionMap::new(); + vm.new_version().set_type_version(SomeStruct::type_id(), 2); + + let mut f = MessageFamStructWrapper::new(0).unwrap(); + f.as_mut_fam_struct().padding = 8; + + f.push(1).unwrap(); + f.push(2).unwrap(); + + let state = SomeStruct { + message: f, + some_u16: 2, + }; + + let mut snapshot_mem = vec![0u8; 128]; + + // Serialize as v1. + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 1) + .unwrap(); + let mut restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).unwrap(); + + // Negative scenario - FamStruct versionize impl fails due to SizeLimitExceeded. + assert!( + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 1).is_err() + ); + + let original_values = state.message.as_slice(); + let restored_values = restored_state.message.as_slice(); + + assert_ne!( + state.message.as_fam_struct_ref().padding, + restored_state.message.as_fam_struct_ref().padding + ); + assert_eq!(original_values, restored_values); + // `padding` field will have its value serialized (8), and then it will be incremented with 2 + // by `ser_u16`. + assert_eq!(10, restored_state.message.as_fam_struct_ref().padding); + + // Serialize as v2. + state + .serialize(&mut snapshot_mem.as_mut_slice(), &vm, 2) + .unwrap(); + restored_state = + ::deserialize(&mut snapshot_mem.as_slice(), &vm, 2).unwrap(); + + // Padding is correctly preserved and ser_u16 is not called - it would fail due to ser_u16 + // assert anyways, but we double that check here. + assert_eq!( + state.message.as_fam_struct_ref().padding, + restored_state.message.as_fam_struct_ref().padding + ); + // `padding` field will have its value preserved (8). `ser_u16` won't be called at v2. + assert_eq!(8, restored_state.message.as_fam_struct_ref().padding); +}