From 71475f8b8fd8b03980c4a5f7a66c97671c70a0d9 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 13 Jun 2024 18:41:04 +0700 Subject: [PATCH 1/4] refactor: extract document faker to crate --- .github/package-filters/rs-packages.yml | 5 + .github/workflows/tests-rs-package.yml | 12 +- Cargo.lock | 15 +- Cargo.toml | 1 + packages/rs-dpp/Cargo.toml | 4 - .../document_type/random_document.rs | 86 +------- .../document_type/v0/random_document.rs | 119 +---------- packages/rs-random-document/Cargo.toml | 18 ++ packages/rs-random-document/src/lib.rs | 200 ++++++++++++++++++ 9 files changed, 244 insertions(+), 216 deletions(-) create mode 100644 packages/rs-random-document/Cargo.toml create mode 100644 packages/rs-random-document/src/lib.rs diff --git a/.github/package-filters/rs-packages.yml b/.github/package-filters/rs-packages.yml index f50dea216d..773b2ca1cd 100644 --- a/.github/package-filters/rs-packages.yml +++ b/.github/package-filters/rs-packages.yml @@ -65,3 +65,8 @@ dash-sdk: - packages/rs-sdk/** - *dapi_client - *drive + +random-document: + - .github/workflows/tests* + - packages/rs-random-document/** + - *dpp diff --git a/.github/workflows/tests-rs-package.yml b/.github/workflows/tests-rs-package.yml index 77e28b2232..9fa06462f3 100644 --- a/.github/workflows/tests-rs-package.yml +++ b/.github/workflows/tests-rs-package.yml @@ -42,11 +42,11 @@ jobs: components: clippy - name: Setup Node.JS, required by json-schema-faker-rs - if: inputs.package == 'dpp' + if: inputs.package == 'random-document' uses: ./.github/actions/nodejs - name: Setup Javy, required by json-schema-faker-rs - if: inputs.package == 'dpp' + if: inputs.package == 'random-document' uses: ./.github/actions/javy - uses: clechasseur/rs-clippy-check@v3 @@ -189,11 +189,11 @@ jobs: uses: ./.github/actions/rust - name: Setup Node.JS, required by json-schema-faker-rs - if: inputs.package == 'dpp' + if: inputs.package == 'random-document' uses: ./.github/actions/nodejs - name: Setup Javy, required by json-schema-faker-rs - if: inputs.package == 'dpp' + if: inputs.package == 'random-document' uses: ./.github/actions/javy - name: Run tests @@ -222,11 +222,11 @@ jobs: uses: ./.github/actions/rust - name: Setup Node.JS, required by json-schema-faker-rs - if: inputs.package == 'dpp' + if: inputs.package == 'random-document' uses: ./.github/actions/nodejs - name: Setup Javy, required by json-schema-faker-rs - if: inputs.package == 'dpp' + if: inputs.package == 'random-document' uses: ./.github/actions/javy - name: Get crate ${{ runner.arch }} info diff --git a/Cargo.lock b/Cargo.lock index 19c63f2380..c8512c1dad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -128,9 +128,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.82" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "arbitrary" @@ -1570,7 +1570,6 @@ dependencies = [ "integer-encoding", "itertools 0.12.1", "json-schema-compatibility-validator", - "json-schema-faker", "jsonschema", "lazy_static", "log", @@ -4010,6 +4009,16 @@ dependencies = [ "getrandom", ] +[[package]] +name = "random-document" +version = "1.0.0-dev.15" +dependencies = [ + "anyhow", + "dpp", + "json-schema-faker", + "platform-value", +] + [[package]] name = "raw-cpuid" version = "10.7.0" diff --git a/Cargo.toml b/Cargo.toml index bdf98a874f..9f68947c4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ members = [ "packages/simple-signer", "packages/rs-json-schema-compatibility-validator", "packages/check-features", + "packages/rs-random-document", ] [patch.crates-io] diff --git a/packages/rs-dpp/Cargo.toml b/packages/rs-dpp/Cargo.toml index da94abc573..14b1905b9c 100644 --- a/packages/rs-dpp/Cargo.toml +++ b/packages/rs-dpp/Cargo.toml @@ -38,7 +38,6 @@ itertools = { version = "0.12.1" } jsonschema = { git = "https://github.com/fominok/jsonschema-rs", branch = "feat-unevaluated-properties", default-features = false, features = [ "draft202012", ], optional = true } -json-schema-faker = { git = "https://github.com/fominok/json-schema-faker-rs", rev = "2dd1cf7", optional = true } lazy_static = { version = "1.4" } log = { version = "0.4.6" } num_enum = "0.5.7" @@ -266,6 +265,3 @@ extended-document = [ factories = [] client = ["factories", "state-transitions"] - -# Faker feature requires `npm` and `javy` to be installed at build time -documents-faker = ["dep:json-schema-faker", "document-serde-conversion"] diff --git a/packages/rs-dpp/src/data_contract/document_type/random_document.rs b/packages/rs-dpp/src/data_contract/document_type/random_document.rs index 258b58d79b..16a0327778 100644 --- a/packages/rs-dpp/src/data_contract/document_type/random_document.rs +++ b/packages/rs-dpp/src/data_contract/document_type/random_document.rs @@ -1,9 +1,5 @@ -#[cfg(feature = "documents-faker")] -use std::collections::BTreeMap; - use bincode::{Decode, Encode}; -#[cfg(feature = "documents-faker")] -use platform_value::Value; + use platform_value::{Bytes32, Identifier}; use rand::prelude::StdRng; @@ -35,17 +31,6 @@ pub enum DocumentFieldFillSize { // TODO The factory is used in benchmark and tests. Probably it should be available under the test feature /// Functions for creating various types of random documents. pub trait CreateRandomDocument { - #[cfg(feature = "documents-faker")] - /// Create random documents using json-schema-faker-rs - fn random_documents_faker( - &self, - owner_id: Identifier, - entropy: &Bytes32, - count: u32, - platform_version: &PlatformVersion, - substitutions: &BTreeMap<&str, Value>, - ) -> Result, ProtocolError>; - /// Generates a single random document, employing default behavior for document field /// filling where fields that are not required will not be filled (`DoNotFillIfNotRequired`) and /// any fill size that is contractually allowed may be used (`AnyDocumentFillSize`). @@ -212,22 +197,6 @@ pub trait CreateRandomDocument { } impl CreateRandomDocument for DocumentType { - #[cfg(feature = "documents-faker")] - fn random_documents_faker( - &self, - owner_id: Identifier, - entropy: &Bytes32, - count: u32, - platform_version: &PlatformVersion, - substitutions: &BTreeMap<&str, Value>, - ) -> Result, ProtocolError> { - match self { - DocumentType::V0(v0) => { - v0.random_documents_faker(owner_id, entropy, count, platform_version, substitutions) - } - } - } - fn random_document( &self, seed: Option, @@ -346,22 +315,6 @@ impl CreateRandomDocument for DocumentType { } impl<'a> CreateRandomDocument for DocumentTypeRef<'a> { - #[cfg(feature = "documents-faker")] - fn random_documents_faker( - &self, - owner_id: Identifier, - entropy: &Bytes32, - count: u32, - platform_version: &PlatformVersion, - substitutions: &BTreeMap<&str, Value>, - ) -> Result, ProtocolError> { - match self { - DocumentTypeRef::V0(v0) => { - v0.random_documents_faker(owner_id, entropy, count, platform_version, substitutions) - } - } - } - fn random_document( &self, seed: Option, @@ -479,40 +432,3 @@ impl<'a> CreateRandomDocument for DocumentTypeRef<'a> { } } } - -#[cfg(all(test, feature = "documents-faker"))] -mod faker_tests { - use data_contracts::SystemDataContract; - use platform_version::TryIntoPlatformVersioned; - use rand::SeedableRng; - - use crate::{ - data_contract::accessors::v0::DataContractV0Getters, - system_data_contracts::load_system_data_contract, - }; - - use super::*; - - #[test] - fn test_random_document_faker() { - let data_contract = - load_system_data_contract(SystemDataContract::DPNS, &PlatformVersion::latest()) - .unwrap(); - let mut rng = StdRng::from_entropy(); - let entropy = Bytes32::random_with_rng(&mut rng); - - let _random_documents = data_contract - .document_types() - .iter() - .next() - .unwrap() - .1 - .random_documents_faker( - Identifier::random(), - &entropy, - 2, - &PlatformVersion::latest(), - &Default::default(), - ); - } -} diff --git a/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs b/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs index cfaeece525..6cc7ea31d5 100644 --- a/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs +++ b/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs @@ -5,15 +5,10 @@ //! //! -#[cfg(feature = "documents-faker")] -use std::collections::BTreeMap; -use std::time::{SystemTime, UNIX_EPOCH}; - -#[cfg(feature = "documents-faker")] -use platform_value::Value; use platform_value::{Bytes32, Identifier}; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; +use std::time::{SystemTime, UNIX_EPOCH}; use crate::data_contract::document_type::methods::DocumentTypeV0Methods; use crate::data_contract::document_type::random_document::{ @@ -32,118 +27,6 @@ use crate::version::PlatformVersion; use crate::ProtocolError; impl CreateRandomDocument for DocumentTypeV0 { - /// Create random documents using json-schema-faker-rs - #[cfg(feature = "documents-faker")] - fn random_documents_faker( - &self, - owner_id: Identifier, - entropy: &Bytes32, - count: u32, - platform_version: &PlatformVersion, - substitutions: &BTreeMap<&str, Value>, - ) -> Result, ProtocolError> { - use anyhow::Context; - - use crate::document::{ - extended_document_property_names::FEATURE_VERSION, - property_names::{ID, OWNER_ID, REVISION}, - serialization_traits::DocumentPlatformValueMethodsV0, - }; - - let json_schema = &self.schema.clone().try_into()?; - let json_documents = json_schema_faker::generate(json_schema, count as u16) - .context("cannot generate a random document with json-schema-faker-rs")?; - - let fix_document = |mut document: platform_value::Value| { - let id = Document::generate_document_id_v0( - &self.data_contract_id, - &owner_id, - self.name.as_str(), - entropy.as_slice(), - ); - let now = SystemTime::now(); - let duration_since_epoch = now.duration_since(UNIX_EPOCH).expect("Time went backwards"); - let time_ms = duration_since_epoch.as_millis() as u64; - - if self.documents_mutable { - document.as_map_mut().into_iter().for_each(|d| { - d.push((REVISION.into(), 1.into())); - }); - } - document.as_map_mut().into_iter().for_each(|d| { - d.push((ID.into(), id.into())); - }); - document.as_map_mut().into_iter().for_each(|d| { - d.push((OWNER_ID.into(), owner_id.into())); - }); - if self.required_fields.contains(FEATURE_VERSION) { - document.as_map_mut().into_iter().for_each(|d| { - d.push((FEATURE_VERSION.into(), "0".into())); - }); - } - if self.required_fields.contains(CREATED_AT) { - document.as_map_mut().into_iter().for_each(|d| { - d.push((CREATED_AT.into(), time_ms.into())); - }); - } - if self.required_fields.contains(UPDATED_AT) { - document.as_map_mut().into_iter().for_each(|d| { - d.push((UPDATED_AT.into(), time_ms.into())); - }); - } - - document - }; - - json_documents - .into_iter() - .map(|d| { - let p_value: Value = d.into(); - let fixed_value = fix_document(p_value); - - // TODO: tl;dr use PlatformDeserialize instead of Deserialize for Documents - // - // `properties` is a `BTreeMap` with `platform_value::Value` as values, since - // `Document::from_platform_value` does deserialization through Serde's data model - // it losts some information like distinction between `Value::Bytes` and `Value::Bytes32`; - // The solution here is to let deserialize a `Document`, but put `properties` unprocessed - // since they were `platform_value::Value` and will be the same type again and no deserialization - // is needed, especially that lossy kind. - let mut properties = fixed_value - .to_map_ref() - .ok() - .and_then(|m| Value::map_into_btree_string_map(m.clone()).ok()) - .unwrap_or_default(); - let mut document = Document::from_platform_value(fixed_value, platform_version); - if let Ok(Document::V0(d)) = document.as_mut() { - // This moves stored properties back to the document so it could skip unnecessary - // and wrong deserialization part - d.properties.iter_mut().for_each(|(k, v)| { - substitutions - .get(k.as_str()) - .cloned() - .or(properties.remove(k)) - .into_iter() - .for_each(|prop| { - // TODO: schema and internal DocumentType representations are incompatible - // Properties are tweaked though, because the only integer type supported by - // DPP is i64, while `platform_value::Value` distincts them, and json schema is - // even more permissive; however, we want our proofs to work and proofs use the - // DPP model. - *v = match prop { - Value::U64(x) => Value::I64(x as i64), - Value::U32(x) => Value::I64(x as i64), - Value::I32(x) => Value::I64(x as i64), - x => x, - }; - }) - }); - } - document - }) - .collect() - } - /// Creates a random Document using a seed if given, otherwise entropy. fn random_document( &self, diff --git a/packages/rs-random-document/Cargo.toml b/packages/rs-random-document/Cargo.toml new file mode 100644 index 0000000000..84088b9a12 --- /dev/null +++ b/packages/rs-random-document/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "random-document" +version = "1.0.0-dev.15" +edition = "2021" +rust-version = "1.76" +authors = [ + "Anton Suprunchuk ", + "Samuel Westrich ", + "Ivan Shumkov ", + "Djavid Gabibiyan ", + "Igor Markin ", +] + +[dependencies] +json-schema-faker = { git = "https://github.com/fominok/json-schema-faker-rs", rev = "2dd1cf7" } +dpp = { path = "../rs-dpp", features = ["system_contracts", "document-value-conversion"] } +platform-value = { path = "../rs-platform-value", features = ["json"]} +anyhow = "1.0.86" diff --git a/packages/rs-random-document/src/lib.rs b/packages/rs-random-document/src/lib.rs new file mode 100644 index 0000000000..0673fccc24 --- /dev/null +++ b/packages/rs-random-document/src/lib.rs @@ -0,0 +1,200 @@ +use anyhow::Context; +use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; +use dpp::data_contract::document_type::{DocumentType, DocumentTypeRef}; +use dpp::document::property_names::{ + CREATED_AT, FEATURE_VERSION, ID, OWNER_ID, REVISION, UPDATED_AT, +}; +use dpp::document::serialization_traits::DocumentPlatformValueMethodsV0; +use dpp::document::Document; +use dpp::version::PlatformVersion; +use dpp::ProtocolError; +use std::collections::BTreeMap; +use std::time::{SystemTime, UNIX_EPOCH}; + +use platform_value::{Bytes32, Identifier, Value}; + +pub trait SchemaAwareRandomDocument { + fn valid_random_documents( + &self, + owner_id: Identifier, + entropy: &Bytes32, + count: u32, + platform_version: &PlatformVersion, + substitutions: &BTreeMap<&str, Value>, + ) -> Result, ProtocolError>; +} + +impl<'a> SchemaAwareRandomDocument for DocumentTypeRef<'a> { + fn valid_random_documents( + &self, + owner_id: Identifier, + entropy: &Bytes32, + count: u32, + platform_version: &PlatformVersion, + substitutions: &BTreeMap<&str, Value>, + ) -> Result, ProtocolError> { + valid_random_documents( + self, + owner_id, + entropy, + count, + platform_version, + substitutions, + ) + } +} + +impl SchemaAwareRandomDocument for DocumentType { + fn valid_random_documents( + &self, + owner_id: Identifier, + entropy: &Bytes32, + count: u32, + platform_version: &PlatformVersion, + substitutions: &BTreeMap<&str, Value>, + ) -> Result, ProtocolError> { + valid_random_documents( + &self.as_ref(), + owner_id, + entropy, + count, + platform_version, + substitutions, + ) + } +} + +/// Create random documents using json-schema-faker-rs +fn valid_random_documents( + document_type: &DocumentTypeRef, + owner_id: Identifier, + entropy: &Bytes32, + count: u32, + platform_version: &PlatformVersion, + substitutions: &BTreeMap<&str, Value>, +) -> Result, ProtocolError> { + let json_schema = document_type.schema().try_to_validating_json()?; + let json_documents = json_schema_faker::generate(&json_schema, count as u16) + .context("cannot generate a random document with json-schema-faker-rs")?; + + let fix_document = |mut document: Value| { + let id = Document::generate_document_id_v0( + &document_type.data_contract_id(), + &owner_id, + document_type.name().as_str(), + entropy.as_slice(), + ); + let now = SystemTime::now(); + let duration_since_epoch = now.duration_since(UNIX_EPOCH).expect("Time went backwards"); + let time_ms = duration_since_epoch.as_millis() as u64; + + if document_type.documents_mutable() { + document.as_map_mut().into_iter().for_each(|d| { + d.push((REVISION.into(), 1.into())); + }); + } + document.as_map_mut().into_iter().for_each(|d| { + d.push((ID.into(), id.into())); + }); + document.as_map_mut().into_iter().for_each(|d| { + d.push((OWNER_ID.into(), owner_id.into())); + }); + if document_type.required_fields().contains(FEATURE_VERSION) { + document.as_map_mut().into_iter().for_each(|d| { + d.push((FEATURE_VERSION.into(), "0".into())); + }); + } + if document_type.required_fields().contains(CREATED_AT) { + document.as_map_mut().into_iter().for_each(|d| { + d.push((CREATED_AT.into(), time_ms.into())); + }); + } + if document_type.required_fields().contains(UPDATED_AT) { + document.as_map_mut().into_iter().for_each(|d| { + d.push((UPDATED_AT.into(), time_ms.into())); + }); + } + + document + }; + + json_documents + .into_iter() + .map(|d| { + let p_value: Value = d.into(); + let fixed_value = fix_document(p_value); + + // TODO: tl;dr use PlatformDeserialize instead of Deserialize for Documents + // + // `properties` is a `BTreeMap` with `platform_value::Value` as values, since + // `Document::from_platform_value` does deserialization through Serde's data model + // it losts some information like distinction between `Value::Bytes` and `Value::Bytes32`; + // The solution here is to let deserialize a `Document`, but put `properties` unprocessed + // since they were `platform_value::Value` and will be the same type again and no deserialization + // is needed, especially that lossy kind. + let mut properties = fixed_value + .to_map_ref() + .ok() + .and_then(|m| Value::map_into_btree_string_map(m.clone()).ok()) + .unwrap_or_default(); + let mut document = Document::from_platform_value(fixed_value, platform_version); + if let Ok(Document::V0(d)) = document.as_mut() { + // This moves stored properties back to the document so it could skip unnecessary + // and wrong deserialization part + d.properties.iter_mut().for_each(|(k, v)| { + substitutions + .get(k.as_str()) + .cloned() + .or(properties.remove(k)) + .into_iter() + .for_each(|prop| { + // TODO: schema and internal DocumentType representations are incompatible + // Properties are tweaked though, because the only integer type supported by + // DPP is i64, while `platform_value::Value` distincts them, and json schema is + // even more permissive; however, we want our proofs to work and proofs use the + // DPP model. + *v = match prop { + Value::U64(x) => Value::I64(x as i64), + Value::U32(x) => Value::I64(x as i64), + Value::I32(x) => Value::I64(x as i64), + x => x, + }; + }) + }); + } + document + }) + .collect() +} + +#[cfg(test)] +mod tests { + use dpp::dashcore::secp256k1::rand::rngs::StdRng; + use dpp::dashcore::secp256k1::rand::SeedableRng; + use dpp::data_contract::accessors::v0::DataContractV0Getters; + use dpp::system_data_contracts::{load_system_data_contract, SystemDataContract}; + + use super::*; + + #[test] + fn test_random_document_faker() { + let data_contract = + load_system_data_contract(SystemDataContract::DPNS, PlatformVersion::latest()).unwrap(); + let mut rng = StdRng::from_entropy(); + let entropy = Bytes32::random_with_rng(&mut rng); + + let _random_documents = data_contract + .document_types() + .iter() + .next() + .unwrap() + .1 + .valid_random_documents( + Identifier::random(), + &entropy, + 2, + PlatformVersion::latest(), + &Default::default(), + ); + } +} From 32d5d5a565f166eaf038276e81237604ce53bec9 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 19 Jun 2024 13:28:57 +0800 Subject: [PATCH 2/4] docs: updated authors --- packages/rs-random-document/Cargo.toml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/rs-random-document/Cargo.toml b/packages/rs-random-document/Cargo.toml index 84088b9a12..f599a29e42 100644 --- a/packages/rs-random-document/Cargo.toml +++ b/packages/rs-random-document/Cargo.toml @@ -4,11 +4,8 @@ version = "1.0.0-dev.15" edition = "2021" rust-version = "1.76" authors = [ - "Anton Suprunchuk ", - "Samuel Westrich ", - "Ivan Shumkov ", - "Djavid Gabibiyan ", - "Igor Markin ", + "Ivan Shumkov ", + "Evgeny Fomin ", ] [dependencies] From f7e4fbc88c2638ff1b667315abbf851cc5583501 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 19 Jun 2024 13:30:42 +0800 Subject: [PATCH 3/4] docs: updated authors again --- packages/rs-random-document/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rs-random-document/Cargo.toml b/packages/rs-random-document/Cargo.toml index f599a29e42..a2ea73a2f0 100644 --- a/packages/rs-random-document/Cargo.toml +++ b/packages/rs-random-document/Cargo.toml @@ -4,8 +4,8 @@ version = "1.0.0-dev.15" edition = "2021" rust-version = "1.76" authors = [ - "Ivan Shumkov ", "Evgeny Fomin ", + "Ivan Shumkov ", ] [dependencies] From 16f6029ef1d2c74da0f2835c8ec997f90f205d67 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 19 Jun 2024 14:48:34 +0800 Subject: [PATCH 4/4] Update Cargo.toml --- packages/rs-random-document/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rs-random-document/Cargo.toml b/packages/rs-random-document/Cargo.toml index a2ea73a2f0..47355b5b98 100644 --- a/packages/rs-random-document/Cargo.toml +++ b/packages/rs-random-document/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" rust-version = "1.76" authors = [ "Evgeny Fomin ", - "Ivan Shumkov ", + "Ivan Shumkov ", ] [dependencies]