From 48a151120c65ad5e4d4241a04412199f25ab511e Mon Sep 17 00:00:00 2001 From: pauldelucia Date: Tue, 21 May 2024 22:42:18 +0200 Subject: [PATCH 01/12] fix: select random identities for strategy documents --- packages/strategy-tests/src/lib.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/strategy-tests/src/lib.rs b/packages/strategy-tests/src/lib.rs index 3550524472..f68af5b0e5 100644 --- a/packages/strategy-tests/src/lib.rs +++ b/packages/strategy-tests/src/lib.rs @@ -569,8 +569,9 @@ impl Strategy { document_type, contract, }) => { - // Get the first 10 identities who are eligible to submit documents for this contract - let first_10_eligible_identities: Vec = current_identities + // Filter and collect eligible identities + // Eligible identities have less than 24 documents in the mempool for this contract + let eligible_identities: Vec = current_identities .iter() .filter(|identity| { mempool_document_counter @@ -578,10 +579,26 @@ impl Strategy { .unwrap_or(&0) < &24u64 }) - .take(10) .cloned() .collect(); + let k = 10; // Number of random elements to select + let mut reservoir = Vec::new(); + + // Reservoir sampling algorithm + for (i, identity) in eligible_identities.into_iter().enumerate() { + if i < k { + reservoir.push(identity); + } else { + let j = rng.gen_range(0..=i); + if j < k { + reservoir[j] = identity; + } + } + } + + let first_10_eligible_identities: Vec = reservoir; + if first_10_eligible_identities.len() == 0 { tracing::warn!( "No eligible identities to submit a document to contract {}", From afb16a86e9fc94da34f40df1626adc64fc8cd93d Mon Sep 17 00:00:00 2001 From: pauldelucia Date: Wed, 22 May 2024 18:22:22 +0200 Subject: [PATCH 02/12] fix: improve efficiency of identity random sampling --- packages/strategy-tests/src/lib.rs | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/packages/strategy-tests/src/lib.rs b/packages/strategy-tests/src/lib.rs index f68af5b0e5..73f2973a0a 100644 --- a/packages/strategy-tests/src/lib.rs +++ b/packages/strategy-tests/src/lib.rs @@ -42,7 +42,7 @@ use dpp::state_transition::data_contract_update_transition::methods::DataContrac use operations::{DataContractUpdateAction, DataContractUpdateOp}; use platform_version::TryFromPlatformVersioned; use rand::prelude::StdRng; -use rand::seq::SliceRandom; +use rand::seq::{IteratorRandom, SliceRandom}; use rand::Rng; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use bincode::{Decode, Encode}; @@ -571,7 +571,7 @@ impl Strategy { }) => { // Filter and collect eligible identities // Eligible identities have less than 24 documents in the mempool for this contract - let eligible_identities: Vec = current_identities + let first_10_eligible_identities: Vec = current_identities .iter() .filter(|identity| { mempool_document_counter @@ -580,24 +580,7 @@ impl Strategy { < &24u64 }) .cloned() - .collect(); - - let k = 10; // Number of random elements to select - let mut reservoir = Vec::new(); - - // Reservoir sampling algorithm - for (i, identity) in eligible_identities.into_iter().enumerate() { - if i < k { - reservoir.push(identity); - } else { - let j = rng.gen_range(0..=i); - if j < k { - reservoir[j] = identity; - } - } - } - - let first_10_eligible_identities: Vec = reservoir; + .choose_multiple(rng, 10); if first_10_eligible_identities.len() == 0 { tracing::warn!( From 9e5d088b4cfa37335c1315104dc9fbc77e3f9610 Mon Sep 17 00:00:00 2001 From: pauldelucia Date: Wed, 22 May 2024 18:45:52 +0200 Subject: [PATCH 03/12] fix: only clone the eligible identities --- packages/strategy-tests/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/strategy-tests/src/lib.rs b/packages/strategy-tests/src/lib.rs index 73f2973a0a..cd971a6ea7 100644 --- a/packages/strategy-tests/src/lib.rs +++ b/packages/strategy-tests/src/lib.rs @@ -579,8 +579,10 @@ impl Strategy { .unwrap_or(&0) < &24u64 }) + .choose_multiple(rng, 10) + .into_iter() .cloned() - .choose_multiple(rng, 10); + .collect(); if first_10_eligible_identities.len() == 0 { tracing::warn!( From f85d9cb35aa9a0abe9b61bc9730caddae1f972b0 Mon Sep 17 00:00:00 2001 From: pauldelucia Date: Wed, 22 May 2024 20:29:38 +0200 Subject: [PATCH 04/12] feat: use all eligible identities and slightly more robust checking --- packages/strategy-tests/src/lib.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/strategy-tests/src/lib.rs b/packages/strategy-tests/src/lib.rs index cd971a6ea7..65e91ee23c 100644 --- a/packages/strategy-tests/src/lib.rs +++ b/packages/strategy-tests/src/lib.rs @@ -571,7 +571,7 @@ impl Strategy { }) => { // Filter and collect eligible identities // Eligible identities have less than 24 documents in the mempool for this contract - let first_10_eligible_identities: Vec = current_identities + let eligible_identities: Vec = current_identities .iter() .filter(|identity| { mempool_document_counter @@ -579,23 +579,35 @@ impl Strategy { .unwrap_or(&0) < &24u64 }) - .choose_multiple(rng, 10) .into_iter() .cloned() .collect(); - if first_10_eligible_identities.len() == 0 { + // Check we are able to submit `count` docs + // This also prevents the case where there are no eligible identities + let mut num_docs_possible: i32 = 0; + for identity in &eligible_identities { + let num_docs_possible_identity: i32 = (24 + - mempool_document_counter + .get(&(identity.id(), contract.id())) + .unwrap_or(&0)) + .try_into() + .unwrap(); + num_docs_possible += num_docs_possible_identity; + } + if !(num_docs_possible >= count as i32) { tracing::warn!( - "No eligible identities to submit a document to contract {}", - contract.id().to_string(Encoding::Base64) + "No eligible identities to submit documents to contract {}", + contract.id().to_string(Encoding::Base58) ); + continue; } // TO-DO: these documents should be created according to the data contract's validation rules let documents = document_type .random_documents_with_params( count as u32, - &first_10_eligible_identities, + &eligible_identities, Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), From 4ae93abf1bed616a5957b03a04b34f6c688e784d Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 23 May 2024 20:59:59 +0700 Subject: [PATCH 05/12] chore: chose capable identities for random documents --- .../document_type/random_document.rs | 18 +-- .../document_type/v0/random_document.rs | 16 ++- .../tests/strategy_tests/strategy.rs | 28 +++- packages/strategy-tests/src/lib.rs | 129 ++++++++++++------ 4 files changed, 126 insertions(+), 65 deletions(-) diff --git a/packages/rs-dpp/src/data_contract/document_type/random_document.rs b/packages/rs-dpp/src/data_contract/document_type/random_document.rs index 258b58d79b..8fd83d3ca0 100644 --- a/packages/rs-dpp/src/data_contract/document_type/random_document.rs +++ b/packages/rs-dpp/src/data_contract/document_type/random_document.rs @@ -197,10 +197,10 @@ pub trait CreateRandomDocument { /// A `Result, ProtocolError>` which is `Ok` containing a vector of tuples /// if successful, each tuple consisting of a Document, its associated Identity, and a Bytes32 value, or an error /// if the operation fails. - fn random_documents_with_params( + fn random_documents_with_params<'i>( &self, count: u32, - identities: &[Identity], + identities: &[&'i Identity], time_ms: Option, block_height: Option, core_block_height: Option, @@ -208,7 +208,7 @@ pub trait CreateRandomDocument { document_field_fill_size: DocumentFieldFillSize, rng: &mut StdRng, platform_version: &PlatformVersion, - ) -> Result, ProtocolError>; + ) -> Result, ProtocolError>; } impl CreateRandomDocument for DocumentType { @@ -317,10 +317,10 @@ impl CreateRandomDocument for DocumentType { ), // Add more cases as necessary for other variants } } - fn random_documents_with_params( + fn random_documents_with_params<'i>( &self, count: u32, - identities: &[Identity], + identities: &[&'i Identity], time_ms: Option, block_height: Option, core_block_height: Option, @@ -328,7 +328,7 @@ impl CreateRandomDocument for DocumentType { document_field_fill_size: DocumentFieldFillSize, rng: &mut StdRng, platform_version: &PlatformVersion, - ) -> Result, ProtocolError> { + ) -> Result, ProtocolError> { match self { DocumentType::V0(v0) => v0.random_documents_with_params( count, @@ -452,10 +452,10 @@ impl<'a> CreateRandomDocument for DocumentTypeRef<'a> { } } - fn random_documents_with_params( + fn random_documents_with_params<'i>( &self, count: u32, - identities: &[Identity], + identities: &[&'i Identity], time_ms: Option, block_height: Option, core_block_height: Option, @@ -463,7 +463,7 @@ impl<'a> CreateRandomDocument for DocumentTypeRef<'a> { document_field_fill_size: DocumentFieldFillSize, rng: &mut StdRng, platform_version: &PlatformVersion, - ) -> Result, ProtocolError> { + ) -> Result, ProtocolError> { match self { DocumentTypeRef::V0(v0) => v0.random_documents_with_params( count, diff --git a/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs b/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs index cfaeece525..09f01a843b 100644 --- a/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs +++ b/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs @@ -386,7 +386,7 @@ impl CreateRandomDocument for DocumentTypeV0 { fn random_documents_with_params( &self, count: u32, - identities: &[Identity], + identities: &[&Identity], time_ms: Option, block_height: Option, core_block_height: Option, @@ -394,11 +394,17 @@ impl CreateRandomDocument for DocumentTypeV0 { document_field_fill_size: DocumentFieldFillSize, rng: &mut StdRng, platform_version: &PlatformVersion, - ) -> Result, ProtocolError> { + ) -> Result, ProtocolError> { let mut vec = vec![]; - for _i in 0..count { - let identity_num = rng.gen_range(0..identities.len()); - let identity = identities.get(identity_num).unwrap().clone(); + + if identities.len() < count as usize { + return Err(ProtocolError::CorruptedCodeExecution(format!( + "expected {count} identities" + ))); + } + + for i in 0..count { + let identity = identities.get(i).expect("identity with index must exist"); let entropy = Bytes32::random_with_rng(rng); vec.push(( self.random_document_with_params( diff --git a/packages/rs-drive-abci/tests/strategy_tests/strategy.rs b/packages/rs-drive-abci/tests/strategy_tests/strategy.rs index dae0d7acc8..13d7072749 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/strategy.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/strategy.rs @@ -532,17 +532,26 @@ impl NetworkStrategy { let mut deleted = vec![]; for op in &self.strategy.operations { if op.frequency.check_hit(rng) { - let count = rng.gen_range(op.frequency.times_per_block_range.clone()); + let mut count = rng.gen_range(op.frequency.times_per_block_range.clone()); match &op.op_type { OperationType::Document(DocumentOp { action: DocumentAction::DocumentActionInsertRandom(fill_type, fill_size), document_type, contract, }) => { + if current_identities.len() < count as usize { + count = current_identities.len() as u16; + + tracing::warn!( + "Not enough identities to insert documents, reducing count to {}", + count + ); + } + let documents = document_type .random_documents_with_params( count as u32, - current_identities, + current_identities.as_ref(), Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), @@ -631,15 +640,20 @@ impl NetworkStrategy { contract, }) => { let documents = if let Some(identifier) = identifier { - let held_identity = vec![current_identities + let held_identity = current_identities .iter() .find(|identity| identity.id() == identifier) - .expect("expected to find identifier, review strategy params") - .clone()]; + .expect("expected to find identifier, review strategy params"); + + let mut eligible_identities = Vec::with_capacity(count as usize); + for _ in 0..count { + eligible_identities.push(held_identity); + } + document_type .random_documents_with_params( count as u32, - &held_identity, + &eligible_identities, Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), @@ -653,7 +667,7 @@ impl NetworkStrategy { document_type .random_documents_with_params( count as u32, - current_identities, + current_identities.as_ref(), Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), diff --git a/packages/strategy-tests/src/lib.rs b/packages/strategy-tests/src/lib.rs index 65e91ee23c..4a3beb2575 100644 --- a/packages/strategy-tests/src/lib.rs +++ b/packages/strategy-tests/src/lib.rs @@ -323,6 +323,54 @@ impl PlatformDeserializableWithPotentialValidationFromVersionedStructure for Str } } +type MempoolDocumentCounter = &BTreeMap<(Identifier, Identifier), u64>; + +fn choose_capable_identities<'i>(identities: &'i [Identity], contract: &DataContract, mempool_document_counter: MempoolDocumentCounter, count: usize, rng: &mut StdRng) -> Vec<&'i Identity> { + let mut all_capable_identities = identities + .iter() + .filter(|identity| { + mempool_document_counter + .get(&(identity.id(), contract.id())) + .unwrap_or(&0) + < &24u64 + }) + .collect(); + + if all_capable_identities.len() < count { + let additional_identities_count = count - all_capable_identities.len(); + + let mut additional_identities = Vec::with_capacity(additional_identities_count); + let mut index = 0; + while additional_identities_count > 0 && index < all_capable_identities.len() { + let identity = all_capable_identities[index]; + + // How many documents are in the mempool for this contract including + // one what we are already planning to use + let mempool_documents_count = mempool_document_counter + .get(&(identity.id(), contract.id())) + .unwrap_or(&0) + 1; + + let identity_capacity = mempool_documents_count - 24; + + if identity_capacity > 0 { + for _ in 0..identity_capacity { + additional_identities.push(identity); + additional_identities_count -= 1; + if additional_identities_count == 0 { + break; + } + } + } + + index += 1; + } + + all_capable_identities.extend(additional_identities) + } else { + all_capable_identities.iter().choose_multiple(rng, count) + } +} + impl Strategy { /// Generates comprehensive state transitions for a given block, including handling new identities and contracts. /// @@ -543,7 +591,7 @@ impl Strategy { signer: &mut SimpleSigner, identity_nonce_counter: &mut BTreeMap, contract_nonce_counter: &mut BTreeMap<(Identifier, Identifier), u64>, - mempool_document_counter: BTreeMap<(Identifier, Identifier), u64>, + mempool_document_counter: &BTreeMap<(Identifier, Identifier), u64>, rng: &mut StdRng, platform_version: &PlatformVersion, ) -> (Vec, Vec) { @@ -569,42 +617,16 @@ impl Strategy { document_type, contract, }) => { - // Filter and collect eligible identities - // Eligible identities have less than 24 documents in the mempool for this contract - let eligible_identities: Vec = current_identities - .iter() - .filter(|identity| { - mempool_document_counter - .get(&(identity.id(), contract.id())) - .unwrap_or(&0) - < &24u64 - }) - .into_iter() - .cloned() - .collect(); - - // Check we are able to submit `count` docs - // This also prevents the case where there are no eligible identities - let mut num_docs_possible: i32 = 0; - for identity in &eligible_identities { - let num_docs_possible_identity: i32 = (24 - - mempool_document_counter - .get(&(identity.id(), contract.id())) - .unwrap_or(&0)) - .try_into() - .unwrap(); - num_docs_possible += num_docs_possible_identity; - } - if !(num_docs_possible >= count as i32) { - tracing::warn!( - "No eligible identities to submit documents to contract {}", - contract.id().to_string(Encoding::Base58) - ); - continue; - } + let eligible_identities = choose_capable_identities( + current_identities, + contract, + mempool_document_counter, + count as usize, + rng, + ); // TO-DO: these documents should be created according to the data contract's validation rules - let documents = document_type + let documents_with_identity_and_entropy = document_type .random_documents_with_params( count as u32, &eligible_identities, @@ -618,9 +640,8 @@ impl Strategy { ) .expect("expected random_documents_with_params"); - documents - .into_iter() - .for_each(|(document, identity, entropy)| { + documents_with_identity_and_entropy.into_iter().for_each( + |(document, identity, entropy)| { let identity_contract_nonce = if contract.owner_id() == identity.id() { contract_nonce_counter @@ -687,7 +708,8 @@ impl Strategy { .expect("expected to sign"); operations.push(document_batch_transition); - }); + }, + ); } // Generate state transition for specific document insert operation @@ -703,15 +725,20 @@ impl Strategy { contract, }) => { let documents = if let Some(identifier) = identifier { - let held_identity = vec![current_identities + let held_identity = current_identities .iter() .find(|identity| identity.id() == identifier) - .expect("expected to find identifier, review strategy params") - .clone()]; + .expect("expected to find identifier, review strategy params"); + + let mut eligible_identities = Vec::with_capacity(count as usize); + for _ in 0..count { + eligible_identities.push(held_identity); + } + document_type .random_documents_with_params( count as u32, - &held_identity, + &eligible_identities, Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), @@ -722,10 +749,24 @@ impl Strategy { ) .expect("expected random_documents_with_params") } else { + let eligible_identities = if current_identities.len() < count as usize { + let identities = Vec::with_capacity(count as usize); + for _ in 0..(count as usize - current_identities.len()) { + let identity = Identity::new( + current_identities.extend( + + ); + ); + + let mut eligible_identities = Vec::with_capacity(count as usize); + for _ in 0..count { + eligible_identities.push(held_identity); + } + document_type .random_documents_with_params( count as u32, - current_identities, + current_identities.as_ref(), Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), From a7e05eb55cbc45c0657cd5c828c7770705e064a5 Mon Sep 17 00:00:00 2001 From: pauldelucia Date: Thu, 23 May 2024 20:14:27 +0200 Subject: [PATCH 06/12] clean up --- .../document_type/v0/random_document.rs | 10 +-- .../tests/strategy_tests/strategy.rs | 10 ++- packages/strategy-tests/src/lib.rs | 61 +++++++++++-------- 3 files changed, 49 insertions(+), 32 deletions(-) diff --git a/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs b/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs index 09f01a843b..213e314abd 100644 --- a/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs +++ b/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs @@ -13,7 +13,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use platform_value::Value; use platform_value::{Bytes32, Identifier}; use rand::rngs::StdRng; -use rand::{Rng, SeedableRng}; +use rand::SeedableRng; use crate::data_contract::document_type::methods::DocumentTypeV0Methods; use crate::data_contract::document_type::random_document::{ @@ -383,10 +383,10 @@ impl CreateRandomDocument for DocumentTypeV0 { } /// Creates `count` Documents with random data using the random number generator given. - fn random_documents_with_params( + fn random_documents_with_params<'i>( &self, count: u32, - identities: &[&Identity], + identities: &[&'i Identity], time_ms: Option, block_height: Option, core_block_height: Option, @@ -394,7 +394,7 @@ impl CreateRandomDocument for DocumentTypeV0 { document_field_fill_size: DocumentFieldFillSize, rng: &mut StdRng, platform_version: &PlatformVersion, - ) -> Result, ProtocolError> { + ) -> Result, ProtocolError> { let mut vec = vec![]; if identities.len() < count as usize { @@ -404,7 +404,7 @@ impl CreateRandomDocument for DocumentTypeV0 { } for i in 0..count { - let identity = identities.get(i).expect("identity with index must exist"); + let identity = identities[i as usize]; let entropy = Bytes32::random_with_rng(rng); vec.push(( self.random_document_with_params( diff --git a/packages/rs-drive-abci/tests/strategy_tests/strategy.rs b/packages/rs-drive-abci/tests/strategy_tests/strategy.rs index 13d7072749..7b48fbda91 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/strategy.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/strategy.rs @@ -548,10 +548,13 @@ impl NetworkStrategy { ); } + let current_identities_as_refs: Vec<&dpp::identity::Identity> = + current_identities.iter().collect(); + let documents = document_type .random_documents_with_params( count as u32, - current_identities.as_ref(), + current_identities_as_refs.as_ref(), Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), @@ -664,10 +667,13 @@ impl NetworkStrategy { ) .expect("expected random_documents_with_params") } else { + let current_identities_as_refs: Vec<&dpp::identity::Identity> = + current_identities.iter().collect(); + document_type .random_documents_with_params( count as u32, - current_identities.as_ref(), + current_identities_as_refs.as_ref(), Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), diff --git a/packages/strategy-tests/src/lib.rs b/packages/strategy-tests/src/lib.rs index 4a3beb2575..ae01eda093 100644 --- a/packages/strategy-tests/src/lib.rs +++ b/packages/strategy-tests/src/lib.rs @@ -323,10 +323,17 @@ impl PlatformDeserializableWithPotentialValidationFromVersionedStructure for Str } } -type MempoolDocumentCounter = &BTreeMap<(Identifier, Identifier), u64>; - -fn choose_capable_identities<'i>(identities: &'i [Identity], contract: &DataContract, mempool_document_counter: MempoolDocumentCounter, count: usize, rng: &mut StdRng) -> Vec<&'i Identity> { - let mut all_capable_identities = identities +type MempoolDocumentCounter<'c> = &'c BTreeMap<(Identifier, Identifier), u64>; + +fn choose_capable_identities<'i>( + identities: &'i [Identity], + contract: &DataContract, + mempool_document_counter: MempoolDocumentCounter, + count: usize, + rng: &mut StdRng, +) -> Vec<&'i Identity> { + // Filter out those that already have 24 documents in the mempool for this contract + let mut all_capable_identities: Vec<&'i Identity> = identities .iter() .filter(|identity| { mempool_document_counter @@ -336,21 +343,25 @@ fn choose_capable_identities<'i>(identities: &'i [Identity], contract: &DataCont }) .collect(); + // If we need more documents than we have eligible identities, we will need to use some identities more than once if all_capable_identities.len() < count { - let additional_identities_count = count - all_capable_identities.len(); + let mut additional_identities_count = count - all_capable_identities.len(); let mut additional_identities = Vec::with_capacity(additional_identities_count); let mut index = 0; while additional_identities_count > 0 && index < all_capable_identities.len() { + // Iterate through the identities only one time, using as many documents as each identity can submit + // We should see if this often causes identities to max out their mempool count, which could potentially + // lead to issues, even though this function should really only be covering an edge case let identity = all_capable_identities[index]; - // How many documents are in the mempool for this contract including - // one what we are already planning to use + // How many documents are in the mempool for this (identity, contract) plus the one that we are already planning to use let mempool_documents_count = mempool_document_counter .get(&(identity.id(), contract.id())) - .unwrap_or(&0) + 1; + .unwrap_or(&0) + + 1; - let identity_capacity = mempool_documents_count - 24; + let identity_capacity = 24 - mempool_documents_count; if identity_capacity > 0 { for _ in 0..identity_capacity { @@ -365,9 +376,15 @@ fn choose_capable_identities<'i>(identities: &'i [Identity], contract: &DataCont index += 1; } - all_capable_identities.extend(additional_identities) + all_capable_identities.extend(additional_identities); + all_capable_identities } else { - all_capable_identities.iter().choose_multiple(rng, count) + all_capable_identities + .iter() + .choose_multiple(rng, count) + .into_iter() + .cloned() + .collect() } } @@ -442,7 +459,7 @@ impl Strategy { signer: &mut SimpleSigner, identity_nonce_counter: &mut BTreeMap, contract_nonce_counter: &mut BTreeMap<(Identifier, Identifier), u64>, - mempool_document_counter: BTreeMap<(Identifier, Identifier), u64>, + mempool_document_counter: &BTreeMap<(Identifier, Identifier), u64>, rng: &mut StdRng, config: &StrategyConfig, platform_version: &PlatformVersion, @@ -749,24 +766,18 @@ impl Strategy { ) .expect("expected random_documents_with_params") } else { - let eligible_identities = if current_identities.len() < count as usize { - let identities = Vec::with_capacity(count as usize); - for _ in 0..(count as usize - current_identities.len()) { - let identity = Identity::new( - current_identities.extend( - - ); + let eligible_identities = choose_capable_identities( + current_identities, + contract, + mempool_document_counter, + count as usize, + rng, ); - let mut eligible_identities = Vec::with_capacity(count as usize); - for _ in 0..count { - eligible_identities.push(held_identity); - } - document_type .random_documents_with_params( count as u32, - current_identities.as_ref(), + &eligible_identities, Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), From e091a64f5abae2f2b025dea7c690da0d2f978b50 Mon Sep 17 00:00:00 2001 From: pauldelucia Date: Sat, 25 May 2024 08:49:19 +0200 Subject: [PATCH 07/12] handle panics --- .../document_type/v0/random_document.rs | 2 +- packages/strategy-tests/src/lib.rs | 34 +++++++++++-------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs b/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs index 213e314abd..8237489906 100644 --- a/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs +++ b/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs @@ -399,7 +399,7 @@ impl CreateRandomDocument for DocumentTypeV0 { if identities.len() < count as usize { return Err(ProtocolError::CorruptedCodeExecution(format!( - "expected {count} identities" + "not enough identities to create {count} documents" ))); } diff --git a/packages/strategy-tests/src/lib.rs b/packages/strategy-tests/src/lib.rs index ae01eda093..2c2439b38e 100644 --- a/packages/strategy-tests/src/lib.rs +++ b/packages/strategy-tests/src/lib.rs @@ -50,7 +50,7 @@ use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; use dpp::identifier::Identifier; use dpp::data_contract::document_type::DocumentType; use dpp::identity::accessors::IdentityGettersV0; -use dpp::platform_value::{BinaryData, Value}; +use dpp::platform_value::{BinaryData, Bytes32, Value}; use dpp::ProtocolError; use dpp::ProtocolError::{PlatformDeserializationError, PlatformSerializationError}; use dpp::state_transition::documents_batch_transition::document_base_transition::v0::DocumentBaseTransitionV0; @@ -643,19 +643,25 @@ impl Strategy { ); // TO-DO: these documents should be created according to the data contract's validation rules - let documents_with_identity_and_entropy = document_type - .random_documents_with_params( - count as u32, - &eligible_identities, - Some(block_info.time_ms), - Some(block_info.height), - Some(block_info.core_height), - *fill_type, - *fill_size, - rng, - platform_version, - ) - .expect("expected random_documents_with_params"); + let mut documents_with_identity_and_entropy: Vec<( + Document, + &Identity, + Bytes32, + )> = Vec::new(); + match document_type.random_documents_with_params( + count as u32, + &eligible_identities, + Some(block_info.time_ms), + Some(block_info.height), + Some(block_info.core_height), + *fill_type, + *fill_size, + rng, + platform_version, + ) { + Ok(documents) => documents_with_identity_and_entropy = documents, + Err(e) => tracing::warn!("Failed to create random documents: {}", e), + } documents_with_identity_and_entropy.into_iter().for_each( |(document, identity, entropy)| { From 71c4c1efdface31bee88b063c6882cad276374e3 Mon Sep 17 00:00:00 2001 From: pauldelucia Date: Sat, 25 May 2024 17:39:11 +0200 Subject: [PATCH 08/12] use MempoolDocumentCounter type --- packages/strategy-tests/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/strategy-tests/src/lib.rs b/packages/strategy-tests/src/lib.rs index 2c2439b38e..89831748f1 100644 --- a/packages/strategy-tests/src/lib.rs +++ b/packages/strategy-tests/src/lib.rs @@ -323,7 +323,7 @@ impl PlatformDeserializableWithPotentialValidationFromVersionedStructure for Str } } -type MempoolDocumentCounter<'c> = &'c BTreeMap<(Identifier, Identifier), u64>; +pub type MempoolDocumentCounter<'c> = &'c BTreeMap<(Identifier, Identifier), u64>; fn choose_capable_identities<'i>( identities: &'i [Identity], @@ -445,7 +445,7 @@ impl Strategy { /// This function is central to simulating the lifecycle of block processing and strategy execution /// on the Dash Platform. It encapsulates the complexity of transaction generation, identity management, /// and contract dynamics within a block's context. - pub async fn state_transitions_for_block( + pub fn state_transitions_for_block( &mut self, document_query_callback: &mut impl FnMut(LocalDocumentQuery) -> Vec, identity_fetch_callback: &mut impl FnMut( @@ -459,7 +459,7 @@ impl Strategy { signer: &mut SimpleSigner, identity_nonce_counter: &mut BTreeMap, contract_nonce_counter: &mut BTreeMap<(Identifier, Identifier), u64>, - mempool_document_counter: &BTreeMap<(Identifier, Identifier), u64>, + mempool_document_counter: MempoolDocumentCounter, rng: &mut StdRng, config: &StrategyConfig, platform_version: &PlatformVersion, @@ -608,7 +608,7 @@ impl Strategy { signer: &mut SimpleSigner, identity_nonce_counter: &mut BTreeMap, contract_nonce_counter: &mut BTreeMap<(Identifier, Identifier), u64>, - mempool_document_counter: &BTreeMap<(Identifier, Identifier), u64>, + mempool_document_counter: MempoolDocumentCounter, rng: &mut StdRng, platform_version: &PlatformVersion, ) -> (Vec, Vec) { From 449d30ef90f085ff637b69346f10cd00af02e6af Mon Sep 17 00:00:00 2001 From: pauldelucia Date: Tue, 28 May 2024 11:33:35 -0400 Subject: [PATCH 09/12] make MempoolDocumentCounter type not a reference --- packages/strategy-tests/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/strategy-tests/src/lib.rs b/packages/strategy-tests/src/lib.rs index 89831748f1..e2907cfc81 100644 --- a/packages/strategy-tests/src/lib.rs +++ b/packages/strategy-tests/src/lib.rs @@ -323,12 +323,12 @@ impl PlatformDeserializableWithPotentialValidationFromVersionedStructure for Str } } -pub type MempoolDocumentCounter<'c> = &'c BTreeMap<(Identifier, Identifier), u64>; +pub type MempoolDocumentCounter<'c> = BTreeMap<(Identifier, Identifier), u64>; fn choose_capable_identities<'i>( identities: &'i [Identity], contract: &DataContract, - mempool_document_counter: MempoolDocumentCounter, + mempool_document_counter: &MempoolDocumentCounter, count: usize, rng: &mut StdRng, ) -> Vec<&'i Identity> { @@ -459,7 +459,7 @@ impl Strategy { signer: &mut SimpleSigner, identity_nonce_counter: &mut BTreeMap, contract_nonce_counter: &mut BTreeMap<(Identifier, Identifier), u64>, - mempool_document_counter: MempoolDocumentCounter, + mempool_document_counter: &MempoolDocumentCounter, rng: &mut StdRng, config: &StrategyConfig, platform_version: &PlatformVersion, @@ -608,7 +608,7 @@ impl Strategy { signer: &mut SimpleSigner, identity_nonce_counter: &mut BTreeMap, contract_nonce_counter: &mut BTreeMap<(Identifier, Identifier), u64>, - mempool_document_counter: MempoolDocumentCounter, + mempool_document_counter: &MempoolDocumentCounter, rng: &mut StdRng, platform_version: &PlatformVersion, ) -> (Vec, Vec) { From d0035ae9edb1e24782f24b1d259b933208b57db9 Mon Sep 17 00:00:00 2001 From: pauldelucia Date: Tue, 11 Jun 2024 13:05:33 -0400 Subject: [PATCH 10/12] fix tests --- packages/rs-drive-abci/tests/strategy_tests/main.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/rs-drive-abci/tests/strategy_tests/main.rs b/packages/rs-drive-abci/tests/strategy_tests/main.rs index 43ea171e36..c8a70111f1 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/main.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/main.rs @@ -1778,7 +1778,7 @@ mod tests { .unwrap() .unwrap() ), - "10fbe38850d752b491305b2587468788acbd6dd130272c5a8844d0da05e3a9cf".to_string() + "0f16c735739411f8dff9640b3ca244b5838480f5634b8ab7744743b116fd9218".to_string() ) } @@ -1903,7 +1903,7 @@ mod tests { .unwrap() .unwrap() ), - "9ed86b1ed5d0f7a981cb0ff2f3943603a095a031c463b3389bce8a6f60e2b21e".to_string() + "a20b7973b60f51edbef1a3a378c6b0fa64e3de917754c0ddc78dc30cff7fd9f6".to_string() ) } @@ -2238,7 +2238,7 @@ mod tests { .build_with_mock_rpc(); let outcome = run_chain_for_strategy(&mut platform, block_count, strategy, config, 15); - assert_eq!(outcome.identities.len() as u64, 470); + assert_eq!(outcome.identities.len() as u64, 472); assert_eq!(outcome.masternode_identity_balances.len(), 100); let balance_count = outcome .masternode_identity_balances @@ -2367,7 +2367,7 @@ mod tests { .build_with_mock_rpc(); let outcome = run_chain_for_strategy(&mut platform, block_count, strategy, config, 15); - assert_eq!(outcome.identities.len() as u64, 90); + assert_eq!(outcome.identities.len() as u64, 83); assert_eq!(outcome.masternode_identity_balances.len(), 100); let balance_count = outcome .masternode_identity_balances @@ -2512,7 +2512,7 @@ mod tests { .build_with_mock_rpc(); let outcome = run_chain_for_strategy(&mut platform, block_count, strategy, config, 15); - assert_eq!(outcome.identities.len() as u64, 97); + assert_eq!(outcome.identities.len() as u64, 79); assert_eq!(outcome.masternode_identity_balances.len(), 100); let balance_count = outcome .masternode_identity_balances From 2704188231ba979cdce2db56f1825825891431bd Mon Sep 17 00:00:00 2001 From: pauldelucia Date: Fri, 28 Jun 2024 00:19:45 +0900 Subject: [PATCH 11/12] use new tenderdash version --- Cargo.lock | 34 +++++++++------------ packages/dapi-grpc/Cargo.toml | 2 +- packages/rs-drive-abci/Cargo.toml | 2 +- packages/rs-drive-proof-verifier/Cargo.toml | 2 +- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5e046b964..712d71925f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -409,7 +409,7 @@ dependencies = [ "bitflags 2.5.0", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.10.5", "lazy_static", "lazycell", "proc-macro2", @@ -2494,7 +2494,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.6", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -2813,7 +2813,7 @@ source = "git+https://github.com/fominok/jsonschema-rs?branch=feat-unevaluated-p dependencies = [ "ahash 0.8.11", "anyhow", - "base64 0.22.0", + "base64 0.13.1", "bytecount", "fancy-regex", "fraction", @@ -3175,12 +3175,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "multimap" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" - [[package]] name = "murmur3" version = "0.5.2" @@ -3819,7 +3813,7 @@ dependencies = [ "itertools 0.10.5", "lazy_static", "log", - "multimap 0.8.3", + "multimap", "petgraph", "prettyplease 0.1.25", "prost 0.11.9", @@ -3837,10 +3831,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1" dependencies = [ "bytes", - "heck 0.5.0", - "itertools 0.12.1", + "heck 0.4.1", + "itertools 0.10.5", "log", - "multimap 0.10.0", + "multimap", "once_cell", "petgraph", "prettyplease 0.2.19", @@ -3871,7 +3865,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.60", @@ -4963,8 +4957,8 @@ dependencies = [ [[package]] name = "tenderdash-abci" -version = "0.14.0-dev.12" -source = "git+https://github.com/dashpay/rs-tenderdash-abci#a848ad6d67b1c1d8f053c9ca97b89e5e4c929d96" +version = "1.0.0-dev.1" +source = "git+https://github.com/dashpay/rs-tenderdash-abci#cfb6ae41b4100c4b0af88ad1a201c5266a2ddc8c" dependencies = [ "bytes", "futures", @@ -4985,8 +4979,8 @@ dependencies = [ [[package]] name = "tenderdash-proto" -version = "0.14.0-dev.12" -source = "git+https://github.com/dashpay/rs-tenderdash-abci#a848ad6d67b1c1d8f053c9ca97b89e5e4c929d96" +version = "1.0.0-dev.1" +source = "git+https://github.com/dashpay/rs-tenderdash-abci#cfb6ae41b4100c4b0af88ad1a201c5266a2ddc8c" dependencies = [ "bytes", "chrono", @@ -5005,8 +4999,8 @@ dependencies = [ [[package]] name = "tenderdash-proto-compiler" -version = "0.14.0-dev.12" -source = "git+https://github.com/dashpay/rs-tenderdash-abci#a848ad6d67b1c1d8f053c9ca97b89e5e4c929d96" +version = "1.0.0-dev.1" +source = "git+https://github.com/dashpay/rs-tenderdash-abci#cfb6ae41b4100c4b0af88ad1a201c5266a2ddc8c" dependencies = [ "fs_extra", "prost-build 0.12.4", diff --git a/packages/dapi-grpc/Cargo.toml b/packages/dapi-grpc/Cargo.toml index 21e63ea3e3..085300789d 100644 --- a/packages/dapi-grpc/Cargo.toml +++ b/packages/dapi-grpc/Cargo.toml @@ -42,7 +42,7 @@ tonic = { version = "0.11", features = [ serde = { version = "1.0.197", optional = true, features = ["derive"] } serde_bytes = { version = "0.11.12", optional = true } serde_json = { version = "1.0", optional = true } -tenderdash-proto = { git = "https://github.com/dashpay/rs-tenderdash-abci", version = "0.14.0-dev.12", default-features = false } +tenderdash-proto = { git = "https://github.com/dashpay/rs-tenderdash-abci", version = "1.0.0-dev.1", default-features = false } dapi-grpc-macros = { path = "../rs-dapi-grpc-macros" } platform-version = { path = "../rs-platform-version" } diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index eb01bfd39d..378bfd3b22 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -55,7 +55,7 @@ tracing-subscriber = { version = "0.3.16", default-features = false, features = "tracing-log", ], optional = false } atty = { version = "0.2.14", optional = false } -tenderdash-abci = { git = "https://github.com/dashpay/rs-tenderdash-abci", version = "0.14.0-dev.12", features = [ +tenderdash-abci = { git = "https://github.com/dashpay/rs-tenderdash-abci", version = "1.0.0-dev.1", features = [ "grpc", ] } lazy_static = "1.4.0" diff --git a/packages/rs-drive-proof-verifier/Cargo.toml b/packages/rs-drive-proof-verifier/Cargo.toml index 685f0727ba..220726ac76 100644 --- a/packages/rs-drive-proof-verifier/Cargo.toml +++ b/packages/rs-drive-proof-verifier/Cargo.toml @@ -19,7 +19,7 @@ drive = { path = "../rs-drive", default-features = false, features = [ "verify", ] } dpp = { path = "../rs-dpp", features = ["bls-signatures"], default-features = false } -tenderdash-abci = { git = "https://github.com/dashpay/rs-tenderdash-abci", version = "0.14.0-dev.12", features = [ +tenderdash-abci = { git = "https://github.com/dashpay/rs-tenderdash-abci", version = "1.0.0-dev.1", features = [ "crypto" ], default-features = false } tracing = { version = "0.1.37" } From a4ad9c7239b5332ad02d1fbb5f81a372e2fde737 Mon Sep 17 00:00:00 2001 From: pauldelucia Date: Sun, 21 Jul 2024 13:38:07 +0900 Subject: [PATCH 12/12] fix: voting test --- .../tests/strategy_tests/voting_tests.rs | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/packages/rs-drive-abci/tests/strategy_tests/voting_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/voting_tests.rs index c111ac2fac..fe94b47fce 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/voting_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/voting_tests.rs @@ -121,7 +121,7 @@ mod tests { ), ]), Some(start_identities.first().unwrap().0.id()), - DocumentFieldFillType::FillIfNotRequired, + DocumentFieldFillType::DoNotFillIfNotRequired, DocumentFieldFillSize::AnyDocumentFillSize, ), document_type: document_type.clone(), @@ -144,7 +144,7 @@ mod tests { ), ]), Some(start_identities.last().unwrap().0.id()), - DocumentFieldFillType::FillIfNotRequired, + DocumentFieldFillType::DoNotFillIfNotRequired, DocumentFieldFillSize::AnyDocumentFillSize, ), document_type: document_type.clone(), @@ -294,35 +294,32 @@ mod tests { assert_eq!( first_contender.document, Some(vec![ - 0, 24, 85, 248, 135, 55, 81, 210, 5, 93, 112, 104, 77, 97, 177, 49, 255, 108, 242, - 0, 83, 232, 168, 214, 145, 55, 49, 246, 246, 126, 99, 17, 108, 41, 18, 75, 231, + 0, 23, 127, 36, 121, 9, 10, 2, 134, 166, 125, 106, 31, 103, 181, 99, 181, 21, 24, + 237, 214, 238, 160, 70, 24, 41, 247, 214, 48, 253, 101, 112, 141, 41, 18, 75, 231, 232, 111, 151, 233, 89, 137, 74, 103, 169, 204, 7, 140, 62, 1, 6, 212, 191, 207, 191, 52, 188, 64, 58, 79, 9, 153, 37, 180, 1, 0, 7, 0, 0, 1, 135, 105, 8, 149, 152, 0, 0, 1, 135, 105, 8, 149, 152, 0, 0, 1, 135, 105, 8, 149, 152, 0, 7, 113, 117, 97, - 110, 116, 117, 109, 7, 113, 117, 97, 110, 116, 117, 109, 1, 9, 112, 48, 81, 101, - 48, 107, 49, 65, 122, 4, 100, 97, 115, 104, 48, 165, 41, 91, 32, 215, 12, 4, 215, - 10, 9, 207, 71, 187, 248, 211, 105, 252, 147, 22, 127, 31, 203, 145, 6, 255, 132, - 220, 231, 96, 76, 195, 34, 1, 41, 18, 75, 231, 232, 111, 151, 233, 89, 137, 74, - 103, 169, 204, 7, 140, 62, 1, 6, 212, 191, 207, 191, 52, 188, 64, 58, 79, 9, 153, - 37, 180, 0, 1, 0 + 110, 116, 117, 109, 7, 113, 117, 97, 110, 116, 117, 109, 0, 4, 100, 97, 115, 104, + 97, 228, 155, 126, 30, 18, 134, 198, 154, 239, 219, 139, 51, 36, 53, 36, 212, 198, + 180, 139, 46, 87, 130, 184, 52, 174, 119, 176, 192, 51, 239, 245, 34, 1, 41, 18, + 75, 231, 232, 111, 151, 233, 89, 137, 74, 103, 169, 204, 7, 140, 62, 1, 6, 212, + 191, 207, 191, 52, 188, 64, 58, 79, 9, 153, 37, 180, 0, 1, 1 ]) ); assert_eq!( second_contender.document, Some(vec![ - 0, 23, 193, 35, 24, 227, 101, 215, 103, 217, 98, 152, 114, 80, 94, 3, 27, 65, 246, - 202, 212, 59, 205, 101, 140, 243, 61, 26, 152, 167, 199, 96, 133, 139, 137, 72, + 0, 73, 14, 33, 37, 147, 161, 211, 204, 106, 225, 123, 241, 7, 171, 156, 180, 101, + 23, 94, 120, 119, 252, 247, 208, 133, 237, 47, 206, 39, 190, 17, 214, 139, 137, 72, 166, 128, 21, 1, 187, 224, 67, 30, 61, 153, 77, 207, 113, 207, 90, 42, 9, 57, 254, 81, 176, 230, 0, 7, 97, 153, 171, 164, 251, 1, 0, 7, 0, 0, 1, 135, 105, 8, 149, 152, 0, 0, 1, 135, 105, 8, 149, 152, 0, 0, 1, 135, 105, 8, 149, 152, 0, 7, 113, - 117, 97, 110, 116, 117, 109, 7, 113, 117, 97, 110, 116, 117, 109, 1, 36, 65, 50, - 104, 52, 88, 69, 66, 112, 116, 74, 101, 99, 48, 101, 98, 87, 53, 67, 52, 89, 106, - 72, 119, 82, 81, 48, 51, 88, 54, 83, 99, 75, 103, 89, 111, 97, 4, 100, 97, 115, - 104, 110, 35, 254, 120, 68, 194, 240, 23, 122, 207, 220, 40, 135, 147, 185, 9, 126, - 239, 26, 0, 22, 196, 197, 243, 182, 218, 58, 240, 230, 102, 185, 157, 34, 1, 139, - 137, 72, 166, 128, 21, 1, 187, 224, 67, 30, 61, 153, 77, 207, 113, 207, 90, 42, 9, - 57, 254, 81, 176, 230, 0, 7, 97, 153, 171, 164, 251, 0, 1, 0 + 117, 97, 110, 116, 117, 109, 7, 113, 117, 97, 110, 116, 117, 109, 0, 4, 100, 97, + 115, 104, 148, 190, 52, 134, 51, 240, 205, 206, 235, 85, 72, 228, 129, 105, 99, + 173, 158, 47, 240, 100, 50, 213, 109, 127, 82, 26, 217, 2, 184, 229, 172, 129, 34, + 1, 139, 137, 72, 166, 128, 21, 1, 187, 224, 67, 30, 61, 153, 77, 207, 113, 207, 90, + 42, 9, 57, 254, 81, 176, 230, 0, 7, 97, 153, 171, 164, 251, 0, 1, 0 ]) );