diff --git a/Cargo.lock b/Cargo.lock index ef9a70d187..0a2a3a2659 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -468,7 +468,7 @@ dependencies = [ "bitflags 2.6.0", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.10.5", "lazy_static", "lazycell", "proc-macro2", @@ -3025,12 +3025,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "multimap" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" - [[package]] name = "murmur3" version = "0.5.2" @@ -3690,7 +3684,7 @@ dependencies = [ "itertools 0.10.5", "lazy_static", "log", - "multimap 0.8.3", + "multimap", "petgraph", "prettyplease 0.1.25", "prost 0.11.9", @@ -3708,10 +3702,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", - "heck 0.5.0", - "itertools 0.12.1", + "heck 0.4.1", + "itertools 0.10.5", "log", - "multimap 0.10.0", + "multimap", "once_cell", "petgraph", "prettyplease 0.2.20", @@ -3742,7 +3736,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.71", diff --git a/packages/rs-dpp/src/data_contract/document_type/random_document.rs b/packages/rs-dpp/src/data_contract/document_type/random_document.rs index 16a0327778..9f98397ef9 100644 --- a/packages/rs-dpp/src/data_contract/document_type/random_document.rs +++ b/packages/rs-dpp/src/data_contract/document_type/random_document.rs @@ -182,10 +182,10 @@ pub trait CreateRandomDocument { /// A `Result, ProtocolError>` which is `Ok` containing a vector of tuples /// if successful, each tuple consisting of a Document, its associated Identity, and a Bytes32 value, or an error /// if the operation fails. - fn random_documents_with_params( + fn random_documents_with_params<'i>( &self, count: u32, - identities: &[Identity], + identities: &[&'i Identity], time_ms: Option, block_height: Option, core_block_height: Option, @@ -193,7 +193,7 @@ pub trait CreateRandomDocument { document_field_fill_size: DocumentFieldFillSize, rng: &mut StdRng, platform_version: &PlatformVersion, - ) -> Result, ProtocolError>; + ) -> Result, ProtocolError>; } impl CreateRandomDocument for DocumentType { @@ -286,10 +286,10 @@ impl CreateRandomDocument for DocumentType { ), // Add more cases as necessary for other variants } } - fn random_documents_with_params( + fn random_documents_with_params<'i>( &self, count: u32, - identities: &[Identity], + identities: &[&'i Identity], time_ms: Option, block_height: Option, core_block_height: Option, @@ -297,7 +297,7 @@ impl CreateRandomDocument for DocumentType { document_field_fill_size: DocumentFieldFillSize, rng: &mut StdRng, platform_version: &PlatformVersion, - ) -> Result, ProtocolError> { + ) -> Result, ProtocolError> { match self { DocumentType::V0(v0) => v0.random_documents_with_params( count, @@ -405,10 +405,10 @@ impl<'a> CreateRandomDocument for DocumentTypeRef<'a> { } } - fn random_documents_with_params( + fn random_documents_with_params<'i>( &self, count: u32, - identities: &[Identity], + identities: &[&'i Identity], time_ms: Option, block_height: Option, core_block_height: Option, @@ -416,7 +416,7 @@ impl<'a> CreateRandomDocument for DocumentTypeRef<'a> { document_field_fill_size: DocumentFieldFillSize, rng: &mut StdRng, platform_version: &PlatformVersion, - ) -> Result, ProtocolError> { + ) -> Result, ProtocolError> { match self { DocumentTypeRef::V0(v0) => v0.random_documents_with_params( count, diff --git a/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs b/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs index 6cc7ea31d5..66f3733860 100644 --- a/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs +++ b/packages/rs-dpp/src/data_contract/document_type/v0/random_document.rs @@ -7,7 +7,7 @@ use platform_value::{Bytes32, Identifier}; use rand::rngs::StdRng; -use rand::{Rng, SeedableRng}; +use rand::SeedableRng; use std::time::{SystemTime, UNIX_EPOCH}; use crate::data_contract::document_type::methods::DocumentTypeV0Methods; @@ -266,10 +266,10 @@ impl CreateRandomDocument for DocumentTypeV0 { } /// Creates `count` Documents with random data using the random number generator given. - fn random_documents_with_params( + fn random_documents_with_params<'i>( &self, count: u32, - identities: &[Identity], + identities: &[&'i Identity], time_ms: Option, block_height: Option, core_block_height: Option, @@ -277,11 +277,17 @@ impl CreateRandomDocument for DocumentTypeV0 { document_field_fill_size: DocumentFieldFillSize, rng: &mut StdRng, platform_version: &PlatformVersion, - ) -> Result, ProtocolError> { + ) -> Result, ProtocolError> { let mut vec = vec![]; - for _i in 0..count { - let identity_num = rng.gen_range(0..identities.len()); - let identity = identities.get(identity_num).unwrap().clone(); + + if identities.len() < count as usize { + return Err(ProtocolError::CorruptedCodeExecution(format!( + "not enough identities to create {count} documents" + ))); + } + + for i in 0..count { + let identity = identities[i as usize]; let entropy = Bytes32::random_with_rng(rng); vec.push(( self.random_document_with_params( diff --git a/packages/rs-drive-abci/tests/strategy_tests/main.rs b/packages/rs-drive-abci/tests/strategy_tests/main.rs index 7c2ba078bd..321c7eac20 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/main.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/main.rs @@ -1915,7 +1915,7 @@ mod tests { .unwrap() .unwrap() ), - "6e4fb02e2619715b4a190ee206b560a8d34830a721d02d29bb5731783ba2b08a".to_string() + "cae9c984e5fe548cc7a3edd720ab0f58a2d25e5a16071402605a9ff8cfe870c7".to_string() ) } @@ -2050,7 +2050,7 @@ mod tests { .unwrap() .unwrap() ), - "e3cd37a5139f08af5ae5933c71ba7f3d1a922291bb625e78c245f115034f8408".to_string() + "6cbe10061aac832a35ec0eb9b22826aebfcbe697cec76fe899afa1b1803dd2c7".to_string() ) } @@ -2393,7 +2393,7 @@ mod tests { let outcome = run_chain_for_strategy(&mut platform, block_count, strategy, config, 15, &mut None); - assert_eq!(outcome.identities.len() as u64, 470); + assert_eq!(outcome.identities.len() as u64, 472); assert_eq!(outcome.masternode_identity_balances.len(), 100); let balance_count = outcome .masternode_identity_balances @@ -2528,7 +2528,7 @@ mod tests { let outcome = run_chain_for_strategy(&mut platform, block_count, strategy, config, 15, &mut None); - assert_eq!(outcome.identities.len() as u64, 90); + assert_eq!(outcome.identities.len() as u64, 83); assert_eq!(outcome.masternode_identity_balances.len(), 100); let balance_count = outcome .masternode_identity_balances @@ -2679,7 +2679,7 @@ mod tests { let outcome = run_chain_for_strategy(&mut platform, block_count, strategy, config, 15, &mut None); - assert_eq!(outcome.identities.len() as u64, 97); + assert_eq!(outcome.identities.len() as u64, 79); assert_eq!(outcome.masternode_identity_balances.len(), 100); let balance_count = outcome .masternode_identity_balances diff --git a/packages/rs-drive-abci/tests/strategy_tests/strategy.rs b/packages/rs-drive-abci/tests/strategy_tests/strategy.rs index e59179c62d..715031739b 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/strategy.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/strategy.rs @@ -564,17 +564,29 @@ impl NetworkStrategy { let mut deleted = vec![]; for op in &self.strategy.operations { if op.frequency.check_hit(rng) { - let count = rng.gen_range(op.frequency.times_per_block_range.clone()); + let mut count = rng.gen_range(op.frequency.times_per_block_range.clone()); match &op.op_type { OperationType::Document(DocumentOp { action: DocumentAction::DocumentActionInsertRandom(fill_type, fill_size), document_type, contract, }) => { + if current_identities.len() < count as usize { + count = current_identities.len() as u16; + + tracing::warn!( + "Not enough identities to insert documents, reducing count to {}", + count + ); + } + + let current_identities_as_refs: Vec<&dpp::identity::Identity> = + current_identities.iter().collect(); + let documents = document_type .random_documents_with_params( count as u32, - current_identities, + current_identities_as_refs.as_ref(), Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), @@ -673,15 +685,20 @@ impl NetworkStrategy { contract, }) => { let documents = if let Some(identifier) = identifier { - let held_identity = vec![current_identities + let held_identity = current_identities .iter() .find(|identity| identity.id() == identifier) - .expect("expected to find identifier, review strategy params") - .clone()]; + .expect("expected to find identifier, review strategy params"); + + let mut eligible_identities = Vec::with_capacity(count as usize); + for _ in 0..count { + eligible_identities.push(held_identity); + } + document_type .random_documents_with_params( count as u32, - &held_identity, + &eligible_identities, Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), @@ -692,10 +709,13 @@ impl NetworkStrategy { ) .expect("expected random_documents_with_params") } else { + let current_identities_as_refs: Vec<&dpp::identity::Identity> = + current_identities.iter().collect(); + document_type .random_documents_with_params( count as u32, - current_identities, + current_identities_as_refs.as_ref(), Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), diff --git a/packages/rs-drive-abci/tests/strategy_tests/voting_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/voting_tests.rs index c111ac2fac..fe94b47fce 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/voting_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/voting_tests.rs @@ -121,7 +121,7 @@ mod tests { ), ]), Some(start_identities.first().unwrap().0.id()), - DocumentFieldFillType::FillIfNotRequired, + DocumentFieldFillType::DoNotFillIfNotRequired, DocumentFieldFillSize::AnyDocumentFillSize, ), document_type: document_type.clone(), @@ -144,7 +144,7 @@ mod tests { ), ]), Some(start_identities.last().unwrap().0.id()), - DocumentFieldFillType::FillIfNotRequired, + DocumentFieldFillType::DoNotFillIfNotRequired, DocumentFieldFillSize::AnyDocumentFillSize, ), document_type: document_type.clone(), @@ -294,35 +294,32 @@ mod tests { assert_eq!( first_contender.document, Some(vec![ - 0, 24, 85, 248, 135, 55, 81, 210, 5, 93, 112, 104, 77, 97, 177, 49, 255, 108, 242, - 0, 83, 232, 168, 214, 145, 55, 49, 246, 246, 126, 99, 17, 108, 41, 18, 75, 231, + 0, 23, 127, 36, 121, 9, 10, 2, 134, 166, 125, 106, 31, 103, 181, 99, 181, 21, 24, + 237, 214, 238, 160, 70, 24, 41, 247, 214, 48, 253, 101, 112, 141, 41, 18, 75, 231, 232, 111, 151, 233, 89, 137, 74, 103, 169, 204, 7, 140, 62, 1, 6, 212, 191, 207, 191, 52, 188, 64, 58, 79, 9, 153, 37, 180, 1, 0, 7, 0, 0, 1, 135, 105, 8, 149, 152, 0, 0, 1, 135, 105, 8, 149, 152, 0, 0, 1, 135, 105, 8, 149, 152, 0, 7, 113, 117, 97, - 110, 116, 117, 109, 7, 113, 117, 97, 110, 116, 117, 109, 1, 9, 112, 48, 81, 101, - 48, 107, 49, 65, 122, 4, 100, 97, 115, 104, 48, 165, 41, 91, 32, 215, 12, 4, 215, - 10, 9, 207, 71, 187, 248, 211, 105, 252, 147, 22, 127, 31, 203, 145, 6, 255, 132, - 220, 231, 96, 76, 195, 34, 1, 41, 18, 75, 231, 232, 111, 151, 233, 89, 137, 74, - 103, 169, 204, 7, 140, 62, 1, 6, 212, 191, 207, 191, 52, 188, 64, 58, 79, 9, 153, - 37, 180, 0, 1, 0 + 110, 116, 117, 109, 7, 113, 117, 97, 110, 116, 117, 109, 0, 4, 100, 97, 115, 104, + 97, 228, 155, 126, 30, 18, 134, 198, 154, 239, 219, 139, 51, 36, 53, 36, 212, 198, + 180, 139, 46, 87, 130, 184, 52, 174, 119, 176, 192, 51, 239, 245, 34, 1, 41, 18, + 75, 231, 232, 111, 151, 233, 89, 137, 74, 103, 169, 204, 7, 140, 62, 1, 6, 212, + 191, 207, 191, 52, 188, 64, 58, 79, 9, 153, 37, 180, 0, 1, 1 ]) ); assert_eq!( second_contender.document, Some(vec![ - 0, 23, 193, 35, 24, 227, 101, 215, 103, 217, 98, 152, 114, 80, 94, 3, 27, 65, 246, - 202, 212, 59, 205, 101, 140, 243, 61, 26, 152, 167, 199, 96, 133, 139, 137, 72, + 0, 73, 14, 33, 37, 147, 161, 211, 204, 106, 225, 123, 241, 7, 171, 156, 180, 101, + 23, 94, 120, 119, 252, 247, 208, 133, 237, 47, 206, 39, 190, 17, 214, 139, 137, 72, 166, 128, 21, 1, 187, 224, 67, 30, 61, 153, 77, 207, 113, 207, 90, 42, 9, 57, 254, 81, 176, 230, 0, 7, 97, 153, 171, 164, 251, 1, 0, 7, 0, 0, 1, 135, 105, 8, 149, 152, 0, 0, 1, 135, 105, 8, 149, 152, 0, 0, 1, 135, 105, 8, 149, 152, 0, 7, 113, - 117, 97, 110, 116, 117, 109, 7, 113, 117, 97, 110, 116, 117, 109, 1, 36, 65, 50, - 104, 52, 88, 69, 66, 112, 116, 74, 101, 99, 48, 101, 98, 87, 53, 67, 52, 89, 106, - 72, 119, 82, 81, 48, 51, 88, 54, 83, 99, 75, 103, 89, 111, 97, 4, 100, 97, 115, - 104, 110, 35, 254, 120, 68, 194, 240, 23, 122, 207, 220, 40, 135, 147, 185, 9, 126, - 239, 26, 0, 22, 196, 197, 243, 182, 218, 58, 240, 230, 102, 185, 157, 34, 1, 139, - 137, 72, 166, 128, 21, 1, 187, 224, 67, 30, 61, 153, 77, 207, 113, 207, 90, 42, 9, - 57, 254, 81, 176, 230, 0, 7, 97, 153, 171, 164, 251, 0, 1, 0 + 117, 97, 110, 116, 117, 109, 7, 113, 117, 97, 110, 116, 117, 109, 0, 4, 100, 97, + 115, 104, 148, 190, 52, 134, 51, 240, 205, 206, 235, 85, 72, 228, 129, 105, 99, + 173, 158, 47, 240, 100, 50, 213, 109, 127, 82, 26, 217, 2, 184, 229, 172, 129, 34, + 1, 139, 137, 72, 166, 128, 21, 1, 187, 224, 67, 30, 61, 153, 77, 207, 113, 207, 90, + 42, 9, 57, 254, 81, 176, 230, 0, 7, 97, 153, 171, 164, 251, 0, 1, 0 ]) ); diff --git a/packages/strategy-tests/src/lib.rs b/packages/strategy-tests/src/lib.rs index 168509ca05..bb5415e12b 100644 --- a/packages/strategy-tests/src/lib.rs +++ b/packages/strategy-tests/src/lib.rs @@ -42,7 +42,7 @@ use dpp::state_transition::data_contract_update_transition::methods::DataContrac use operations::{DataContractUpdateAction, DataContractUpdateOp}; use platform_version::TryFromPlatformVersioned; use rand::prelude::StdRng; -use rand::seq::SliceRandom; +use rand::seq::{IteratorRandom, SliceRandom}; use rand::Rng; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use bincode::{Decode, Encode}; @@ -50,7 +50,7 @@ use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; use dpp::identifier::Identifier; use dpp::data_contract::document_type::DocumentType; use dpp::identity::accessors::IdentityGettersV0; -use dpp::platform_value::{BinaryData, Value}; +use dpp::platform_value::{BinaryData, Bytes32, Value}; use dpp::ProtocolError; use dpp::ProtocolError::{PlatformDeserializationError, PlatformSerializationError}; use dpp::state_transition::documents_batch_transition::document_base_transition::v0::DocumentBaseTransitionV0; @@ -324,6 +324,71 @@ impl PlatformDeserializableWithPotentialValidationFromVersionedStructure for Str } } +pub type MempoolDocumentCounter<'c> = BTreeMap<(Identifier, Identifier), u64>; + +fn choose_capable_identities<'i>( + identities: &'i [Identity], + contract: &DataContract, + mempool_document_counter: &MempoolDocumentCounter, + count: usize, + rng: &mut StdRng, +) -> Vec<&'i Identity> { + // Filter out those that already have 24 documents in the mempool for this contract + let mut all_capable_identities: Vec<&'i Identity> = identities + .iter() + .filter(|identity| { + mempool_document_counter + .get(&(identity.id(), contract.id())) + .unwrap_or(&0) + < &24u64 + }) + .collect(); + + // If we need more documents than we have eligible identities, we will need to use some identities more than once + if all_capable_identities.len() < count { + let mut additional_identities_count = count - all_capable_identities.len(); + + let mut additional_identities = Vec::with_capacity(additional_identities_count); + let mut index = 0; + while additional_identities_count > 0 && index < all_capable_identities.len() { + // Iterate through the identities only one time, using as many documents as each identity can submit + // We should see if this often causes identities to max out their mempool count, which could potentially + // lead to issues, even though this function should really only be covering an edge case + let identity = all_capable_identities[index]; + + // How many documents are in the mempool for this (identity, contract) plus the one that we are already planning to use + let mempool_documents_count = mempool_document_counter + .get(&(identity.id(), contract.id())) + .unwrap_or(&0) + + 1; + + let identity_capacity = 24 - mempool_documents_count; + + if identity_capacity > 0 { + for _ in 0..identity_capacity { + additional_identities.push(identity); + additional_identities_count -= 1; + if additional_identities_count == 0 { + break; + } + } + } + + index += 1; + } + + all_capable_identities.extend(additional_identities); + all_capable_identities + } else { + all_capable_identities + .iter() + .choose_multiple(rng, count) + .into_iter() + .cloned() + .collect() + } +} + impl Strategy { /// Generates comprehensive state transitions for a given block, including handling new identities and contracts. /// @@ -381,7 +446,7 @@ impl Strategy { /// This function is central to simulating the lifecycle of block processing and strategy execution /// on the Dash Platform. It encapsulates the complexity of transaction generation, identity management, /// and contract dynamics within a block's context. - pub async fn state_transitions_for_block( + pub fn state_transitions_for_block( &mut self, document_query_callback: &mut impl FnMut(LocalDocumentQuery) -> Vec, identity_fetch_callback: &mut impl FnMut( @@ -395,7 +460,7 @@ impl Strategy { signer: &mut SimpleSigner, identity_nonce_counter: &mut BTreeMap, contract_nonce_counter: &mut BTreeMap<(Identifier, Identifier), u64>, - mempool_document_counter: BTreeMap<(Identifier, Identifier), u64>, + mempool_document_counter: &MempoolDocumentCounter, rng: &mut StdRng, config: &StrategyConfig, platform_version: &PlatformVersion, @@ -544,7 +609,7 @@ impl Strategy { signer: &mut SimpleSigner, identity_nonce_counter: &mut BTreeMap, contract_nonce_counter: &mut BTreeMap<(Identifier, Identifier), u64>, - mempool_document_counter: BTreeMap<(Identifier, Identifier), u64>, + mempool_document_counter: &MempoolDocumentCounter, rng: &mut StdRng, platform_version: &PlatformVersion, ) -> (Vec, Vec) { @@ -570,44 +635,37 @@ impl Strategy { document_type, contract, }) => { - // Get the first 10 identities who are eligible to submit documents for this contract - let first_10_eligible_identities: Vec = current_identities - .iter() - .filter(|identity| { - mempool_document_counter - .get(&(identity.id(), contract.id())) - .unwrap_or(&0) - < &24u64 - }) - .take(10) - .cloned() - .collect(); - - if first_10_eligible_identities.len() == 0 { - tracing::warn!( - "No eligible identities to submit a document to contract {}", - contract.id().to_string(Encoding::Base64) - ); - } + let eligible_identities = choose_capable_identities( + current_identities, + contract, + mempool_document_counter, + count as usize, + rng, + ); // TO-DO: these documents should be created according to the data contract's validation rules - let documents = document_type - .random_documents_with_params( - count as u32, - &first_10_eligible_identities, - Some(block_info.time_ms), - Some(block_info.height), - Some(block_info.core_height), - *fill_type, - *fill_size, - rng, - platform_version, - ) - .expect("expected random_documents_with_params"); + let mut documents_with_identity_and_entropy: Vec<( + Document, + &Identity, + Bytes32, + )> = Vec::new(); + match document_type.random_documents_with_params( + count as u32, + &eligible_identities, + Some(block_info.time_ms), + Some(block_info.height), + Some(block_info.core_height), + *fill_type, + *fill_size, + rng, + platform_version, + ) { + Ok(documents) => documents_with_identity_and_entropy = documents, + Err(e) => tracing::warn!("Failed to create random documents: {}", e), + } - documents - .into_iter() - .for_each(|(document, identity, entropy)| { + documents_with_identity_and_entropy.into_iter().for_each( + |(document, identity, entropy)| { let identity_contract_nonce = if contract.owner_id() == identity.id() { contract_nonce_counter @@ -675,7 +733,8 @@ impl Strategy { .expect("expected to sign"); operations.push(document_batch_transition); - }); + }, + ); } // Generate state transition for specific document insert operation @@ -691,15 +750,20 @@ impl Strategy { contract, }) => { let documents = if let Some(identifier) = identifier { - let held_identity = vec![current_identities + let held_identity = current_identities .iter() .find(|identity| identity.id() == identifier) - .expect("expected to find identifier, review strategy params") - .clone()]; + .expect("expected to find identifier, review strategy params"); + + let mut eligible_identities = Vec::with_capacity(count as usize); + for _ in 0..count { + eligible_identities.push(held_identity); + } + document_type .random_documents_with_params( count as u32, - &held_identity, + &eligible_identities, Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height), @@ -710,10 +774,18 @@ impl Strategy { ) .expect("expected random_documents_with_params") } else { + let eligible_identities = choose_capable_identities( + current_identities, + contract, + mempool_document_counter, + count as usize, + rng, + ); + document_type .random_documents_with_params( count as u32, - current_identities, + &eligible_identities, Some(block_info.time_ms), Some(block_info.height), Some(block_info.core_height),