From 793142e5c2e97253fe21e1e80217b08e077857bd Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 17 Jul 2024 15:19:14 +0300 Subject: [PATCH 001/103] WIP primitives Signed-off-by: Andrei Sandu --- polkadot/primitives/src/v7/mod.rs | 2 +- polkadot/primitives/src/vstaging/mod.rs | 215 +++++++++++++++++++++++- 2 files changed, 215 insertions(+), 2 deletions(-) diff --git a/polkadot/primitives/src/v7/mod.rs b/polkadot/primitives/src/v7/mod.rs index a729d8cee4c7..18f52951986a 100644 --- a/polkadot/primitives/src/v7/mod.rs +++ b/polkadot/primitives/src/v7/mod.rs @@ -2052,7 +2052,7 @@ pub mod node_features { } #[cfg(test)] -mod tests { +pub mod tests { use super::*; use bitvec::bitvec; use sp_core::sr25519; diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 27296213e611..ff138781153f 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -17,7 +17,11 @@ //! Staging Primitives. // Put any primitives used by staging APIs functions here -use crate::v7::*; +use super::{ + Balance, CoreIndex, CandidateCommitments, Hash, Id, ValidationCode, ValidationCodeHash, + ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, +}; +use sp_std::prelude::*; use codec::{Decode, Encode}; use scale_info::TypeInfo; @@ -98,3 +102,212 @@ impl> Default for SchedulerParams } } } + +/// A unique descriptor of the candidate receipt. +#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(Hash))] +pub struct CandidateDescriptor { + /// The ID of the para this is a candidate for. + pub para_id: Id, + /// The hash of the relay-chain block this is executed in the context of. + pub relay_parent: H, + /// The core index where the candidate is backed. + pub core_index: CoreIndex, + /// Reserved bytes. + pub reserved28b: [u8; 27], + /// The blake2-256 hash of the persisted validation data. This is extra data derived from + /// relay-chain state which may vary based on bitfields included before the candidate. + /// Thus it cannot be derived entirely from the relay-parent. + pub persisted_validation_data_hash: Hash, + /// The blake2-256 hash of the PoV. + pub pov_hash: Hash, + /// The root of a block's erasure encoding Merkle tree. + pub erasure_root: Hash, + /// Reserved bytes. + pub reserved64b: [u8; 64], + /// Hash of the para header that is being generated by this candidate. + pub para_head: Hash, + /// The blake2-256 hash of the validation code bytes. + pub validation_code_hash: ValidationCodeHash, +} + +#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +pub enum ExtendedCommitment { + #[codec(index = 0)] + CoreIndex(CoreIndex), +} + +pub const UMP_SEPARATOR: Vec = vec![]; + + +// /// Commitments made in a `CandidateReceipt`. Many of these are outputs of validation. +// #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +// #[cfg_attr(feature = "std", derive(Default, Hash))] +// pub struct CandidateCommitments { +// /// Messages destined to be interpreted by the Relay chain itself. +// pub upward_messages: UpwardMessages, +// /// Horizontal messages sent by the parachain. +// pub horizontal_messages: HorizontalMessages, +// /// New validation code. +// pub new_validation_code: Option, +// /// The head-data produced as a result of execution. +// pub head_data: HeadData, +// /// The number of messages processed from the DMQ. +// pub processed_downward_messages: u32, +// /// The mark which specifies the block number up to which all inbound HRMP messages are +// /// processed. +// pub hrmp_watermark: N, +// } + +/// A candidate-receipt with commitments directly included. +#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(Hash))] +pub struct CommittedCandidateReceipt { + /// The descriptor of the candidate. + descriptor: CandidateDescriptor, + /// The commitments of the candidate receipt. + commitments: CandidateCommitments, +} + +impl CandidateCommitments { + /// Returns the core index the candidate has commited to. + pub fn core_index(&self) -> Option { + /// We need at least 2 messages for the separator and core index + if self.upward_messages.len() < 2 { + return None + } + + let upward_commitments = self + .upward_messages + .iter() + .cloned() + .rev() + .take_while(|message| message != &UMP_SEPARATOR) + .collect::>(); + + // We didn't find the separator, no core index commitment. + if upward_commitments.len() == self.upward_messages.len() || + upward_commitments.is_empty() + { + return None + } + + // Use first commitment + let Some(message) = upward_commitments.into_iter().rev().next() else { return None }; + + match Commitment::decode(&mut message.as_slice()).ok()? { + Commitment::CoreIndex(core_index) => Some(core_index), + } + } +} + +#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +pub enum CandidateReceiptError { + /// The specified core index is invalid. + InvalidCoreIndex, + /// The core index in commitments doesnt match the one in descriptor + CoreIndexMismatch, +} + +impl CommittedCandidateReceipt { + /// Constructor from descriptor and commitments after sanity checking core index commitments. + pub fn new(descriptor: CandidateDescriptor, commitments: CandidateCommitments, n_cores: u32) -> Result { + // First check if we have a core index commitment + if commitments.core_index() != descriptor.core_index { + return Err(CandidateReceiptError::CoreIndexMismatch) + } + + match descriptor.core_index { + Some(core_index) => { + if core_index.0 > n_cores - 1 { + return Err(CandidateReceiptError::InvalidCoreIndex) + } + + Ok(Self { descriptor, commitments }) + }, + None => Ok(Self { descriptor, commitments }) + } + } + + /// Returns are reference to commitments + pub fn commitments(&self) -> &CandidateCommitments { + &self.commitments + } + + /// Returns a reference to the descriptor + pub fn descriptor(&self) -> &CandidateDescriptor { + &self.descriptor + } + + +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + v7::{ + tests::dummy_committed_candidate_receipt as dummy_old_committed_candidate_receipt, + CandidateReceipt as OldCandidateReceipt, Hash, HeadData, + }, + vstaging::CommittedCandidateReceipt, + }; + + pub fn dummy_committed_candidate_receipt() -> CommittedCandidateReceipt { + let zeros = Hash::zero(); + let reserved64b = [0; 64]; + + CommittedCandidateReceipt { + descriptor: CandidateDescriptor { + para_id: 0.into(), + relay_parent: zeros, + core_index: Some(CoreIndex(123)), + reserved28b: Default::default(), + persisted_validation_data_hash: zeros, + pov_hash: zeros, + erasure_root: zeros, + reserved64b, + para_head: zeros, + validation_code_hash: ValidationCode(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]).hash(), + }, + commitments: CandidateCommitments { + head_data: HeadData(vec![]), + upward_messages: vec![].try_into().expect("empty vec fits within bounds"), + new_validation_code: None, + horizontal_messages: vec![].try_into().expect("empty vec fits within bounds"), + processed_downward_messages: 0, + hrmp_watermark: 0_u32, + }, + } + } + + #[test] + fn is_binary_compatibile() { + let mut old_ccr = dummy_old_committed_candidate_receipt(); + let mut new_ccr = dummy_committed_candidate_receipt(); + + assert_eq!(old_ccr.encoded_size(), new_ccr.encoded_size()); + assert_eq!(new_ccr.commitments().core_index(), None); + } + + #[test] + fn test_ump_commitment() { + let old_ccr = dummy_old_committed_candidate_receipt(); + let mut new_ccr = dummy_committed_candidate_receipt(); + + // XCM messages + new_ccr.commitments.upward_messages.force_push(vec![0u8; 256]); + new_ccr.commitments.upward_messages.force_push(vec![0xff; 256]); + + // separator + new_ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); + + // CoreIndex commitment + new_ccr + .commitments + .upward_messages + .force_push(Commitment::CoreIndex(CoreIndex(123)).encode()); + + assert_eq!(new_ccr.descriptor.core_index, Some(CoreIndex(123))); + } +} From 3a29fdf3bf05f0558abccfb1a3f6be8c42473918 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 5 Aug 2024 19:43:14 +0300 Subject: [PATCH 002/103] WIP Signed-off-by: Andrei Sandu --- Cargo.lock | 2 + polkadot/primitives/Cargo.toml | 4 + polkadot/primitives/src/vstaging/mod.rs | 489 +++++++++++++++----- polkadot/primitives/test-helpers/src/lib.rs | 69 ++- 4 files changed, 459 insertions(+), 105 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ebacc9ec5a8..8d0aa320c83c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13982,6 +13982,7 @@ dependencies = [ "parity-scale-codec", "polkadot-core-primitives", "polkadot-parachain-primitives", + "polkadot-primitives-test-helpers", "scale-info", "serde", "sp-api", @@ -13995,6 +13996,7 @@ dependencies = [ "sp-keystore", "sp-runtime", "sp-staking", + "sp-std 14.0.0", ] [[package]] diff --git a/polkadot/primitives/Cargo.toml b/polkadot/primitives/Cargo.toml index 8f7ec314ecff..09afc9662d62 100644 --- a/polkadot/primitives/Cargo.toml +++ b/polkadot/primitives/Cargo.toml @@ -28,10 +28,14 @@ sp-consensus-slots = { features = ["serde"], workspace = true } sp-io = { workspace = true } sp-keystore = { optional = true, workspace = true } sp-staking = { features = ["serde"], workspace = true } +sp-std = { workspace = true, optional = true } polkadot-core-primitives = { workspace = true } polkadot-parachain-primitives = { workspace = true } +[dev-dependencies] +polkadot-primitives-test-helpers = { workspace = true } + [features] default = ["std"] std = [ diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index ff138781153f..b259ba022772 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -18,15 +18,18 @@ // Put any primitives used by staging APIs functions here use super::{ - Balance, CoreIndex, CandidateCommitments, Hash, Id, ValidationCode, ValidationCodeHash, + Balance, BlakeTwo256, BlockNumber, CandidateCommitments, CandidateDescriptor, CandidateHash, + CoreIndex, Hash, HashT, Id, Id as ParaId, ValidationCode, ValidationCodeHash, ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, }; -use sp_std::prelude::*; +use alloc::collections::{btree_map::BTreeMap, vec_deque::VecDeque}; use codec::{Decode, Encode}; use scale_info::TypeInfo; +use sp_api::Core; use sp_arithmetic::Perbill; use sp_core::RuntimeDebug; +use sp_staking::SessionIndex; /// Scheduler configuration parameters. All coretime/ondemand parameters are here. #[derive( @@ -106,73 +109,312 @@ impl> Default for SchedulerParams /// A unique descriptor of the candidate receipt. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Hash))] -pub struct CandidateDescriptor { +pub struct CandidateDescriptorV2 { /// The ID of the para this is a candidate for. - pub para_id: Id, + para_id: ParaId, /// The hash of the relay-chain block this is executed in the context of. - pub relay_parent: H, + relay_parent: H, /// The core index where the candidate is backed. - pub core_index: CoreIndex, + core_index: u16, + /// The session index of the candidate relay parent. + session_index: SessionIndex, /// Reserved bytes. - pub reserved28b: [u8; 27], + reserved28b: [u8; 26], /// The blake2-256 hash of the persisted validation data. This is extra data derived from /// relay-chain state which may vary based on bitfields included before the candidate. /// Thus it cannot be derived entirely from the relay-parent. - pub persisted_validation_data_hash: Hash, + persisted_validation_data_hash: Hash, /// The blake2-256 hash of the PoV. - pub pov_hash: Hash, + pov_hash: Hash, /// The root of a block's erasure encoding Merkle tree. - pub erasure_root: Hash, + erasure_root: Hash, /// Reserved bytes. - pub reserved64b: [u8; 64], + reserved64b: [u8; 64], /// Hash of the para header that is being generated by this candidate. - pub para_head: Hash, + para_head: Hash, /// The blake2-256 hash of the validation code bytes. - pub validation_code_hash: ValidationCodeHash, + validation_code_hash: ValidationCodeHash, } -#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] -pub enum ExtendedCommitment { - #[codec(index = 0)] - CoreIndex(CoreIndex), +impl CandidateDescriptorV2 { + /// Constructor + pub fn new( + para_id: Id, + relay_parent: H, + core_index: CoreIndex, + session_index: SessionIndex, + persisted_validation_data_hash: Hash, + pov_hash: Hash, + erasure_root: Hash, + para_head: Hash, + validation_code_hash: ValidationCodeHash, + ) -> Self { + Self { + para_id, + relay_parent, + core_index: core_index.0 as u16, + session_index, + reserved28b: [0; 26], + persisted_validation_data_hash, + pov_hash, + erasure_root, + reserved64b: [0; 64], + para_head, + validation_code_hash, + } + } } +/// Version 1 API to access information stored by candidate descriptors. +pub trait CandidateApiV1 { + /// Returns the ID of the para this is a candidate for. + fn para_id(&self) -> Id; -pub const UMP_SEPARATOR: Vec = vec![]; + /// Returns the blake2-256 hash of the persisted validation data. + fn persisted_validation_data_hash(&self) -> Hash; + + /// Returns the blake2-256 hash of the PoV. + fn pov_hash(&self) -> Hash; + + /// Returns the root of a block's erasure encoding Merkle tree. + fn erasure_root(&self) -> Hash; + + /// Returns the hash of the para header generated by this candidate. + fn para_head(&self) -> Hash; + + /// Return the blake2-256 hash of the validation code bytes. + fn validation_code_hash(&self) -> ValidationCodeHash; +} + +/// Version 2 API to access additional information stored by candidate descriptors +pub trait CandidateApiV2 { + /// Returns the core index where the candidate is backed. + fn core_index(&self) -> CoreIndex; + + /// Returns the session index of the candidate relay parent. + fn session_index(&self) -> SessionIndex; +} + +impl CandidateApiV1 for CandidateDescriptor { + fn para_id(&self) -> Id { + self.para_id + } + + fn persisted_validation_data_hash(&self) -> Hash { + self.persisted_validation_data_hash + } + + fn pov_hash(&self) -> Hash { + self.pov_hash + } + + fn erasure_root(&self) -> Hash { + self.erasure_root + } + + fn para_head(&self) -> Hash { + self.para_head + } + + fn validation_code_hash(&self) -> ValidationCodeHash { + self.validation_code_hash + } +} + +impl CandidateApiV1 for CandidateDescriptorV2 { + fn para_id(&self) -> Id { + self.para_id + } + + fn persisted_validation_data_hash(&self) -> Hash { + self.persisted_validation_data_hash + } + fn pov_hash(&self) -> Hash { + self.pov_hash + } + + fn erasure_root(&self) -> Hash { + self.erasure_root + } + + fn para_head(&self) -> Hash { + self.para_head + } + + fn validation_code_hash(&self) -> ValidationCodeHash { + self.validation_code_hash + } +} -// /// Commitments made in a `CandidateReceipt`. Many of these are outputs of validation. -// #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] -// #[cfg_attr(feature = "std", derive(Default, Hash))] -// pub struct CandidateCommitments { -// /// Messages destined to be interpreted by the Relay chain itself. -// pub upward_messages: UpwardMessages, -// /// Horizontal messages sent by the parachain. -// pub horizontal_messages: HorizontalMessages, -// /// New validation code. -// pub new_validation_code: Option, -// /// The head-data produced as a result of execution. -// pub head_data: HeadData, -// /// The number of messages processed from the DMQ. -// pub processed_downward_messages: u32, -// /// The mark which specifies the block number up to which all inbound HRMP messages are -// /// processed. -// pub hrmp_watermark: N, -// } +impl CandidateApiV2 for CandidateDescriptorV2 { + fn core_index(&self) -> CoreIndex { + CoreIndex(self.core_index as u32) + } + + fn session_index(&self) -> SessionIndex { + self.session_index + } +} + +/// A candidate-receipt at version 2. +#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +pub struct CandidateReceiptV2 { + /// The descriptor of the candidate. + pub descriptor: CandidateDescriptorV2, + /// The hash of the encoded commitments made as a result of candidate execution. + pub commitments_hash: Hash, +} /// A candidate-receipt with commitments directly included. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Hash))] -pub struct CommittedCandidateReceipt { +pub struct CommittedCandidateReceiptV2 { /// The descriptor of the candidate. - descriptor: CandidateDescriptor, + pub descriptor: CandidateDescriptorV2, /// The commitments of the candidate receipt. - commitments: CandidateCommitments, + pub commitments: CandidateCommitments, } -impl CandidateCommitments { - /// Returns the core index the candidate has commited to. +impl CandidateReceiptV2 { + /// Get a reference to the candidate descriptor. + pub fn descriptor(&self) -> &CandidateDescriptorV2 { + &self.descriptor + } + + /// Computes the blake2-256 hash of the receipt. + pub fn hash(&self) -> CandidateHash + where + H: Encode, + { + CandidateHash(BlakeTwo256::hash_of(self)) + } +} + +impl CommittedCandidateReceiptV2 { + /// Transforms this into a plain `CandidateReceipt`. + pub fn to_plain(&self) -> CandidateReceiptV2 { + CandidateReceiptV2 { + descriptor: self.descriptor.clone(), + commitments_hash: self.commitments.hash(), + } + } + + /// Computes the hash of the committed candidate receipt. + /// + /// This computes the canonical hash, not the hash of the directly encoded data. + /// Thus this is a shortcut for `candidate.to_plain().hash()`. + pub fn hash(&self) -> CandidateHash + where + H: Encode, + { + self.to_plain().hash() + } + + /// Does this committed candidate receipt corresponds to the given [`CandidateReceipt`]? + pub fn corresponds_to(&self, receipt: &CandidateReceiptV2) -> bool + where + H: PartialEq, + { + receipt.descriptor == self.descriptor && receipt.commitments_hash == self.commitments.hash() + } +} + +impl PartialOrd for CommittedCandidateReceiptV2 { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for CommittedCandidateReceiptV2 { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.descriptor() + .para_id + .cmp(&other.descriptor().para_id) + .then_with(|| self.commitments.head_data.cmp(&other.commitments.head_data)) + } +} + +/// A strictly increasing sequence number, tipically this would be the parachain block number. +#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +pub struct CoreSelector(pub BlockNumber); + +/// An offset in the relay chain claim queue. +#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +pub struct ClaimQueueOffset(pub u8); + +/// Default claim queue offset +pub const DEFAULT_CLAIM_QUEUE_OFFSET: ClaimQueueOffset = ClaimQueueOffset(1); + +#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +pub enum UMPSignal { + /// A message sent by a parachain to select the core the candidate is commited to. + /// Relay chain validators, in particular backers, use the `CoreSelector` and + /// `ClaimQueueOffset` to compute the index of the core the candidate has commited to. + SelectCore(CoreSelector, ClaimQueueOffset), +} +/// Separator between `XCM` and `UMPSignal`. +pub const UMP_SEPARATOR: Vec = vec![]; + +/// A versioned unique descriptor of the candidate receipt. +#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +pub enum VersionedCandidateReceipt { + /// Version 1 of candidate receipt. + V1(super::CandidateReceipt), + /// Version 2 of candidate receipts with `core_index` and `session_index`. + V2(CandidateReceiptV2), +} + +/// A versioned unique descriptor of the candidate receipt. +#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +pub enum VersionedCommittedCandidateReceipt { + V1(super::CommittedCandidateReceipt), + V2(CommittedCandidateReceiptV2), +} + +impl VersionedCandidateReceipt { + /// Returns the core index the candidate has commited to. Returns `None`` if + /// the receipt is version 1. + pub fn core_index(&self) -> Option { + match self { + Self::V1(_receipt_v1) => None, + Self::V2(receipt_v2) => Some(receipt_v2.descriptor.core_index()), + } + } + + /// Returns the session index of the relay parent. Returns `None`` if + /// the receipt is version 1. + pub fn session_index(&self) -> Option { + match self { + Self::V1(_receipt_v1) => None, + Self::V2(receipt_v2) => Some(receipt_v2.descriptor.session_index()), + } + } +} + +impl VersionedCommittedCandidateReceipt { + /// Returns the core index the candidate has commited to. Returns `None`` if + /// the receipt is version 1. pub fn core_index(&self) -> Option { - /// We need at least 2 messages for the separator and core index + match self { + Self::V1(_receipt_v1) => None, + Self::V2(receipt_v2) => Some(receipt_v2.descriptor.core_index()), + } + } + + /// Returns the session index of the relay parent. Returns `None` if + /// the receipt is version 1. + pub fn session_index(&self) -> Option { + match self { + Self::V1(_receipt_v1) => None, + Self::V2(receipt_v2) => Some(receipt_v2.descriptor.session_index()), + } + } +} + +impl CandidateCommitments { + /// Returns the core selector and claim queue offset the candidate has commited to, if any. + pub fn selected_core(&self) -> Option<(CoreSelector, ClaimQueueOffset)> { + // We need at least 2 messages for the separator and core index if self.upward_messages.len() < 2 { return None } @@ -186,60 +428,98 @@ impl CandidateCommitments { .collect::>(); // We didn't find the separator, no core index commitment. - if upward_commitments.len() == self.upward_messages.len() || - upward_commitments.is_empty() - { + if upward_commitments.len() == self.upward_messages.len() || upward_commitments.is_empty() { return None } // Use first commitment let Some(message) = upward_commitments.into_iter().rev().next() else { return None }; - match Commitment::decode(&mut message.as_slice()).ok()? { - Commitment::CoreIndex(core_index) => Some(core_index), + match UMPSignal::decode(&mut message.as_slice()).ok()? { + UMPSignal::SelectCore(core_selector, cq_offset) => Some((core_selector, cq_offset)), } } } +/// CandidateReceipt construction errors. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] pub enum CandidateReceiptError { /// The specified core index is invalid. InvalidCoreIndex, /// The core index in commitments doesnt match the one in descriptor CoreIndexMismatch, + /// The core selector or claim queue offset is invalid. + InvalidSelectedCore, } -impl CommittedCandidateReceipt { - /// Constructor from descriptor and commitments after sanity checking core index commitments. - pub fn new(descriptor: CandidateDescriptor, commitments: CandidateCommitments, n_cores: u32) -> Result { - // First check if we have a core index commitment - if commitments.core_index() != descriptor.core_index { - return Err(CandidateReceiptError::CoreIndexMismatch) - } - - match descriptor.core_index { - Some(core_index) => { - if core_index.0 > n_cores - 1 { - return Err(CandidateReceiptError::InvalidCoreIndex) - } - - Ok(Self { descriptor, commitments }) - }, - None => Ok(Self { descriptor, commitments }) - } +impl CommittedCandidateReceiptV2 { + /// Returns a v2 commited candidate receipt if the committed selected core + /// matches the core index in the descriptor. + pub fn new(descriptor: CandidateDescriptorV2, commitments: CandidateCommitments) -> Self { + Self { descriptor, commitments } } - /// Returns are reference to commitments + /// Returns a reference to commitments pub fn commitments(&self) -> &CandidateCommitments { &self.commitments } /// Returns a reference to the descriptor - pub fn descriptor(&self) -> &CandidateDescriptor { + pub fn descriptor(&self) -> &CandidateDescriptorV2 { &self.descriptor } - + /// Performs a sanity check of the receipt. + /// + /// Returns error if: + /// - descriptor core index is different than the core selected + /// by the commitments + /// - the core index is out of bounds wrt `n_cores`. + pub fn check( + &self, + n_cores: u32, + claimed_cores: Vec, + // TODO: consider promoting `ClaimQueueSnapshot` as primitive + claim_queue: BTreeMap>, + ) -> Result<(), CandidateReceiptError> { + let descriptor_core_index = CoreIndex(descriptor.core_index as u32); + let (core_selector, cq_offset) = self + .commitments + .selected_core() + .ok_or(Err(CandidateReceiptError::InvalidSelectedCore))?; + let para_id = self.descriptor.para_id; + + // TODO: bail out early if cq_offset > depth of claim queue. + + // Get a vec of the core indices the parachain is assigned to at `cq_offset`. + let assigned_cores = claim_queue + .iter() + .filter_map(|(core_index, queue)| { + let queued_para = queue.get(cq_offset)?; + + if queued_para == para_id { + Some(core_index) + } else { + None + } + }) + .cloned() + .collect::>(); + + let core_index = *assigned_cores + .get(core_selector.0 as usize % assigned_cores.len()) + .expect("provided index is always less than queue len; qed"); + + if core_index != Some(descriptor_core_index) { + return Err(CandidateReceiptError::CoreIndexMismatch) + } + + if descriptor_core_index.0 > n_cores - 1 { + return Err(CandidateReceiptError::InvalidCoreIndex) + } + + Ok(()) + } } #[cfg(test)] @@ -248,43 +528,46 @@ mod tests { use crate::{ v7::{ tests::dummy_committed_candidate_receipt as dummy_old_committed_candidate_receipt, - CandidateReceipt as OldCandidateReceipt, Hash, HeadData, + CandidateDescriptor, CandidateReceipt as OldCandidateReceipt, + CommittedCandidateReceipt, Hash, HeadData, }, - vstaging::CommittedCandidateReceipt, + vstaging::CommittedCandidateReceiptV2, }; - - pub fn dummy_committed_candidate_receipt() -> CommittedCandidateReceipt { - let zeros = Hash::zero(); - let reserved64b = [0; 64]; - - CommittedCandidateReceipt { - descriptor: CandidateDescriptor { - para_id: 0.into(), - relay_parent: zeros, - core_index: Some(CoreIndex(123)), - reserved28b: Default::default(), - persisted_validation_data_hash: zeros, - pov_hash: zeros, - erasure_root: zeros, - reserved64b, - para_head: zeros, - validation_code_hash: ValidationCode(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]).hash(), - }, - commitments: CandidateCommitments { - head_data: HeadData(vec![]), - upward_messages: vec![].try_into().expect("empty vec fits within bounds"), - new_validation_code: None, - horizontal_messages: vec![].try_into().expect("empty vec fits within bounds"), - processed_downward_messages: 0, - hrmp_watermark: 0_u32, - }, - } - } + use polkadot_primitives_test_helpers::dummy_committed_candidate_receipt_v2; + + // pub fn dummy_committed_candidate_receipt() -> CommittedCandidateReceiptV2 { + // let zeros = Hash::zero(); + // let reserved64b = [0; 64]; + + // CommittedCandidateReceiptV2 { + // descriptor: CandidateDescriptorV2 { + // para_id: 0.into(), + // relay_parent: zeros, + // core_index: 123, + // session_index: 1, + // reserved28b: Default::default(), + // persisted_validation_data_hash: zeros, + // pov_hash: zeros, + // erasure_root: zeros, + // reserved64b, + // para_head: zeros, + // validation_code_hash: ValidationCode(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]).hash(), + // }, + // commitments: CandidateCommitments { + // head_data: HeadData(vec![]), + // upward_messages: vec![].try_into().expect("empty vec fits within bounds"), + // new_validation_code: None, + // horizontal_messages: vec![].try_into().expect("empty vec fits within bounds"), + // processed_downward_messages: 0, + // hrmp_watermark: 0_u32, + // }, + // } + // } #[test] fn is_binary_compatibile() { let mut old_ccr = dummy_old_committed_candidate_receipt(); - let mut new_ccr = dummy_committed_candidate_receipt(); + let mut new_ccr = dummy_committed_candidate_receipt_v2(Hash::zero()); assert_eq!(old_ccr.encoded_size(), new_ccr.encoded_size()); assert_eq!(new_ccr.commitments().core_index(), None); @@ -292,8 +575,7 @@ mod tests { #[test] fn test_ump_commitment() { - let old_ccr = dummy_old_committed_candidate_receipt(); - let mut new_ccr = dummy_committed_candidate_receipt(); + let mut new_ccr = dummy_committed_candidate_receipt_v2(Hash::zero()); // XCM messages new_ccr.commitments.upward_messages.force_push(vec![0u8; 256]); @@ -306,8 +588,9 @@ mod tests { new_ccr .commitments .upward_messages - .force_push(Commitment::CoreIndex(CoreIndex(123)).encode()); + .force_push(UMPSignal::OnCore(CoreIndex(123)).encode()); - assert_eq!(new_ccr.descriptor.core_index, Some(CoreIndex(123))); + let new_ccr = VersionedCommittedCandidateReceipt::V2(new_ccr); + assert_eq!(new_ccr.core_index(), Some(CoreIndex(123))); } } diff --git a/polkadot/primitives/test-helpers/src/lib.rs b/polkadot/primitives/test-helpers/src/lib.rs index d43cf3317e57..c06c0420fd70 100644 --- a/polkadot/primitives/test-helpers/src/lib.rs +++ b/polkadot/primitives/test-helpers/src/lib.rs @@ -23,9 +23,13 @@ //! Note that `dummy_` prefixed values are meant to be fillers, that should not matter, and will //! contain randomness based data. use polkadot_primitives::{ + vstaging::{ + CandidateDescriptorV2, CandidateReceiptV2, CommittedCandidateReceiptV2, + VersionedCandidateReceipt, + }, CandidateCommitments, CandidateDescriptor, CandidateReceipt, CollatorId, CollatorSignature, - CommittedCandidateReceipt, Hash, HeadData, Id as ParaId, PersistedValidationData, - ValidationCode, ValidationCodeHash, ValidatorId, + CommittedCandidateReceipt, CoreIndex, Hash, HeadData, Id as ParaId, PersistedValidationData, + SessionIndex, ValidationCode, ValidationCodeHash, ValidatorId, }; pub use rand; use sp_application_crypto::sr25519; @@ -42,6 +46,14 @@ pub fn dummy_candidate_receipt>(relay_parent: H) -> CandidateRece } } +/// Creates a v2 candidate receipt with filler data. +pub fn dummy_candidate_receipt_v2>(relay_parent: H) -> CandidateReceiptV2 { + CandidateReceiptV2:: { + commitments_hash: dummy_candidate_commitments(dummy_head_data()).hash(), + descriptor: dummy_candidate_descriptor_v2(relay_parent), + } +} + /// Creates a committed candidate receipt with filler data. pub fn dummy_committed_candidate_receipt>( relay_parent: H, @@ -52,6 +64,14 @@ pub fn dummy_committed_candidate_receipt>( } } +/// Creates a v2 committed candidate receipt with filler data. +pub fn dummy_committed_candidate_receipt_v2(relay_parent: H) -> CommittedCandidateReceiptV2 { + CommittedCandidateReceiptV2 { + descriptor: dummy_candidate_descriptor_v2::(relay_parent), + commitments: dummy_candidate_commitments(dummy_head_data()), + } +} + /// Create a candidate receipt with a bogus signature and filler data. Optionally set the commitment /// hash with the `commitments` arg. pub fn dummy_candidate_receipt_bad_sig( @@ -124,6 +144,23 @@ pub fn dummy_candidate_descriptor>(relay_parent: H) -> CandidateD descriptor } +/// Create a v2 candidate descriptor with filler data. +pub fn dummy_candidate_descriptor_v2(relay_parent: H) -> CandidateDescriptorV2 { + let invalid = Hash::zero(); + let descriptor = make_valid_candidate_descriptor_v2( + 1.into(), + relay_parent, + CoreIndex(1), + 1, + invalid, + invalid, + invalid, + invalid, + invalid, + ); + descriptor +} + /// Create meaningless validation code. pub fn dummy_validation_code() -> ValidationCode { ValidationCode(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -232,6 +269,34 @@ pub fn make_valid_candidate_descriptor>( descriptor } +/// Create a v2 candidate descriptor. +pub fn make_valid_candidate_descriptor_v2( + para_id: ParaId, + relay_parent: H, + core_index: CoreIndex, + session_index: SessionIndex, + persisted_validation_data_hash: Hash, + pov_hash: Hash, + validation_code_hash: impl Into, + para_head: Hash, + erasure_root: Hash, +) -> CandidateDescriptorV2 { + let validation_code_hash = validation_code_hash.into(); + + let descriptor = CandidateDescriptorV2::new( + para_id, + relay_parent, + core_index, + session_index, + persisted_validation_data_hash, + pov_hash, + erasure_root, + para_head, + validation_code_hash, + ); + + descriptor +} /// After manually modifying the candidate descriptor, resign with a defined collator key. pub fn resign_candidate_descriptor_with_collator>( descriptor: &mut CandidateDescriptor, From 4a53577abb8234e4174f4ab1545012f7cf066bf0 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 7 Aug 2024 17:01:00 +0300 Subject: [PATCH 003/103] Working version. Signed-off-by: Andrei Sandu --- polkadot/primitives/src/v7/mod.rs | 1 - polkadot/primitives/src/vstaging/mod.rs | 572 +++++++++++++++----- polkadot/primitives/test-helpers/src/lib.rs | 8 +- 3 files changed, 452 insertions(+), 129 deletions(-) diff --git a/polkadot/primitives/src/v7/mod.rs b/polkadot/primitives/src/v7/mod.rs index 18f52951986a..4b7abdb311f5 100644 --- a/polkadot/primitives/src/v7/mod.rs +++ b/polkadot/primitives/src/v7/mod.rs @@ -15,7 +15,6 @@ // along with Polkadot. If not, see . //! `V7` Primitives. - use alloc::{ vec, vec::{IntoIter, Vec}, diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index b259ba022772..749645753991 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -15,18 +15,20 @@ // along with Polkadot. If not, see . //! Staging Primitives. +use crate::{CandidateReceipt, CommittedCandidateReceipt, ValidityAttestation}; // Put any primitives used by staging APIs functions here use super::{ Balance, BlakeTwo256, BlockNumber, CandidateCommitments, CandidateDescriptor, CandidateHash, - CoreIndex, Hash, HashT, Id, Id as ParaId, ValidationCode, ValidationCodeHash, + CollatorId, CollatorSignature, CoreIndex, Hash, HashT, Id, Id as ParaId, ValidationCodeHash, ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, }; +use bitvec::prelude::*; +use sp_application_crypto::ByteArray; use alloc::collections::{btree_map::BTreeMap, vec_deque::VecDeque}; use codec::{Decode, Encode}; use scale_info::TypeInfo; -use sp_api::Core; use sp_arithmetic::Perbill; use sp_core::RuntimeDebug; use sp_staking::SessionIndex; @@ -119,7 +121,7 @@ pub struct CandidateDescriptorV2 { /// The session index of the candidate relay parent. session_index: SessionIndex, /// Reserved bytes. - reserved28b: [u8; 26], + reserved26b: [u8; 26], /// The blake2-256 hash of the persisted validation data. This is extra data derived from /// relay-chain state which may vary based on bitfields included before the candidate. /// Thus it cannot be derived entirely from the relay-parent. @@ -154,7 +156,7 @@ impl CandidateDescriptorV2 { relay_parent, core_index: core_index.0 as u16, session_index, - reserved28b: [0; 26], + reserved26b: [0; 26], persisted_validation_data_hash, pov_hash, erasure_root, @@ -183,81 +185,37 @@ pub trait CandidateApiV1 { /// Return the blake2-256 hash of the validation code bytes. fn validation_code_hash(&self) -> ValidationCodeHash; + + /// The collator's sr25519 public key. + fn collator(&self) -> Option<&CollatorId>; + + /// The parachain index, the relay parent, the validation data hash, and the `pov_hash`. + fn signature(&self) -> Option<&CollatorSignature>; } /// Version 2 API to access additional information stored by candidate descriptors pub trait CandidateApiV2 { /// Returns the core index where the candidate is backed. - fn core_index(&self) -> CoreIndex; + fn core_index(&self) -> Option; /// Returns the session index of the candidate relay parent. - fn session_index(&self) -> SessionIndex; -} - -impl CandidateApiV1 for CandidateDescriptor { - fn para_id(&self) -> Id { - self.para_id - } - - fn persisted_validation_data_hash(&self) -> Hash { - self.persisted_validation_data_hash - } - - fn pov_hash(&self) -> Hash { - self.pov_hash - } - - fn erasure_root(&self) -> Hash { - self.erasure_root - } - - fn para_head(&self) -> Hash { - self.para_head - } - - fn validation_code_hash(&self) -> ValidationCodeHash { - self.validation_code_hash - } + fn session_index(&self) -> Option; } -impl CandidateApiV1 for CandidateDescriptorV2 { - fn para_id(&self) -> Id { - self.para_id - } - - fn persisted_validation_data_hash(&self) -> Hash { - self.persisted_validation_data_hash - } - - fn pov_hash(&self) -> Hash { - self.pov_hash - } - - fn erasure_root(&self) -> Hash { - self.erasure_root - } - - fn para_head(&self) -> Hash { - self.para_head - } - - fn validation_code_hash(&self) -> ValidationCodeHash { - self.validation_code_hash - } -} impl CandidateApiV2 for CandidateDescriptorV2 { - fn core_index(&self) -> CoreIndex { - CoreIndex(self.core_index as u32) + fn core_index(&self) -> Option { + Some(CoreIndex(self.core_index as u32)) } - fn session_index(&self) -> SessionIndex { - self.session_index + fn session_index(&self) -> Option { + Some(self.session_index) } } /// A candidate-receipt at version 2. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(Hash))] pub struct CandidateReceiptV2 { /// The descriptor of the candidate. pub descriptor: CandidateDescriptorV2, @@ -356,28 +314,84 @@ pub enum UMPSignal { pub const UMP_SEPARATOR: Vec = vec![]; /// A versioned unique descriptor of the candidate receipt. -#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] -pub enum VersionedCandidateReceipt { +#[derive(PartialEq, Eq, Clone, TypeInfo, RuntimeDebug)] +pub enum VersionedCandidateReceipt { /// Version 1 of candidate receipt. - V1(super::CandidateReceipt), + V1(CandidateReceipt), /// Version 2 of candidate receipts with `core_index` and `session_index`. - V2(CandidateReceiptV2), + V2(CandidateReceiptV2), +} + +impl Encode for VersionedCandidateReceipt { + fn encode(&self) -> Vec { + match self { + VersionedCandidateReceipt::V1(inner) => inner.encode(), + VersionedCandidateReceipt::V2(inner) => inner.encode(), + } + } +} + +impl Decode for VersionedCandidateReceipt { + fn decode(input: &mut I) -> Result { + let descriptor: CandidateDescriptorV2 = Decode::decode(input)?; + let commitments_hash = Hash::decode(input)?; + + // Check if descriptor is v2. + if descriptor.reserved64b == [0u8; 64] && descriptor.reserved26b == [0u8; 26] { + return Ok(VersionedCandidateReceipt::V2(CandidateReceiptV2 { + descriptor, + commitments_hash, + })) + } + + // Fall back to v1. + // TODO: optimize, an additional encode and decode is not nice. + let descriptor = CandidateDescriptor::decode(&mut descriptor.encode().as_slice())?; + + Ok(VersionedCandidateReceipt::V1(CandidateReceipt { descriptor, commitments_hash })) + } +} + +impl Decode for VersionedCommittedCandidateReceipt { + fn decode(input: &mut I) -> Result { + let descriptor = CandidateDescriptorV2::decode(input)?; + let commitments = CandidateCommitments::decode(input)?; + + // Check if descriptor is v2. + if descriptor.reserved64b == [0u8; 64] && descriptor.reserved26b == [0u8; 26] { + return Ok(VersionedCommittedCandidateReceipt::V2(CommittedCandidateReceiptV2 { + descriptor, + commitments, + })) + } + + // Fall back to v1. + // TODO: optimize, an additional encode and decode is not nice. + let descriptor = CandidateDescriptor::decode(&mut descriptor.encode().as_slice())?; + + Ok(VersionedCommittedCandidateReceipt::V1(CommittedCandidateReceipt { + descriptor, + commitments, + })) + } } /// A versioned unique descriptor of the candidate receipt. -#[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] -pub enum VersionedCommittedCandidateReceipt { +#[derive(PartialEq, Eq, Clone, TypeInfo, RuntimeDebug)] +pub enum VersionedCommittedCandidateReceipt { + /// Version 1 V1(super::CommittedCandidateReceipt), + /// Version 2 V2(CommittedCandidateReceiptV2), } -impl VersionedCandidateReceipt { +impl VersionedCandidateReceipt { /// Returns the core index the candidate has commited to. Returns `None`` if /// the receipt is version 1. pub fn core_index(&self) -> Option { match self { Self::V1(_receipt_v1) => None, - Self::V2(receipt_v2) => Some(receipt_v2.descriptor.core_index()), + Self::V2(receipt_v2) => receipt_v2.descriptor.core_index(), } } @@ -386,18 +400,29 @@ impl VersionedCandidateReceipt { pub fn session_index(&self) -> Option { match self { Self::V1(_receipt_v1) => None, - Self::V2(receipt_v2) => Some(receipt_v2.descriptor.session_index()), + Self::V2(receipt_v2) => receipt_v2.descriptor.session_index(), + } + } + + /// Computes the blake2-256 hash of the receipt. + pub fn hash(&self) -> CandidateHash + where + H: Encode, + { + match self { + Self::V1(receipt_v1) => receipt_v1.hash(), + Self::V2(receipt_v2) => receipt_v2.hash(), } } } -impl VersionedCommittedCandidateReceipt { +impl VersionedCommittedCandidateReceipt { /// Returns the core index the candidate has commited to. Returns `None`` if /// the receipt is version 1. pub fn core_index(&self) -> Option { match self { Self::V1(_receipt_v1) => None, - Self::V2(receipt_v2) => Some(receipt_v2.descriptor.core_index()), + Self::V2(receipt_v2) => receipt_v2.descriptor.core_index(), } } @@ -406,7 +431,24 @@ impl VersionedCommittedCandidateReceipt { pub fn session_index(&self) -> Option { match self { Self::V1(_receipt_v1) => None, - Self::V2(receipt_v2) => Some(receipt_v2.descriptor.session_index()), + Self::V2(receipt_v2) => receipt_v2.descriptor.session_index(), + } + } + + /// Convert to a plain `CandidateReceipt`. + pub fn to_plain(&self) -> VersionedCandidateReceipt { + match self { + Self::V1(receipt) => VersionedCandidateReceipt::V1(receipt.to_plain()), + Self::V2(receipt) => VersionedCandidateReceipt::V2(receipt.to_plain()), + } + } + + /// Returns the candidate commitments. + /// Convert to a plain `CandidateReceipt`. + pub fn commitments(&self) -> &CandidateCommitments { + match self { + Self::V1(receipt) => &receipt.commitments, + Self::V2(receipt) => &receipt.commitments, } } } @@ -450,6 +492,44 @@ pub enum CandidateReceiptError { CoreIndexMismatch, /// The core selector or claim queue offset is invalid. InvalidSelectedCore, + /// The parachain is not assigned to any core at specified claim queue offset. + NoAssignment, + /// No core was selected. + NoCoreSelected, +} + +macro_rules! impl_candidate_api_v1 { + ($field:ident, $type:ident) => { + fn $field(&self) -> $type { + match self { + Self::V1(receipt) => receipt.descriptor.$field, + Self::V2(receipt) => receipt.descriptor.$field, + } + } + }; +} + +impl CandidateApiV1 for VersionedCommittedCandidateReceipt { + impl_candidate_api_v1!(erasure_root, Hash); + impl_candidate_api_v1!(para_head, Hash); + impl_candidate_api_v1!(para_id, ParaId); + impl_candidate_api_v1!(persisted_validation_data_hash, Hash); + impl_candidate_api_v1!(pov_hash, Hash); + impl_candidate_api_v1!(validation_code_hash, ValidationCodeHash); + + fn collator(&self) -> Option<&CollatorId> { + match self { + Self::V1(receipt) => Some(&receipt.descriptor.collator), + Self::V2(receipt) => None, + } + } + + fn signature(&self) -> Option<&CollatorSignature> { + match self { + Self::V1(receipt) => Some(&receipt.descriptor.signature), + Self::V2(receipt) => None, + } + } } impl CommittedCandidateReceiptV2 { @@ -478,26 +558,35 @@ impl CommittedCandidateReceiptV2 { pub fn check( &self, n_cores: u32, - claimed_cores: Vec, // TODO: consider promoting `ClaimQueueSnapshot` as primitive - claim_queue: BTreeMap>, + claim_queue: &BTreeMap>, ) -> Result<(), CandidateReceiptError> { - let descriptor_core_index = CoreIndex(descriptor.core_index as u32); - let (core_selector, cq_offset) = self - .commitments - .selected_core() - .ok_or(Err(CandidateReceiptError::InvalidSelectedCore))?; + if claim_queue.is_empty() { + return Err(CandidateReceiptError::NoAssignment) + } + + let claim_queue_depth = claim_queue + .first_key_value() + .ok_or(CandidateReceiptError::NoAssignment)? + .1 + .len(); + + let descriptor_core_index = CoreIndex(self.descriptor.core_index as u32); + let (core_selector, cq_offset) = + self.commitments.selected_core().ok_or(CandidateReceiptError::NoCoreSelected)?; let para_id = self.descriptor.para_id; - // TODO: bail out early if cq_offset > depth of claim queue. - + if cq_offset.0 as usize >= claim_queue_depth { + return Err(CandidateReceiptError::InvalidSelectedCore) + } + // Get a vec of the core indices the parachain is assigned to at `cq_offset`. let assigned_cores = claim_queue .iter() .filter_map(|(core_index, queue)| { - let queued_para = queue.get(cq_offset)?; + let queued_para = queue.get(cq_offset.0 as usize)?; - if queued_para == para_id { + if queued_para == ¶_id { Some(core_index) } else { None @@ -506,11 +595,15 @@ impl CommittedCandidateReceiptV2 { .cloned() .collect::>(); + if assigned_cores.is_empty() { + return Err(CandidateReceiptError::NoAssignment) + } + let core_index = *assigned_cores .get(core_selector.0 as usize % assigned_cores.len()) .expect("provided index is always less than queue len; qed"); - if core_index != Some(descriptor_core_index) { + if core_index != descriptor_core_index { return Err(CandidateReceiptError::CoreIndexMismatch) } @@ -522,62 +615,230 @@ impl CommittedCandidateReceiptV2 { } } +impl From> for VersionedCommittedCandidateReceipt { + fn from(value: CommittedCandidateReceipt) -> Self { + Self::V1(value) + } +} + +impl From> for VersionedCommittedCandidateReceipt { + fn from(value: CommittedCandidateReceiptV2) -> Self { + Self::V2(value) + } +} + +impl From> for VersionedCandidateReceipt { + fn from(value: CandidateReceipt) -> Self { + Self::V1(value) + } +} + +impl From> for VersionedCandidateReceipt { + fn from(value: CandidateReceiptV2) -> Self { + Self::V2(value) + } +} + +/// A backed (or backable, depending on context) candidate. +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo)] +pub struct BackedCandidate { + /// The candidate referred to. + candidate: VersionedCommittedCandidateReceipt, + /// The validity votes themselves, expressed as signatures. + validity_votes: Vec, + /// The indices of the validators within the group, expressed as a bitfield. May be extended + /// beyond the backing group size to contain the assigned core index, if ElasticScalingMVP is + /// enabled. + validator_indices: BitVec, +} + +impl BackedCandidate { + /// Constructor + pub fn new( + candidate: VersionedCommittedCandidateReceipt, + validity_votes: Vec, + validator_indices: BitVec, + core_index: Option, + ) -> Self { + let mut instance = Self { candidate, validity_votes, validator_indices }; + if let Some(core_index) = core_index { + instance.inject_core_index(core_index); + } + instance + } + + /// Get a reference to the committed candidate receipt of the candidate. + pub fn candidate(&self) -> &VersionedCommittedCandidateReceipt { + &self.candidate + } + + /// Get a reference to the validity votes of the candidate. + pub fn validity_votes(&self) -> &[ValidityAttestation] { + &self.validity_votes + } + + /// Get a mutable reference to validity votes of the para. + pub fn validity_votes_mut(&mut self) -> &mut Vec { + &mut self.validity_votes + } + + /// Compute this candidate's hash. + pub fn hash(&self) -> CandidateHash + where + H: Clone + Encode, + { + self.candidate.to_plain().hash() + } + + /// Get this candidate's receipt. + pub fn receipt(&self) -> VersionedCandidateReceipt + where + H: Clone, + { + self.candidate.to_plain() + } + + /// Get a copy of the validator indices and the assumed core index, if any. + pub fn validator_indices_and_core_index( + &self, + core_index_enabled: bool, + ) -> (&BitSlice, Option) { + // This flag tells us if the block producers must enable Elastic Scaling MVP hack. + // It extends `BackedCandidate::validity_indices` to store a 8 bit core index. + if core_index_enabled { + let core_idx_offset = self.validator_indices.len().saturating_sub(8); + if core_idx_offset > 0 { + let (validator_indices_slice, core_idx_slice) = + self.validator_indices.split_at(core_idx_offset); + return ( + validator_indices_slice, + Some(CoreIndex(core_idx_slice.load::() as u32)), + ); + } + } + + (&self.validator_indices, None) + } + + /// Inject a core index in the validator_indices bitvec. + fn inject_core_index(&mut self, core_index: CoreIndex) { + let core_index_to_inject: BitVec = + BitVec::from_vec(vec![core_index.0 as u8]); + self.validator_indices.extend(core_index_to_inject); + } + + /// Update the validator indices and core index in the candidate. + pub fn set_validator_indices_and_core_index( + &mut self, + new_indices: BitVec, + maybe_core_index: Option, + ) { + self.validator_indices = new_indices; + + if let Some(core_index) = maybe_core_index { + self.inject_core_index(core_index); + } + } +} + #[cfg(test)] mod tests { + use core::prelude::v1; + use super::*; use crate::{ v7::{ tests::dummy_committed_candidate_receipt as dummy_old_committed_candidate_receipt, CandidateDescriptor, CandidateReceipt as OldCandidateReceipt, - CommittedCandidateReceipt, Hash, HeadData, + CommittedCandidateReceipt, Hash, HeadData, ValidationCode, + }, + vstaging::{ + CandidateApiV1, CandidateApiV2, CandidateDescriptorV2, CommittedCandidateReceiptV2, }, - vstaging::CommittedCandidateReceiptV2, }; - use polkadot_primitives_test_helpers::dummy_committed_candidate_receipt_v2; - - // pub fn dummy_committed_candidate_receipt() -> CommittedCandidateReceiptV2 { - // let zeros = Hash::zero(); - // let reserved64b = [0; 64]; - - // CommittedCandidateReceiptV2 { - // descriptor: CandidateDescriptorV2 { - // para_id: 0.into(), - // relay_parent: zeros, - // core_index: 123, - // session_index: 1, - // reserved28b: Default::default(), - // persisted_validation_data_hash: zeros, - // pov_hash: zeros, - // erasure_root: zeros, - // reserved64b, - // para_head: zeros, - // validation_code_hash: ValidationCode(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]).hash(), - // }, - // commitments: CandidateCommitments { - // head_data: HeadData(vec![]), - // upward_messages: vec![].try_into().expect("empty vec fits within bounds"), - // new_validation_code: None, - // horizontal_messages: vec![].try_into().expect("empty vec fits within bounds"), - // processed_downward_messages: 0, - // hrmp_watermark: 0_u32, - // }, - // } - // } + // use polkadot_primitives_test_helpers::dummy_committed_candidate_receipt_v2; + + pub fn dummy_committed_candidate_receipt_v2() -> CommittedCandidateReceiptV2 { + let zeros = Hash::zero(); + let reserved64b = [0; 64]; + + CommittedCandidateReceiptV2 { + descriptor: CandidateDescriptorV2 { + para_id: 0.into(), + relay_parent: zeros, + core_index: 123, + session_index: 1, + reserved26b: Default::default(), + persisted_validation_data_hash: zeros, + pov_hash: zeros, + erasure_root: zeros, + reserved64b, + para_head: zeros, + validation_code_hash: ValidationCode(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]).hash(), + }, + commitments: CandidateCommitments { + head_data: HeadData(vec![]), + upward_messages: vec![].try_into().expect("empty vec fits within bounds"), + new_validation_code: None, + horizontal_messages: vec![].try_into().expect("empty vec fits within bounds"), + processed_downward_messages: 0, + hrmp_watermark: 0_u32, + }, + } + } #[test] fn is_binary_compatibile() { - let mut old_ccr = dummy_old_committed_candidate_receipt(); - let mut new_ccr = dummy_committed_candidate_receipt_v2(Hash::zero()); + let old_ccr = dummy_old_committed_candidate_receipt(); + let new_ccr = dummy_committed_candidate_receipt_v2(); assert_eq!(old_ccr.encoded_size(), new_ccr.encoded_size()); - assert_eq!(new_ccr.commitments().core_index(), None); + + let encoded_old = old_ccr.encode(); + + // Deserialize from old candidate receipt. + let new_ccr: CommittedCandidateReceiptV2 = + Decode::decode(&mut encoded_old.as_slice()).unwrap(); + + // We get same candidate hash. + assert_eq!(old_ccr.hash(), new_ccr.hash()); } #[test] fn test_ump_commitment() { - let mut new_ccr = dummy_committed_candidate_receipt_v2(Hash::zero()); + let mut new_ccr = dummy_committed_candidate_receipt_v2(); + new_ccr.descriptor.core_index = 123; + new_ccr.descriptor.para_id = ParaId::new(1000); + + // dummy XCM messages + new_ccr.commitments.upward_messages.force_push(vec![0u8; 256]); + new_ccr.commitments.upward_messages.force_push(vec![0xff; 256]); + + // separator + new_ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); + + // CoreIndex commitment + new_ccr + .commitments + .upward_messages + .force_push(UMPSignal::SelectCore(CoreSelector(0), ClaimQueueOffset(1)).encode()); + + let mut claim_queue = BTreeMap::new(); + claim_queue.insert( + new_ccr.descriptor().core_index().unwrap(), + vec![2.into(), new_ccr.descriptor.para_id, 3.into()].into(), + ); + + assert_eq!(new_ccr.check(200, &claim_queue), Ok(())); + } - // XCM messages + #[test] + fn test_versioned_receipt() { + let mut new_ccr = dummy_committed_candidate_receipt_v2(); + new_ccr.descriptor.core_index = 123; + new_ccr.descriptor.para_id = ParaId::new(1000); + + // dummy XCM messages new_ccr.commitments.upward_messages.force_push(vec![0u8; 256]); new_ccr.commitments.upward_messages.force_push(vec![0xff; 256]); @@ -588,9 +849,70 @@ mod tests { new_ccr .commitments .upward_messages - .force_push(UMPSignal::OnCore(CoreIndex(123)).encode()); + .force_push(UMPSignal::SelectCore(CoreSelector(0), ClaimQueueOffset(1)).encode()); + + let mut claim_queue = BTreeMap::new(); + claim_queue.insert( + new_ccr.descriptor().core_index().unwrap(), + vec![2.into(), new_ccr.descriptor.para_id, 3.into()].into(), + ); + + let encoded_ccr = new_ccr.encode(); + let versioned_ccr = + VersionedCommittedCandidateReceipt::decode(&mut encoded_ccr.as_slice()).unwrap(); + + assert_eq!(versioned_ccr.core_index(), Some(CoreIndex(123))); + + if let VersionedCommittedCandidateReceipt::V2(ref v2_receipt) = versioned_ccr { + assert_eq!(v2_receipt.check(200, &claim_queue), Ok(())); + } + + assert_eq!(new_ccr.hash(), versioned_ccr.to_plain().hash()); + } + + #[test] + fn test_backward_compatible() { + // Testing edge case when collators provide zeroed signature and collator id. + let mut old_ccr = dummy_old_committed_candidate_receipt(); + old_ccr.descriptor.para_id = ParaId::new(1000); + let encoded_ccr: Vec = old_ccr.encode(); + + let versioned_ccr = + VersionedCommittedCandidateReceipt::decode(&mut encoded_ccr.as_slice()).unwrap(); + + // Since collator sig and id are zeroed, it was decoded as V2. + // We expect the check to fail in such case because there will be no `SelectCore` + // commitment. + let mut claim_queue = BTreeMap::new(); + claim_queue.insert( + versioned_ccr.core_index().unwrap(), + vec![2.into(), versioned_ccr.para_id(), 3.into()].into(), + ); + + if let VersionedCommittedCandidateReceipt::V2(ref v2_receipt) = versioned_ccr { + assert_eq!( + v2_receipt.check(200, &claim_queue), + Err(CandidateReceiptError::NoCoreSelected) + ); + } else { + panic!("Should have decoded as V2") + } + + // Adding collator signature should make it decode as v1. + old_ccr.descriptor.signature = + CollatorSignature::from_slice(&vec![99u8; 64]).expect("64 bytes; qed"); + let encoded_ccr: Vec = old_ccr.encode(); + let versioned_ccr = + VersionedCommittedCandidateReceipt::decode(&mut encoded_ccr.as_slice()).unwrap(); + + if let VersionedCommittedCandidateReceipt::V1(ref v1_receipt) = versioned_ccr { + assert_eq!(v1_receipt.descriptor.signature, old_ccr.descriptor.signature); + } else { + panic!("Should have decoded as V1") + } - let new_ccr = VersionedCommittedCandidateReceipt::V2(new_ccr); - assert_eq!(new_ccr.core_index(), Some(CoreIndex(123))); + assert_eq!(versioned_ccr.core_index(), None); + assert_eq!(versioned_ccr.para_id(), ParaId::new(1000)); + assert_eq!(old_ccr.hash(), versioned_ccr.to_plain().hash()); } } diff --git a/polkadot/primitives/test-helpers/src/lib.rs b/polkadot/primitives/test-helpers/src/lib.rs index c06c0420fd70..c262f16df077 100644 --- a/polkadot/primitives/test-helpers/src/lib.rs +++ b/polkadot/primitives/test-helpers/src/lib.rs @@ -65,7 +65,9 @@ pub fn dummy_committed_candidate_receipt>( } /// Creates a v2 committed candidate receipt with filler data. -pub fn dummy_committed_candidate_receipt_v2(relay_parent: H) -> CommittedCandidateReceiptV2 { +pub fn dummy_committed_candidate_receipt_v2>( + relay_parent: H, +) -> CommittedCandidateReceiptV2 { CommittedCandidateReceiptV2 { descriptor: dummy_candidate_descriptor_v2::(relay_parent), commitments: dummy_candidate_commitments(dummy_head_data()), @@ -145,7 +147,7 @@ pub fn dummy_candidate_descriptor>(relay_parent: H) -> CandidateD } /// Create a v2 candidate descriptor with filler data. -pub fn dummy_candidate_descriptor_v2(relay_parent: H) -> CandidateDescriptorV2 { +pub fn dummy_candidate_descriptor_v2>(relay_parent: H) -> CandidateDescriptorV2 { let invalid = Hash::zero(); let descriptor = make_valid_candidate_descriptor_v2( 1.into(), @@ -270,7 +272,7 @@ pub fn make_valid_candidate_descriptor>( } /// Create a v2 candidate descriptor. -pub fn make_valid_candidate_descriptor_v2( +pub fn make_valid_candidate_descriptor_v2>( para_id: ParaId, relay_parent: H, core_index: CoreIndex, From 8285ae79e16e62a373d704fed328abf702ed125e Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 8 Aug 2024 17:09:35 +0300 Subject: [PATCH 004/103] Better version Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 459 +++++++------------- polkadot/primitives/test-helpers/src/lib.rs | 5 +- 2 files changed, 159 insertions(+), 305 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 749645753991..03075e820524 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -15,12 +15,13 @@ // along with Polkadot. If not, see . //! Staging Primitives. -use crate::{CandidateReceipt, CommittedCandidateReceipt, ValidityAttestation}; +use crate::ValidityAttestation; // Put any primitives used by staging APIs functions here use super::{ - Balance, BlakeTwo256, BlockNumber, CandidateCommitments, CandidateDescriptor, CandidateHash, - CollatorId, CollatorSignature, CoreIndex, Hash, HashT, Id, Id as ParaId, ValidationCodeHash, + Balance, BlakeTwo256, BlockNumber, CandidateCommitments, CandidateHash, + CollatorId, CollatorSignature, CoreIndex, Hash, HashT, Header, Id, Id as ParaId, + MultiDisputeStatementSet, UncheckedSignedAvailabilityBitfields, ValidationCodeHash, ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, }; use bitvec::prelude::*; @@ -31,6 +32,7 @@ use codec::{Decode, Encode}; use scale_info::TypeInfo; use sp_arithmetic::Perbill; use sp_core::RuntimeDebug; +use sp_runtime::traits::Header as HeaderT; use sp_staking::SessionIndex; /// Scheduler configuration parameters. All coretime/ondemand parameters are here. @@ -108,6 +110,22 @@ impl> Default for SchedulerParams } } +/// A type representing the version of the candidate descriptor and internal version number. +#[derive(PartialEq, Eq, Encode, Decode, Clone, TypeInfo, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(Hash))] +pub struct InternalVersion(pub u8); + +/// A type representing the version of the candidate descriptor and internal version number. +/// For `V1`` the internal version number stores the first byte of the `CollatorId`. +#[derive(PartialEq, Eq, Clone, TypeInfo, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(Hash))] +pub enum CandidateDescriptorVersion { + /// The old candidate descriptor version. + V1, + /// Introduced with `CandidateDescriptorV2` + V2, +} + /// A unique descriptor of the candidate receipt. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Hash))] @@ -116,12 +134,14 @@ pub struct CandidateDescriptorV2 { para_id: ParaId, /// The hash of the relay-chain block this is executed in the context of. relay_parent: H, + /// Version field + version: InternalVersion, /// The core index where the candidate is backed. core_index: u16, /// The session index of the candidate relay parent. session_index: SessionIndex, /// Reserved bytes. - reserved26b: [u8; 26], + reserved25b: [u8; 25], /// The blake2-256 hash of the persisted validation data. This is extra data derived from /// relay-chain state which may vary based on bitfields included before the candidate. /// Thus it cannot be derived entirely from the relay-parent. @@ -154,9 +174,10 @@ impl CandidateDescriptorV2 { Self { para_id, relay_parent, + version: InternalVersion(0), core_index: core_index.0 as u16, session_index, - reserved26b: [0; 26], + reserved25b: [0; 25], persisted_validation_data_hash, pov_hash, erasure_root, @@ -166,52 +187,6 @@ impl CandidateDescriptorV2 { } } } -/// Version 1 API to access information stored by candidate descriptors. -pub trait CandidateApiV1 { - /// Returns the ID of the para this is a candidate for. - fn para_id(&self) -> Id; - - /// Returns the blake2-256 hash of the persisted validation data. - fn persisted_validation_data_hash(&self) -> Hash; - - /// Returns the blake2-256 hash of the PoV. - fn pov_hash(&self) -> Hash; - - /// Returns the root of a block's erasure encoding Merkle tree. - fn erasure_root(&self) -> Hash; - - /// Returns the hash of the para header generated by this candidate. - fn para_head(&self) -> Hash; - - /// Return the blake2-256 hash of the validation code bytes. - fn validation_code_hash(&self) -> ValidationCodeHash; - - /// The collator's sr25519 public key. - fn collator(&self) -> Option<&CollatorId>; - - /// The parachain index, the relay parent, the validation data hash, and the `pov_hash`. - fn signature(&self) -> Option<&CollatorSignature>; -} - -/// Version 2 API to access additional information stored by candidate descriptors -pub trait CandidateApiV2 { - /// Returns the core index where the candidate is backed. - fn core_index(&self) -> Option; - - /// Returns the session index of the candidate relay parent. - fn session_index(&self) -> Option; -} - - -impl CandidateApiV2 for CandidateDescriptorV2 { - fn core_index(&self) -> Option { - Some(CoreIndex(self.core_index as u32)) - } - - fn session_index(&self) -> Option { - Some(self.session_index) - } -} /// A candidate-receipt at version 2. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] @@ -285,9 +260,9 @@ impl PartialOrd for CommittedCandidateReceiptV2 { impl Ord for CommittedCandidateReceiptV2 { fn cmp(&self, other: &Self) -> core::cmp::Ordering { - self.descriptor() + self.descriptor .para_id - .cmp(&other.descriptor().para_id) + .cmp(&other.descriptor.para_id) .then_with(|| self.commitments.head_data.cmp(&other.commitments.head_data)) } } @@ -303,6 +278,7 @@ pub struct ClaimQueueOffset(pub u8); /// Default claim queue offset pub const DEFAULT_CLAIM_QUEUE_OFFSET: ClaimQueueOffset = ClaimQueueOffset(1); +/// Signals that a parachain can send to the relay chain via the UMP queue. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] pub enum UMPSignal { /// A message sent by a parachain to select the core the candidate is commited to. @@ -313,146 +289,6 @@ pub enum UMPSignal { /// Separator between `XCM` and `UMPSignal`. pub const UMP_SEPARATOR: Vec = vec![]; -/// A versioned unique descriptor of the candidate receipt. -#[derive(PartialEq, Eq, Clone, TypeInfo, RuntimeDebug)] -pub enum VersionedCandidateReceipt { - /// Version 1 of candidate receipt. - V1(CandidateReceipt), - /// Version 2 of candidate receipts with `core_index` and `session_index`. - V2(CandidateReceiptV2), -} - -impl Encode for VersionedCandidateReceipt { - fn encode(&self) -> Vec { - match self { - VersionedCandidateReceipt::V1(inner) => inner.encode(), - VersionedCandidateReceipt::V2(inner) => inner.encode(), - } - } -} - -impl Decode for VersionedCandidateReceipt { - fn decode(input: &mut I) -> Result { - let descriptor: CandidateDescriptorV2 = Decode::decode(input)?; - let commitments_hash = Hash::decode(input)?; - - // Check if descriptor is v2. - if descriptor.reserved64b == [0u8; 64] && descriptor.reserved26b == [0u8; 26] { - return Ok(VersionedCandidateReceipt::V2(CandidateReceiptV2 { - descriptor, - commitments_hash, - })) - } - - // Fall back to v1. - // TODO: optimize, an additional encode and decode is not nice. - let descriptor = CandidateDescriptor::decode(&mut descriptor.encode().as_slice())?; - - Ok(VersionedCandidateReceipt::V1(CandidateReceipt { descriptor, commitments_hash })) - } -} - -impl Decode for VersionedCommittedCandidateReceipt { - fn decode(input: &mut I) -> Result { - let descriptor = CandidateDescriptorV2::decode(input)?; - let commitments = CandidateCommitments::decode(input)?; - - // Check if descriptor is v2. - if descriptor.reserved64b == [0u8; 64] && descriptor.reserved26b == [0u8; 26] { - return Ok(VersionedCommittedCandidateReceipt::V2(CommittedCandidateReceiptV2 { - descriptor, - commitments, - })) - } - - // Fall back to v1. - // TODO: optimize, an additional encode and decode is not nice. - let descriptor = CandidateDescriptor::decode(&mut descriptor.encode().as_slice())?; - - Ok(VersionedCommittedCandidateReceipt::V1(CommittedCandidateReceipt { - descriptor, - commitments, - })) - } -} - -/// A versioned unique descriptor of the candidate receipt. -#[derive(PartialEq, Eq, Clone, TypeInfo, RuntimeDebug)] -pub enum VersionedCommittedCandidateReceipt { - /// Version 1 - V1(super::CommittedCandidateReceipt), - /// Version 2 - V2(CommittedCandidateReceiptV2), -} - -impl VersionedCandidateReceipt { - /// Returns the core index the candidate has commited to. Returns `None`` if - /// the receipt is version 1. - pub fn core_index(&self) -> Option { - match self { - Self::V1(_receipt_v1) => None, - Self::V2(receipt_v2) => receipt_v2.descriptor.core_index(), - } - } - - /// Returns the session index of the relay parent. Returns `None`` if - /// the receipt is version 1. - pub fn session_index(&self) -> Option { - match self { - Self::V1(_receipt_v1) => None, - Self::V2(receipt_v2) => receipt_v2.descriptor.session_index(), - } - } - - /// Computes the blake2-256 hash of the receipt. - pub fn hash(&self) -> CandidateHash - where - H: Encode, - { - match self { - Self::V1(receipt_v1) => receipt_v1.hash(), - Self::V2(receipt_v2) => receipt_v2.hash(), - } - } -} - -impl VersionedCommittedCandidateReceipt { - /// Returns the core index the candidate has commited to. Returns `None`` if - /// the receipt is version 1. - pub fn core_index(&self) -> Option { - match self { - Self::V1(_receipt_v1) => None, - Self::V2(receipt_v2) => receipt_v2.descriptor.core_index(), - } - } - - /// Returns the session index of the relay parent. Returns `None` if - /// the receipt is version 1. - pub fn session_index(&self) -> Option { - match self { - Self::V1(_receipt_v1) => None, - Self::V2(receipt_v2) => receipt_v2.descriptor.session_index(), - } - } - - /// Convert to a plain `CandidateReceipt`. - pub fn to_plain(&self) -> VersionedCandidateReceipt { - match self { - Self::V1(receipt) => VersionedCandidateReceipt::V1(receipt.to_plain()), - Self::V2(receipt) => VersionedCandidateReceipt::V2(receipt.to_plain()), - } - } - - /// Returns the candidate commitments. - /// Convert to a plain `CandidateReceipt`. - pub fn commitments(&self) -> &CandidateCommitments { - match self { - Self::V1(receipt) => &receipt.commitments, - Self::V2(receipt) => &receipt.commitments, - } - } -} - impl CandidateCommitments { /// Returns the core selector and claim queue offset the candidate has commited to, if any. pub fn selected_core(&self) -> Option<(CoreSelector, ClaimQueueOffset)> { @@ -498,58 +334,84 @@ pub enum CandidateReceiptError { NoCoreSelected, } -macro_rules! impl_candidate_api_v1 { +macro_rules! impl_getter { ($field:ident, $type:ident) => { fn $field(&self) -> $type { - match self { - Self::V1(receipt) => receipt.descriptor.$field, - Self::V2(receipt) => receipt.descriptor.$field, - } + self.$field } }; } -impl CandidateApiV1 for VersionedCommittedCandidateReceipt { - impl_candidate_api_v1!(erasure_root, Hash); - impl_candidate_api_v1!(para_head, Hash); - impl_candidate_api_v1!(para_id, ParaId); - impl_candidate_api_v1!(persisted_validation_data_hash, Hash); - impl_candidate_api_v1!(pov_hash, Hash); - impl_candidate_api_v1!(validation_code_hash, ValidationCodeHash); - - fn collator(&self) -> Option<&CollatorId> { - match self { - Self::V1(receipt) => Some(&receipt.descriptor.collator), - Self::V2(receipt) => None, +impl CandidateDescriptorV2 { + impl_getter!(erasure_root, Hash); + impl_getter!(para_head, Hash); + impl_getter!(relay_parent, Hash); + impl_getter!(para_id, ParaId); + impl_getter!(persisted_validation_data_hash, Hash); + impl_getter!(pov_hash, Hash); + impl_getter!(validation_code_hash, ValidationCodeHash); + + /// Returns the candidate descriptor version. + /// The candidate is at version 2 if the reserved fields are zeroed out + /// and the internal `version` field is 0. + pub fn version(&self) -> CandidateDescriptorVersion { + if self.reserved64b != [0u8; 64] || self.reserved25b != [0u8; 25] { + return CandidateDescriptorVersion::V1 + } + + match self.version.0 { + 0 => CandidateDescriptorVersion::V2, + _ => CandidateDescriptorVersion::V1, } } - fn signature(&self) -> Option<&CollatorSignature> { - match self { - Self::V1(receipt) => Some(&receipt.descriptor.signature), - Self::V2(receipt) => None, + /// Returns the collator id if this is a v1 `CandidateDescriptor` + pub fn collator(&self) -> Option { + if self.version() == CandidateDescriptorVersion::V1 { + let mut collator_id = vec![self.version.0]; + let core_index: [u8; 2] = unsafe { std::mem::transmute(self.core_index) }; + let session_index: [u8; 4] = unsafe { std::mem::transmute(self.session_index) }; + + collator_id.append(&mut core_index.as_slice().to_vec()); + collator_id.append(&mut session_index.as_slice().to_vec()); + collator_id.append(&mut self.reserved25b.as_slice().to_vec()); + + return Some(CollatorId::from_slice(&collator_id.as_slice()).ok()?) } + + None } -} -impl CommittedCandidateReceiptV2 { - /// Returns a v2 commited candidate receipt if the committed selected core - /// matches the core index in the descriptor. - pub fn new(descriptor: CandidateDescriptorV2, commitments: CandidateCommitments) -> Self { - Self { descriptor, commitments } + /// Returns the collator signature of `V1` candidate descriptors, `None` otherwise. + pub fn signature(&self) -> Option { + if self.version() == CandidateDescriptorVersion::V1 { + return Some(CollatorSignature::from_slice(self.reserved64b.as_slice()).ok()?) + } + + None } - /// Returns a reference to commitments - pub fn commitments(&self) -> &CandidateCommitments { - &self.commitments + /// Returns the `core_index` of `V2` candidate descriptors, `None` otherwise. + pub fn core_index(&self) -> Option { + if self.version() == CandidateDescriptorVersion::V1 { + return None + } + + Some(CoreIndex(self.core_index as u32)) } - /// Returns a reference to the descriptor - pub fn descriptor(&self) -> &CandidateDescriptorV2 { - &self.descriptor + /// Returns the `core_index` of `V2` candidate descriptors, `None` otherwise. + pub fn session_index(&self) -> Option { + if self.version() == CandidateDescriptorVersion::V1 { + return None + } + + Some(self.session_index) } +} - /// Performs a sanity check of the receipt. +impl CommittedCandidateReceiptV2 { + /// Performs a sanity check of the descriptor and commitment. /// /// Returns error if: /// - descriptor core index is different than the core selected @@ -615,35 +477,11 @@ impl CommittedCandidateReceiptV2 { } } -impl From> for VersionedCommittedCandidateReceipt { - fn from(value: CommittedCandidateReceipt) -> Self { - Self::V1(value) - } -} - -impl From> for VersionedCommittedCandidateReceipt { - fn from(value: CommittedCandidateReceiptV2) -> Self { - Self::V2(value) - } -} - -impl From> for VersionedCandidateReceipt { - fn from(value: CandidateReceipt) -> Self { - Self::V1(value) - } -} - -impl From> for VersionedCandidateReceipt { - fn from(value: CandidateReceiptV2) -> Self { - Self::V2(value) - } -} - /// A backed (or backable, depending on context) candidate. #[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo)] pub struct BackedCandidate { /// The candidate referred to. - candidate: VersionedCommittedCandidateReceipt, + candidate: CommittedCandidateReceiptV2, /// The validity votes themselves, expressed as signatures. validity_votes: Vec, /// The indices of the validators within the group, expressed as a bitfield. May be extended @@ -652,10 +490,23 @@ pub struct BackedCandidate { validator_indices: BitVec, } +/// Parachains inherent-data passed into the runtime by a block author +#[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug, TypeInfo)] +pub struct InherentData { + /// Signed bitfields by validators about availability. + pub bitfields: UncheckedSignedAvailabilityBitfields, + /// Backed candidates for inclusion in the block. + pub backed_candidates: Vec>, + /// Sets of dispute votes for inclusion, + pub disputes: MultiDisputeStatementSet, + /// The parent block header. Used for checking state proofs. + pub parent_header: HDR, +} + impl BackedCandidate { /// Constructor pub fn new( - candidate: VersionedCommittedCandidateReceipt, + candidate: CommittedCandidateReceiptV2, validity_votes: Vec, validator_indices: BitVec, core_index: Option, @@ -668,7 +519,7 @@ impl BackedCandidate { } /// Get a reference to the committed candidate receipt of the candidate. - pub fn candidate(&self) -> &VersionedCommittedCandidateReceipt { + pub fn candidate(&self) -> &CommittedCandidateReceiptV2 { &self.candidate } @@ -691,7 +542,7 @@ impl BackedCandidate { } /// Get this candidate's receipt. - pub fn receipt(&self) -> VersionedCandidateReceipt + pub fn receipt(&self) -> CandidateReceiptV2 where H: Clone, { @@ -743,20 +594,14 @@ impl BackedCandidate { #[cfg(test)] mod tests { - use core::prelude::v1; - use super::*; use crate::{ v7::{ tests::dummy_committed_candidate_receipt as dummy_old_committed_candidate_receipt, - CandidateDescriptor, CandidateReceipt as OldCandidateReceipt, CommittedCandidateReceipt, Hash, HeadData, ValidationCode, }, - vstaging::{ - CandidateApiV1, CandidateApiV2, CandidateDescriptorV2, CommittedCandidateReceiptV2, - }, + vstaging::{CandidateDescriptorV2, CommittedCandidateReceiptV2}, }; - // use polkadot_primitives_test_helpers::dummy_committed_candidate_receipt_v2; pub fn dummy_committed_candidate_receipt_v2() -> CommittedCandidateReceiptV2 { let zeros = Hash::zero(); @@ -766,9 +611,10 @@ mod tests { descriptor: CandidateDescriptorV2 { para_id: 0.into(), relay_parent: zeros, + version: InternalVersion(0), core_index: 123, session_index: 1, - reserved26b: Default::default(), + reserved25b: Default::default(), persisted_validation_data_hash: zeros, pov_hash: zeros, erasure_root: zeros, @@ -825,7 +671,7 @@ mod tests { let mut claim_queue = BTreeMap::new(); claim_queue.insert( - new_ccr.descriptor().core_index().unwrap(), + new_ccr.descriptor.core_index().unwrap(), vec![2.into(), new_ccr.descriptor.para_id, 3.into()].into(), ); @@ -833,7 +679,7 @@ mod tests { } #[test] - fn test_versioned_receipt() { + fn test_version2_receipts_decoded_as_v1() { let mut new_ccr = dummy_committed_candidate_receipt_v2(); new_ccr.descriptor.core_index = 123; new_ccr.descriptor.para_id = ParaId::new(1000); @@ -853,66 +699,77 @@ mod tests { let mut claim_queue = BTreeMap::new(); claim_queue.insert( - new_ccr.descriptor().core_index().unwrap(), + new_ccr.descriptor.core_index().unwrap(), vec![2.into(), new_ccr.descriptor.para_id, 3.into()].into(), ); let encoded_ccr = new_ccr.encode(); - let versioned_ccr = - VersionedCommittedCandidateReceipt::decode(&mut encoded_ccr.as_slice()).unwrap(); + let decoded_ccr: CommittedCandidateReceipt = + Decode::decode(&mut encoded_ccr.as_slice()).unwrap(); - assert_eq!(versioned_ccr.core_index(), Some(CoreIndex(123))); + assert_eq!(decoded_ccr.descriptor.relay_parent, new_ccr.descriptor.relay_parent()); + assert_eq!(decoded_ccr.descriptor.para_id, new_ccr.descriptor.para_id()); - if let VersionedCommittedCandidateReceipt::V2(ref v2_receipt) = versioned_ccr { - assert_eq!(v2_receipt.check(200, &claim_queue), Ok(())); - } + assert_eq!(new_ccr.hash(), decoded_ccr.hash()); + + // // // Encode v1 and decode as V2 + let encoded_ccr = new_ccr.encode(); + let v2_ccr: CommittedCandidateReceiptV2 = + Decode::decode(&mut encoded_ccr.as_slice()).unwrap(); + + assert_eq!(v2_ccr.descriptor.core_index(), Some(CoreIndex(123))); + assert_eq!(new_ccr.check(200, &claim_queue), Ok(())); + + assert_eq!(new_ccr.hash(), v2_ccr.hash()); + } - assert_eq!(new_ccr.hash(), versioned_ccr.to_plain().hash()); + fn dummy_collator_signature() -> CollatorSignature { + CollatorSignature::from_slice(&mut (0..64).into_iter().collect::>().as_slice()) + .expect("64 bytes; qed") + } + + fn dummy_collator_id() -> CollatorId { + CollatorId::from_slice(&mut (0..32).into_iter().collect::>().as_slice()) + .expect("32 bytes; qed") } #[test] - fn test_backward_compatible() { + fn test_core_select_is_mandatory() { // Testing edge case when collators provide zeroed signature and collator id. let mut old_ccr = dummy_old_committed_candidate_receipt(); old_ccr.descriptor.para_id = ParaId::new(1000); let encoded_ccr: Vec = old_ccr.encode(); - let versioned_ccr = - VersionedCommittedCandidateReceipt::decode(&mut encoded_ccr.as_slice()).unwrap(); + let new_ccr = CommittedCandidateReceiptV2::decode(&mut encoded_ccr.as_slice()).unwrap(); - // Since collator sig and id are zeroed, it was decoded as V2. - // We expect the check to fail in such case because there will be no `SelectCore` - // commitment. + // Since collator sig and id are zeroed, it means that the descriptor uses format + // version 2. let mut claim_queue = BTreeMap::new(); claim_queue.insert( - versioned_ccr.core_index().unwrap(), - vec![2.into(), versioned_ccr.para_id(), 3.into()].into(), + new_ccr.descriptor.core_index().unwrap(), + vec![2.into(), new_ccr.descriptor.para_id(), 3.into()].into(), ); - if let VersionedCommittedCandidateReceipt::V2(ref v2_receipt) = versioned_ccr { - assert_eq!( - v2_receipt.check(200, &claim_queue), - Err(CandidateReceiptError::NoCoreSelected) - ); - } else { - panic!("Should have decoded as V2") - } + // We expect the check to fail in such case because there will be no `SelectCore` + // commitment. + assert_eq!(new_ccr.check(200, &claim_queue), Err(CandidateReceiptError::NoCoreSelected)); // Adding collator signature should make it decode as v1. - old_ccr.descriptor.signature = - CollatorSignature::from_slice(&vec![99u8; 64]).expect("64 bytes; qed"); + old_ccr.descriptor.signature = dummy_collator_signature(); + old_ccr.descriptor.collator = dummy_collator_id(); + + let old_ccr_hash = old_ccr.hash(); + let encoded_ccr: Vec = old_ccr.encode(); - let versioned_ccr = - VersionedCommittedCandidateReceipt::decode(&mut encoded_ccr.as_slice()).unwrap(); - if let VersionedCommittedCandidateReceipt::V1(ref v1_receipt) = versioned_ccr { - assert_eq!(v1_receipt.descriptor.signature, old_ccr.descriptor.signature); - } else { - panic!("Should have decoded as V1") - } + let new_ccr = CommittedCandidateReceiptV2::decode(&mut encoded_ccr.as_slice()).unwrap(); + + assert_eq!(new_ccr.descriptor.signature(), Some(old_ccr.descriptor.signature)); + assert_eq!(new_ccr.descriptor.collator(), Some(old_ccr.descriptor.collator)); + + assert_eq!(new_ccr.descriptor.core_index(), None); + assert_eq!(new_ccr.descriptor.para_id(), ParaId::new(1000)); - assert_eq!(versioned_ccr.core_index(), None); - assert_eq!(versioned_ccr.para_id(), ParaId::new(1000)); - assert_eq!(old_ccr.hash(), versioned_ccr.to_plain().hash()); + assert_eq!(old_ccr_hash, new_ccr.hash()); } } diff --git a/polkadot/primitives/test-helpers/src/lib.rs b/polkadot/primitives/test-helpers/src/lib.rs index c262f16df077..cb644372f758 100644 --- a/polkadot/primitives/test-helpers/src/lib.rs +++ b/polkadot/primitives/test-helpers/src/lib.rs @@ -23,10 +23,7 @@ //! Note that `dummy_` prefixed values are meant to be fillers, that should not matter, and will //! contain randomness based data. use polkadot_primitives::{ - vstaging::{ - CandidateDescriptorV2, CandidateReceiptV2, CommittedCandidateReceiptV2, - VersionedCandidateReceipt, - }, + vstaging::{CandidateDescriptorV2, CandidateReceiptV2, CommittedCandidateReceiptV2}, CandidateCommitments, CandidateDescriptor, CandidateReceipt, CollatorId, CollatorSignature, CommittedCandidateReceipt, CoreIndex, Hash, HeadData, Id as ParaId, PersistedValidationData, SessionIndex, ValidationCode, ValidationCodeHash, ValidatorId, From 2831c5e0cb2f295609a16649a38babc66d36f409 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 8 Aug 2024 23:19:06 +0300 Subject: [PATCH 005/103] Add missing primitives and fix things Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 139 ++++++++++++++++++++++-- 1 file changed, 131 insertions(+), 8 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 03075e820524..f668b3a362bd 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -15,14 +15,14 @@ // along with Polkadot. If not, see . //! Staging Primitives. -use crate::ValidityAttestation; +use crate::{ValidatorIndex, ValidityAttestation}; // Put any primitives used by staging APIs functions here use super::{ Balance, BlakeTwo256, BlockNumber, CandidateCommitments, CandidateHash, CollatorId, CollatorSignature, CoreIndex, Hash, HashT, Header, Id, Id as ParaId, MultiDisputeStatementSet, UncheckedSignedAvailabilityBitfields, ValidationCodeHash, - ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, + ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, HeadData, GroupIndex, async_backing::Constraints, ScheduledCore }; use bitvec::prelude::*; use sp_application_crypto::ByteArray; @@ -158,6 +158,7 @@ pub struct CandidateDescriptorV2 { validation_code_hash: ValidationCodeHash, } + impl CandidateDescriptorV2 { /// Constructor pub fn new( @@ -208,6 +209,27 @@ pub struct CommittedCandidateReceiptV2 { pub commitments: CandidateCommitments, } + + +/// An even concerning a candidate. +#[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(PartialEq))] +pub enum CandidateEvent { + /// This candidate receipt was backed in the most recent block. + /// This includes the core index the candidate is now occupying. + #[codec(index = 0)] + CandidateBacked(CandidateReceiptV2, HeadData, CoreIndex, GroupIndex), + /// This candidate receipt was included and became a parablock at the most recent block. + /// This includes the core index the candidate was occupying as well as the group responsible + /// for backing the candidate. + #[codec(index = 1)] + CandidateIncluded(CandidateReceiptV2, HeadData, CoreIndex, GroupIndex), + /// This candidate receipt was not made available in time and timed out. + /// This includes the core index the candidate was occupying. + #[codec(index = 2)] + CandidateTimedOut(CandidateReceiptV2, HeadData, CoreIndex), +} + impl CandidateReceiptV2 { /// Get a reference to the candidate descriptor. pub fn descriptor(&self) -> &CandidateDescriptorV2 { @@ -336,16 +358,17 @@ pub enum CandidateReceiptError { macro_rules! impl_getter { ($field:ident, $type:ident) => { - fn $field(&self) -> $type { + /// Returns the value of $field field. + pub fn $field(&self) -> $type { self.$field } }; } -impl CandidateDescriptorV2 { +impl CandidateDescriptorV2 { impl_getter!(erasure_root, Hash); impl_getter!(para_head, Hash); - impl_getter!(relay_parent, Hash); + impl_getter!(relay_parent, H); impl_getter!(para_id, ParaId); impl_getter!(persisted_validation_data_hash, Hash); impl_getter!(pov_hash, Hash); @@ -454,18 +477,17 @@ impl CommittedCandidateReceiptV2 { None } }) - .cloned() .collect::>(); if assigned_cores.is_empty() { return Err(CandidateReceiptError::NoAssignment) } - let core_index = *assigned_cores + let core_index = assigned_cores .get(core_selector.0 as usize % assigned_cores.len()) .expect("provided index is always less than queue len; qed"); - if core_index != descriptor_core_index { + if **core_index != descriptor_core_index { return Err(CandidateReceiptError::CoreIndexMismatch) } @@ -523,6 +545,11 @@ impl BackedCandidate { &self.candidate } + /// Get a reference to the descriptor of the candidate. + pub fn descriptor(&self) -> &CandidateDescriptorV2 { + &self.candidate.descriptor + } + /// Get a reference to the validity votes of the candidate. pub fn validity_votes(&self) -> &[ValidityAttestation] { &self.validity_votes @@ -592,6 +619,102 @@ impl BackedCandidate { } } +/// Scraped runtime backing votes and resolved disputes. +#[derive(Clone, Encode, Decode, RuntimeDebug, TypeInfo)] +#[cfg_attr(feature = "std", derive(PartialEq))] +pub struct ScrapedOnChainVotes { + /// The session in which the block was included. + pub session: SessionIndex, + /// Set of backing validators for each candidate, represented by its candidate + /// receipt. + pub backing_validators_per_candidate: + Vec<(CandidateReceiptV2, Vec<(ValidatorIndex, ValidityAttestation)>)>, + /// On-chain-recorded set of disputes. + /// Note that the above `backing_validators` are + /// unrelated to the backers of the disputes candidates. + pub disputes: MultiDisputeStatementSet, +} + + +/// A candidate pending availability. +#[derive(RuntimeDebug, Clone, PartialEq, Encode, Decode, TypeInfo)] +pub struct CandidatePendingAvailability { + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// The candidate's descriptor. + pub descriptor: CandidateDescriptorV2, + /// The commitments of the candidate. + pub commitments: CandidateCommitments, + /// The candidate's relay parent's number. + pub relay_parent_number: N, + /// The maximum Proof-of-Validity size allowed, in bytes. + pub max_pov_size: u32, +} + +/// The per-parachain state of the backing system, including +/// state-machine constraints and candidates pending availability. +#[derive(RuntimeDebug, Clone, PartialEq, Encode, Decode, TypeInfo)] +pub struct BackingState { + /// The state-machine constraints of the parachain. + pub constraints: Constraints, + /// The candidates pending availability. These should be ordered, i.e. they should form + /// a sub-chain, where the first candidate builds on top of the required parent of the + /// constraints and each subsequent builds on top of the previous head-data. + pub pending_availability: Vec>, +} + + +/// Information about a core which is currently occupied. +#[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(PartialEq))] +pub struct OccupiedCore { + // NOTE: this has no ParaId as it can be deduced from the candidate descriptor. + /// If this core is freed by availability, this is the assignment that is next up on this + /// core, if any. None if there is nothing queued for this core. + pub next_up_on_available: Option, + /// The relay-chain block number this began occupying the core at. + pub occupied_since: N, + /// The relay-chain block this will time-out at, if any. + pub time_out_at: N, + /// If this core is freed by being timed-out, this is the assignment that is next up on this + /// core. None if there is nothing queued for this core or there is no possibility of timing + /// out. + pub next_up_on_time_out: Option, + /// A bitfield with 1 bit for each validator in the set. `1` bits mean that the corresponding + /// validators has attested to availability on-chain. A 2/3+ majority of `1` bits means that + /// this will be available. + pub availability: BitVec, + /// The group assigned to distribute availability pieces of this candidate. + pub group_responsible: GroupIndex, + /// The hash of the candidate occupying the core. + pub candidate_hash: CandidateHash, + /// The descriptor of the candidate occupying the core. + pub candidate_descriptor: CandidateDescriptorV2, +} + + +/// The state of a particular availability core. +#[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] +#[cfg_attr(feature = "std", derive(PartialEq))] +pub enum CoreState { + /// The core is currently occupied. + #[codec(index = 0)] + Occupied(OccupiedCore), + /// The core is currently free, with a para scheduled and given the opportunity + /// to occupy. + /// + /// If a particular Collator is required to author this block, that is also present in this + /// variant. + #[codec(index = 1)] + Scheduled(ScheduledCore), + /// The core is currently free and there is nothing scheduled. This can be the case for + /// parathread cores when there are no parathread blocks queued. Parachain cores will never be + /// left idle. + #[codec(index = 2)] + Free, +} + + #[cfg(test)] mod tests { use super::*; From c767d60da43a8d6aad953c32752bca772f6da494 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 8 Aug 2024 23:20:06 +0300 Subject: [PATCH 006/103] Implement v2 receipts in polkadot-runtime-parachains Signed-off-by: Andrei Sandu --- .../node/test/client/src/block_builder.rs | 2 +- polkadot/runtime/parachains/src/builder.rs | 4 +-- .../parachains/src/inclusion/migration.rs | 3 +- .../runtime/parachains/src/inclusion/mod.rs | 30 +++++++++---------- .../parachains/src/paras_inherent/mod.rs | 28 ++++++++--------- .../parachains/src/runtime_api_impl/v10.rs | 13 ++++---- .../src/runtime_api_impl/vstaging.rs | 2 +- 7 files changed, 41 insertions(+), 41 deletions(-) diff --git a/polkadot/node/test/client/src/block_builder.rs b/polkadot/node/test/client/src/block_builder.rs index 71bcdaffac4e..ed24cd55d084 100644 --- a/polkadot/node/test/client/src/block_builder.rs +++ b/polkadot/node/test/client/src/block_builder.rs @@ -16,7 +16,7 @@ use crate::Client; use codec::{Decode, Encode}; -use polkadot_primitives::{Block, InherentData as ParachainsInherentData}; +use polkadot_primitives::{Block, vstaging::InherentData as ParachainsInherentData}; use polkadot_test_runtime::UncheckedExtrinsic; use polkadot_test_service::GetLastTimestamp; use sc_block_builder::{BlockBuilder, BlockBuilderBuilder}; diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index b2a67ee8dd24..86473fafc8b0 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -31,9 +31,9 @@ use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; use polkadot_primitives::{ node_features::FeatureIndex, AvailabilityBitfield, BackedCandidate, CandidateCommitments, - CandidateDescriptor, CandidateHash, CollatorId, CollatorSignature, CommittedCandidateReceipt, + vstaging::CandidateDescriptorV2 as CandidateDescriptor, CandidateHash, CollatorId, CollatorSignature, vstaging::CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CompactStatement, CoreIndex, DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, - Id as ParaId, IndexedVec, InherentData as ParachainsInherentData, InvalidDisputeStatementKind, + Id as ParaId, IndexedVec, vstaging::InherentData as ParachainsInherentData, InvalidDisputeStatementKind, PersistedValidationData, SessionIndex, SigningContext, UncheckedSigned, ValidDisputeStatementKind, ValidationCode, ValidatorId, ValidatorIndex, ValidityAttestation, }; diff --git a/polkadot/runtime/parachains/src/inclusion/migration.rs b/polkadot/runtime/parachains/src/inclusion/migration.rs index 36a810d341c6..522d224d00d2 100644 --- a/polkadot/runtime/parachains/src/inclusion/migration.rs +++ b/polkadot/runtime/parachains/src/inclusion/migration.rs @@ -20,8 +20,7 @@ pub mod v0 { use frame_support::{storage_alias, Twox64Concat}; use frame_system::pallet_prelude::BlockNumberFor; use polkadot_primitives::{ - AvailabilityBitfield, CandidateCommitments, CandidateDescriptor, CandidateHash, CoreIndex, - GroupIndex, Id as ParaId, ValidatorIndex, + vstaging::CandidateDescriptorV2 as CandidateDescriptor, AvailabilityBitfield, CandidateCommitments, CandidateHash, CoreIndex, GroupIndex, Id as ParaId, ValidatorIndex }; use scale_info::TypeInfo; diff --git a/polkadot/runtime/parachains/src/inclusion/mod.rs b/polkadot/runtime/parachains/src/inclusion/mod.rs index 115eee975530..75b80d06fc18 100644 --- a/polkadot/runtime/parachains/src/inclusion/mod.rs +++ b/polkadot/runtime/parachains/src/inclusion/mod.rs @@ -45,9 +45,9 @@ use frame_support::{ use frame_system::pallet_prelude::*; use pallet_message_queue::OnQueueChanged; use polkadot_primitives::{ - effective_minimum_backing_votes, supermajority_threshold, well_known_keys, BackedCandidate, - CandidateCommitments, CandidateDescriptor, CandidateHash, CandidateReceipt, - CommittedCandidateReceipt, CoreIndex, GroupIndex, Hash, HeadData, Id as ParaId, + effective_minimum_backing_votes, supermajority_threshold, well_known_keys, vstaging::BackedCandidate, + CandidateCommitments, vstaging::CandidateDescriptorV2 as CandidateDescriptor, CandidateHash, vstaging::CandidateReceiptV2 as CandidateReceipt, + vstaging::CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreIndex, GroupIndex, Hash, HeadData, Id as ParaId, SignedAvailabilityBitfields, SigningContext, UpwardMessage, ValidatorId, ValidatorIndex, ValidityAttestation, }; @@ -749,7 +749,7 @@ impl Pallet { let mut backers = bitvec::bitvec![u8, BitOrderLsb0; 0; validators.len()]; let signing_context = SigningContext { - parent_hash: backed_candidate.descriptor().relay_parent, + parent_hash: backed_candidate.descriptor().relay_parent(), session_index: shared::CurrentSessionIndex::::get(), }; @@ -870,7 +870,7 @@ impl Pallet { let now = frame_system::Pallet::::block_number(); weight.saturating_add(paras::Pallet::::schedule_code_upgrade( - receipt.descriptor.para_id, + receipt.descriptor.para_id(), new_code, now, &config, @@ -880,19 +880,19 @@ impl Pallet { // enact the messaging facet of the candidate. weight.saturating_accrue(dmp::Pallet::::prune_dmq( - receipt.descriptor.para_id, + receipt.descriptor.para_id(), commitments.processed_downward_messages, )); weight.saturating_accrue(Self::receive_upward_messages( - receipt.descriptor.para_id, + receipt.descriptor.para_id(), commitments.upward_messages.as_slice(), )); weight.saturating_accrue(hrmp::Pallet::::prune_hrmp( - receipt.descriptor.para_id, + receipt.descriptor.para_id(), BlockNumberFor::::from(commitments.hrmp_watermark), )); weight.saturating_accrue(hrmp::Pallet::::queue_outbound_hrmp( - receipt.descriptor.para_id, + receipt.descriptor.para_id(), commitments.horizontal_messages, )); @@ -904,7 +904,7 @@ impl Pallet { )); weight.saturating_add(paras::Pallet::::note_new_head( - receipt.descriptor.para_id, + receipt.descriptor.para_id(), commitments.head_data, relay_parent_number, )) @@ -1242,8 +1242,8 @@ impl CandidateCheckContext { backed_candidate_receipt: &CommittedCandidateReceipt<::Hash>, parent_head_data: HeadData, ) -> Result, Error> { - let para_id = backed_candidate_receipt.descriptor().para_id; - let relay_parent = backed_candidate_receipt.descriptor().relay_parent; + let para_id = backed_candidate_receipt.descriptor.para_id(); + let relay_parent = backed_candidate_receipt.descriptor.relay_parent(); // Check that the relay-parent is one of the allowed relay-parents. let (relay_parent_storage_root, relay_parent_number) = { @@ -1263,7 +1263,7 @@ impl CandidateCheckContext { let expected = persisted_validation_data.hash(); ensure!( - expected == backed_candidate_receipt.descriptor().persisted_validation_data_hash, + expected == backed_candidate_receipt.descriptor.persisted_validation_data_hash(), Error::::ValidationDataHashMismatch, ); } @@ -1272,12 +1272,12 @@ impl CandidateCheckContext { // A candidate for a parachain without current validation code is not scheduled. .ok_or_else(|| Error::::UnscheduledCandidate)?; ensure!( - backed_candidate_receipt.descriptor().validation_code_hash == validation_code_hash, + backed_candidate_receipt.descriptor.validation_code_hash() == validation_code_hash, Error::::InvalidValidationCodeHash, ); ensure!( - backed_candidate_receipt.descriptor().para_head == + backed_candidate_receipt.descriptor.para_head() == backed_candidate_receipt.commitments.head_data.hash(), Error::::ParaHeadMismatch, ); diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 9d27e86ef901..de616ea763b9 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -48,10 +48,10 @@ use frame_support::{ use frame_system::pallet_prelude::*; use pallet_babe::{self, ParentBlockRandomness}; use polkadot_primitives::{ - effective_minimum_backing_votes, node_features::FeatureIndex, BackedCandidate, CandidateHash, - CandidateReceipt, CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, CoreIndex, - DisputeStatementSet, HeadData, InherentData as ParachainsInherentData, - MultiDisputeStatementSet, ScrapedOnChainVotes, SessionIndex, SignedAvailabilityBitfields, + effective_minimum_backing_votes, node_features::FeatureIndex, vstaging::BackedCandidate, CandidateHash, + vstaging::CandidateReceiptV2 as CandidateReceipt, CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, CoreIndex, + DisputeStatementSet, HeadData, vstaging::InherentData as ParachainsInherentData, + MultiDisputeStatementSet, vstaging::ScrapedOnChainVotes, SessionIndex, SignedAvailabilityBitfields, SigningContext, UncheckedSignedAvailabilityBitfield, UncheckedSignedAvailabilityBitfields, ValidatorId, ValidatorIndex, ValidityAttestation, PARACHAINS_INHERENT_IDENTIFIER, }; @@ -763,7 +763,7 @@ pub(crate) fn apply_weight_limit( let mut current_para_id = None; for candidate in core::mem::take(candidates).into_iter() { - let candidate_para_id = candidate.descriptor().para_id; + let candidate_para_id = candidate.descriptor().para_id(); if Some(candidate_para_id) == current_para_id { let chain = chained_candidates .last_mut() @@ -965,7 +965,7 @@ fn sanitize_backed_candidates( let mut candidates_per_para: BTreeMap> = BTreeMap::new(); for candidate in backed_candidates { candidates_per_para - .entry(candidate.descriptor().para_id) + .entry(candidate.descriptor().para_id()) .or_default() .push(candidate); } @@ -984,7 +984,7 @@ fn sanitize_backed_candidates( target: LOG_TARGET, "Found backed candidate {:?} which was concluded invalid or is a descendant of a concluded invalid candidate, for paraid {:?}.", candidate.candidate().hash(), - candidate.descriptor().para_id + candidate.descriptor().para_id() ); } keep @@ -1165,13 +1165,13 @@ fn filter_backed_statements_from_disabled_validators< // Get relay parent block number of the candidate. We need this to get the group index // assigned to this core at this block number let relay_parent_block_number = - match allowed_relay_parents.acquire_info(bc.descriptor().relay_parent, None) { + match allowed_relay_parents.acquire_info(bc.descriptor().relay_parent(), None) { Some((_, block_num)) => block_num, None => { log::debug!( target: LOG_TARGET, "Relay parent {:?} for candidate is not in the allowed relay parents. Dropping the candidate.", - bc.descriptor().relay_parent + bc.descriptor().relay_parent() ); return false }, @@ -1372,7 +1372,7 @@ fn map_candidates_to_cores block_num, None => { log::debug!( target: LOG_TARGET, "Relay parent {:?} for candidate {:?} is not in the allowed relay parents.", - candidate.descriptor().relay_parent, + candidate.descriptor().relay_parent(), candidate.candidate().hash(), ); return None diff --git a/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs b/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs index 697890232211..f0107d6f21e2 100644 --- a/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs +++ b/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs @@ -27,14 +27,15 @@ use frame_support::traits::{GetStorageVersion, StorageVersion}; use frame_system::pallet_prelude::*; use polkadot_primitives::{ async_backing::{ - AsyncBackingParams, BackingState, CandidatePendingAvailability, Constraints, - InboundHrmpLimitations, OutboundHrmpChannelLimitations, + AsyncBackingParams, Constraints, + InboundHrmpLimitations, OutboundHrmpChannelLimitations, }, - slashing, ApprovalVotingParams, AuthorityDiscoveryId, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreIndex, CoreState, DisputeState, ExecutorParams, GroupIndex, + vstaging::{CoreState,OccupiedCore, BackingState, CandidateEvent, ScrapedOnChainVotes, CandidatePendingAvailability, CommittedCandidateReceiptV2 as CommittedCandidateReceipt}, + slashing, ApprovalVotingParams, AuthorityDiscoveryId, CandidateHash, + CoreIndex, DisputeState, ExecutorParams, GroupIndex, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, - NodeFeatures, OccupiedCore, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, - ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, + NodeFeatures, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, + SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_runtime::traits::One; diff --git a/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs b/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs index 4aa381e33b1b..f110099d968c 100644 --- a/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs +++ b/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs @@ -21,7 +21,7 @@ use alloc::{ collections::{btree_map::BTreeMap, vec_deque::VecDeque}, vec::Vec, }; -use polkadot_primitives::{CommittedCandidateReceipt, CoreIndex, Id as ParaId}; +use polkadot_primitives::{vstaging::CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreIndex, Id as ParaId}; use sp_runtime::traits::One; /// Returns the claimqueue from the scheduler From 96999e31b96f60954bf29f4e7f2db03aa579f0a5 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 12 Aug 2024 15:04:28 +0300 Subject: [PATCH 007/103] add missing stuff Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 176 ++++++++++++++++++++---- 1 file changed, 153 insertions(+), 23 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index f668b3a362bd..9541484d5028 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -19,16 +19,20 @@ use crate::{ValidatorIndex, ValidityAttestation}; // Put any primitives used by staging APIs functions here use super::{ - Balance, BlakeTwo256, BlockNumber, CandidateCommitments, CandidateHash, - CollatorId, CollatorSignature, CoreIndex, Hash, HashT, Header, Id, Id as ParaId, - MultiDisputeStatementSet, UncheckedSignedAvailabilityBitfields, ValidationCodeHash, - ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, HeadData, GroupIndex, async_backing::Constraints, ScheduledCore + async_backing::Constraints, Balance, BlakeTwo256, BlockNumber, CandidateCommitments, + CandidateDescriptor, CandidateHash, CollatorId, CollatorSignature, CoreIndex, GroupIndex, Hash, + HashT, HeadData, Header, Id, Id as ParaId, MultiDisputeStatementSet, ScheduledCore, + UncheckedSignedAvailabilityBitfields, ValidationCodeHash, ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, }; use bitvec::prelude::*; use sp_application_crypto::ByteArray; -use alloc::collections::{btree_map::BTreeMap, vec_deque::VecDeque}; -use codec::{Decode, Encode}; +use alloc::{ + collections::{btree_map::BTreeMap, vec_deque::VecDeque}, + vec, + vec::Vec, +}; +use codec::{Decode, Encode, WrapperTypeDecode}; use scale_info::TypeInfo; use sp_arithmetic::Perbill; use sp_core::RuntimeDebug; @@ -158,6 +162,21 @@ pub struct CandidateDescriptorV2 { validation_code_hash: ValidationCodeHash, } +impl From> for CandidateDescriptor { + fn from(value: CandidateDescriptorV2) -> Self { + Self { + para_id: value.para_id, + relay_parent: value.relay_parent, + collator: value.rebuild_collator_field(), + persisted_validation_data_hash: value.persisted_validation_data_hash, + pov_hash: value.pov_hash, + erasure_root: value.erasure_root, + signature: value.rebuild_signature_field(), + para_head: value.para_head, + validation_code_hash: value.validation_code_hash, + } + } +} impl CandidateDescriptorV2 { /// Constructor @@ -187,6 +206,11 @@ impl CandidateDescriptorV2 { validation_code_hash, } } + + #[cfg(feature = "test")] + pub fn set_pov_hash(&mut self, pov_hash: Hash) { + self.pov_hash = pov_hash; + } } /// A candidate-receipt at version 2. @@ -209,8 +233,6 @@ pub struct CommittedCandidateReceiptV2 { pub commitments: CandidateCommitments, } - - /// An even concerning a candidate. #[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] #[cfg_attr(feature = "std", derive(PartialEq))] @@ -230,6 +252,29 @@ pub enum CandidateEvent { CandidateTimedOut(CandidateReceiptV2, HeadData, CoreIndex), } +impl From> for super::v7::CandidateEvent { + fn from(value: CandidateEvent) -> Self { + match value { + CandidateEvent::CandidateBacked(receipt, head_data, core_index, group_index) => + super::v7::CandidateEvent::CandidateBacked( + receipt.into(), + head_data, + core_index, + group_index, + ), + CandidateEvent::CandidateIncluded(receipt, head_data, core_index, group_index) => + super::v7::CandidateEvent::CandidateIncluded( + receipt.into(), + head_data, + core_index, + group_index, + ), + CandidateEvent::CandidateTimedOut(receipt, head_data, core_index) => + super::v7::CandidateEvent::CandidateTimedOut(receipt.into(), head_data, core_index), + } + } +} + impl CandidateReceiptV2 { /// Get a reference to the candidate descriptor. pub fn descriptor(&self) -> &CandidateDescriptorV2 { @@ -289,6 +334,18 @@ impl Ord for CommittedCandidateReceiptV2 { } } +impl From> for super::v7::CommittedCandidateReceipt { + fn from(value: CommittedCandidateReceiptV2) -> Self { + Self { descriptor: value.descriptor.into(), commitments: value.commitments } + } +} + +impl From> for super::v7::CandidateReceipt { + fn from(value: CandidateReceiptV2) -> Self { + Self { descriptor: value.descriptor.into(), commitments_hash: value.commitments_hash } + } +} + /// A strictly increasing sequence number, tipically this would be the parachain block number. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] pub struct CoreSelector(pub BlockNumber); @@ -388,27 +445,36 @@ impl CandidateDescriptorV2 { } } + fn rebuild_collator_field(&self) -> CollatorId { + let mut collator_id = vec![self.version.0]; + let core_index: [u8; 2] = unsafe { core::mem::transmute(self.core_index) }; + let session_index: [u8; 4] = unsafe { core::mem::transmute(self.session_index) }; + + collator_id.append(&mut core_index.as_slice().to_vec()); + collator_id.append(&mut session_index.as_slice().to_vec()); + collator_id.append(&mut self.reserved25b.as_slice().to_vec()); + + CollatorId::from_slice(&collator_id.as_slice()).expect("Slice size is exactly 32 bytes") + } + /// Returns the collator id if this is a v1 `CandidateDescriptor` pub fn collator(&self) -> Option { if self.version() == CandidateDescriptorVersion::V1 { - let mut collator_id = vec![self.version.0]; - let core_index: [u8; 2] = unsafe { std::mem::transmute(self.core_index) }; - let session_index: [u8; 4] = unsafe { std::mem::transmute(self.session_index) }; - - collator_id.append(&mut core_index.as_slice().to_vec()); - collator_id.append(&mut session_index.as_slice().to_vec()); - collator_id.append(&mut self.reserved25b.as_slice().to_vec()); - - return Some(CollatorId::from_slice(&collator_id.as_slice()).ok()?) + Some(self.rebuild_collator_field()) + } else { + None } + } - None + fn rebuild_signature_field(&self) -> CollatorSignature { + CollatorSignature::from_slice(self.reserved64b.as_slice()) + .expect("Slice size is exactly 64 bytes") } /// Returns the collator signature of `V1` candidate descriptors, `None` otherwise. pub fn signature(&self) -> Option { if self.version() == CandidateDescriptorVersion::V1 { - return Some(CollatorSignature::from_slice(self.reserved64b.as_slice()).ok()?) + return Some(self.rebuild_signature_field()) } None @@ -635,6 +701,19 @@ pub struct ScrapedOnChainVotes { pub disputes: MultiDisputeStatementSet, } +impl From> for super::v7::ScrapedOnChainVotes { + fn from(value: ScrapedOnChainVotes) -> Self { + Self { + session: value.session, + backing_validators_per_candidate: value + .backing_validators_per_candidate + .into_iter() + .map(|(receipt, validators)| (receipt.into(), validators)) + .collect::>(), + disputes: value.disputes, + } + } +} /// A candidate pending availability. #[derive(RuntimeDebug, Clone, PartialEq, Encode, Decode, TypeInfo)] @@ -651,6 +730,20 @@ pub struct CandidatePendingAvailability { pub max_pov_size: u32, } +impl From> + for super::async_backing::CandidatePendingAvailability +{ + fn from(value: CandidatePendingAvailability) -> Self { + Self { + candidate_hash: value.candidate_hash, + descriptor: value.descriptor.into(), + commitments: value.commitments, + relay_parent_number: value.relay_parent_number, + max_pov_size: value.max_pov_size, + } + } +} + /// The per-parachain state of the backing system, including /// state-machine constraints and candidates pending availability. #[derive(RuntimeDebug, Clone, PartialEq, Encode, Decode, TypeInfo)] @@ -663,7 +756,18 @@ pub struct BackingState { pub pending_availability: Vec>, } - +impl From> for super::async_backing::BackingState { + fn from(value: BackingState) -> Self { + Self { + constraints: value.constraints, + pending_availability: value + .pending_availability + .into_iter() + .map(|candidate| candidate.into()) + .collect::>(), + } + } +} /// Information about a core which is currently occupied. #[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] #[cfg_attr(feature = "std", derive(PartialEq))] @@ -692,7 +796,6 @@ pub struct OccupiedCore { pub candidate_descriptor: CandidateDescriptorV2, } - /// The state of a particular availability core. #[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] #[cfg_attr(feature = "std", derive(PartialEq))] @@ -714,6 +817,31 @@ pub enum CoreState { Free, } +impl From> for super::v7::OccupiedCore { + fn from(value: OccupiedCore) -> Self { + Self { + next_up_on_available: value.next_up_on_available, + occupied_since: value.occupied_since, + time_out_at: value.time_out_at, + next_up_on_time_out: value.next_up_on_time_out, + availability: value.availability, + group_responsible: value.group_responsible, + candidate_hash: value.candidate_hash, + candidate_descriptor: value.candidate_descriptor.into(), + } + } +} + +impl From> for super::v7::CoreState { + fn from(value: CoreState) -> Self { + match value { + CoreState::Free => super::v7::CoreState::Free, + CoreState::Scheduled(core) => super::v7::CoreState::Scheduled(core), + CoreState::Occupied(occupied_core) => + super::v7::CoreState::Occupied(occupied_core.into()), + } + } +} #[cfg(test)] mod tests { @@ -863,7 +991,8 @@ mod tests { old_ccr.descriptor.para_id = ParaId::new(1000); let encoded_ccr: Vec = old_ccr.encode(); - let new_ccr = CommittedCandidateReceiptV2::decode(&mut encoded_ccr.as_slice()).unwrap(); + let new_ccr: CommittedCandidateReceiptV2 = + Decode::decode(&mut encoded_ccr.as_slice()).unwrap(); // Since collator sig and id are zeroed, it means that the descriptor uses format // version 2. @@ -885,7 +1014,8 @@ mod tests { let encoded_ccr: Vec = old_ccr.encode(); - let new_ccr = CommittedCandidateReceiptV2::decode(&mut encoded_ccr.as_slice()).unwrap(); + let new_ccr: CommittedCandidateReceiptV2 = + Decode::decode(&mut encoded_ccr.as_slice()).unwrap(); assert_eq!(new_ccr.descriptor.signature(), Some(old_ccr.descriptor.signature)); assert_eq!(new_ccr.descriptor.collator(), Some(old_ccr.descriptor.collator)); From c5f2dc3237809d6094a3ace158ebde1f9cd1678b Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 12 Aug 2024 15:05:18 +0300 Subject: [PATCH 008/103] Switch parachains runtime to use new primitives Signed-off-by: Andrei Sandu --- .../parachains/src/runtime_api_impl/v10.rs | 21 +++++++++++-------- .../src/runtime_api_impl/vstaging.rs | 4 +++- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs b/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs index f0107d6f21e2..608db7434b87 100644 --- a/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs +++ b/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs @@ -27,16 +27,19 @@ use frame_support::traits::{GetStorageVersion, StorageVersion}; use frame_system::pallet_prelude::*; use polkadot_primitives::{ async_backing::{ - AsyncBackingParams, Constraints, - InboundHrmpLimitations, OutboundHrmpChannelLimitations, + AsyncBackingParams, Constraints, InboundHrmpLimitations, OutboundHrmpChannelLimitations, }, - vstaging::{CoreState,OccupiedCore, BackingState, CandidateEvent, ScrapedOnChainVotes, CandidatePendingAvailability, CommittedCandidateReceiptV2 as CommittedCandidateReceipt}, - slashing, ApprovalVotingParams, AuthorityDiscoveryId, CandidateHash, - CoreIndex, DisputeState, ExecutorParams, GroupIndex, - GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, - NodeFeatures, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, - SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, - ValidatorId, ValidatorIndex, ValidatorSignature, + slashing, + vstaging::{ + BackingState, CandidateEvent, CandidatePendingAvailability, + CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreState, OccupiedCore, + ScrapedOnChainVotes, + }, + ApprovalVotingParams, AuthorityDiscoveryId, CandidateHash, CoreIndex, DisputeState, + ExecutorParams, GroupIndex, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, NodeFeatures, OccupiedCoreAssumption, PersistedValidationData, + PvfCheckStatement, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, ValidatorId, + ValidatorIndex, ValidatorSignature, }; use sp_runtime::traits::One; diff --git a/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs b/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs index f110099d968c..a3440f686e94 100644 --- a/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs +++ b/polkadot/runtime/parachains/src/runtime_api_impl/vstaging.rs @@ -21,7 +21,9 @@ use alloc::{ collections::{btree_map::BTreeMap, vec_deque::VecDeque}, vec::Vec, }; -use polkadot_primitives::{vstaging::CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreIndex, Id as ParaId}; +use polkadot_primitives::{ + vstaging::CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreIndex, Id as ParaId, +}; use sp_runtime::traits::One; /// Returns the claimqueue from the scheduler From dbb0160392fa013acbc6560e4b5737c1476d6ce6 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 12 Aug 2024 15:05:58 +0300 Subject: [PATCH 009/103] use vstaging primitives Signed-off-by: Andrei Sandu --- polkadot/primitives/Cargo.toml | 1 + polkadot/primitives/src/runtime_api.rs | 16 ++-- polkadot/runtime/parachains/Cargo.toml | 4 +- polkadot/runtime/parachains/src/builder.rs | 49 ++++++----- .../parachains/src/inclusion/migration.rs | 10 ++- .../runtime/parachains/src/inclusion/mod.rs | 14 +-- .../runtime/parachains/src/inclusion/tests.rs | 88 ++++--------------- .../parachains/src/paras_inherent/mod.rs | 17 ++-- .../parachains/src/paras_inherent/tests.rs | 68 ++------------ 9 files changed, 95 insertions(+), 172 deletions(-) diff --git a/polkadot/primitives/Cargo.toml b/polkadot/primitives/Cargo.toml index 09afc9662d62..311fc210d42b 100644 --- a/polkadot/primitives/Cargo.toml +++ b/polkadot/primitives/Cargo.toml @@ -64,3 +64,4 @@ runtime-benchmarks = [ "sp-runtime/runtime-benchmarks", "sp-staking/runtime-benchmarks", ] +test = [] diff --git a/polkadot/primitives/src/runtime_api.rs b/polkadot/primitives/src/runtime_api.rs index b4816ad15075..6412a67b460d 100644 --- a/polkadot/primitives/src/runtime_api.rs +++ b/polkadot/primitives/src/runtime_api.rs @@ -114,11 +114,15 @@ //! separated from the stable primitives. use crate::{ - async_backing, slashing, ApprovalVotingParams, AsyncBackingParams, BlockNumber, - CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreIndex, - CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash, NodeFeatures, - OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, - SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, ValidatorSignature, + slashing, + vstaging::{ + self, CandidateEvent, CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreState, + ScrapedOnChainVotes, + }, + ApprovalVotingParams, AsyncBackingParams, BlockNumber, CandidateCommitments, CandidateHash, + CoreIndex, DisputeState, ExecutorParams, GroupRotationInfo, Hash, NodeFeatures, + OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, SessionIndex, SessionInfo, + ValidatorId, ValidatorIndex, ValidatorSignature, }; use alloc::{ @@ -260,7 +264,7 @@ sp_api::decl_runtime_apis! { /// Returns the state of parachain backing for a given para. #[api_version(7)] - fn para_backing_state(_: ppp::Id) -> Option>; + fn para_backing_state(_: ppp::Id) -> Option>; /// Returns candidate's acceptance limitations for asynchronous backing for a relay parent. #[api_version(7)] diff --git a/polkadot/runtime/parachains/Cargo.toml b/polkadot/runtime/parachains/Cargo.toml index cfe373e8cba2..59af889fe6e7 100644 --- a/polkadot/runtime/parachains/Cargo.toml +++ b/polkadot/runtime/parachains/Cargo.toml @@ -49,7 +49,7 @@ frame-system = { workspace = true } xcm = { workspace = true } xcm-executor = { workspace = true } -polkadot-primitives = { workspace = true } +polkadot-primitives = { workspace = true, features=["test"] } rand = { workspace = true } rand_chacha = { workspace = true } @@ -59,6 +59,8 @@ polkadot-runtime-metrics = { workspace = true } polkadot-core-primitives = { workspace = true } [dev-dependencies] +polkadot-primitives = { workspace = true } + futures = { workspace = true } hex-literal = { workspace = true, default-features = true } sp-keyring = { workspace = true, default-features = true } diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 86473fafc8b0..c1e7844de470 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -30,12 +30,17 @@ use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; use polkadot_primitives::{ - node_features::FeatureIndex, AvailabilityBitfield, BackedCandidate, CandidateCommitments, - vstaging::CandidateDescriptorV2 as CandidateDescriptor, CandidateHash, CollatorId, CollatorSignature, vstaging::CommittedCandidateReceiptV2 as CommittedCandidateReceipt, + node_features::FeatureIndex, + vstaging::{ + BackedCandidate, CandidateDescriptorV2 as CandidateDescriptor, + CommittedCandidateReceiptV2 as CommittedCandidateReceipt, + InherentData as ParachainsInherentData, + }, + AvailabilityBitfield, CandidateCommitments, CandidateHash, CollatorId, CollatorSignature, CompactStatement, CoreIndex, DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, - Id as ParaId, IndexedVec, vstaging::InherentData as ParachainsInherentData, InvalidDisputeStatementKind, - PersistedValidationData, SessionIndex, SigningContext, UncheckedSigned, - ValidDisputeStatementKind, ValidationCode, ValidatorId, ValidatorIndex, ValidityAttestation, + Id as ParaId, IndexedVec, InvalidDisputeStatementKind, PersistedValidationData, SessionIndex, + SigningContext, UncheckedSigned, ValidDisputeStatementKind, ValidationCode, ValidatorId, + ValidatorIndex, ValidityAttestation, }; use sp_core::{sr25519, ByteArray, H256}; use sp_runtime::{ @@ -295,17 +300,17 @@ impl BenchBuilder { } fn candidate_descriptor_mock() -> CandidateDescriptor { - CandidateDescriptor:: { - para_id: 0.into(), - relay_parent: Default::default(), - collator: CollatorId::from(sr25519::Public::from_raw([42u8; 32])), - persisted_validation_data_hash: Default::default(), - pov_hash: Default::default(), - erasure_root: Default::default(), - signature: CollatorSignature::from(sr25519::Signature::from_raw([42u8; 64])), - para_head: Default::default(), - validation_code_hash: mock_validation_code().hash(), - } + CandidateDescriptor::::new( + 0.into(), + Default::default(), + CoreIndex(0), + 1, + Default::default(), + Default::default(), + Default::default(), + Default::default(), + mock_validation_code().hash(), + ) } /// Create a mock of `CandidatePendingAvailability`. @@ -631,17 +636,17 @@ impl BenchBuilder { scheduler::Pallet::::group_validators(group_idx).unwrap(); let candidate = CommittedCandidateReceipt:: { - descriptor: CandidateDescriptor:: { + descriptor: CandidateDescriptor::::new( para_id, relay_parent, - collator: dummy_collator(), + CoreIndex(0), + 1, persisted_validation_data_hash, pov_hash, - erasure_root: Default::default(), - signature: dummy_collator_signature(), - para_head: head_data.hash(), + Default::default(), + head_data.hash(), validation_code_hash, - }, + ), commitments: CandidateCommitments:: { upward_messages: Default::default(), horizontal_messages: Default::default(), diff --git a/polkadot/runtime/parachains/src/inclusion/migration.rs b/polkadot/runtime/parachains/src/inclusion/migration.rs index 522d224d00d2..143c4e7d629a 100644 --- a/polkadot/runtime/parachains/src/inclusion/migration.rs +++ b/polkadot/runtime/parachains/src/inclusion/migration.rs @@ -20,7 +20,8 @@ pub mod v0 { use frame_support::{storage_alias, Twox64Concat}; use frame_system::pallet_prelude::BlockNumberFor; use polkadot_primitives::{ - vstaging::CandidateDescriptorV2 as CandidateDescriptor, AvailabilityBitfield, CandidateCommitments, CandidateHash, CoreIndex, GroupIndex, Id as ParaId, ValidatorIndex + vstaging::CandidateDescriptorV2 as CandidateDescriptor, AvailabilityBitfield, + CandidateCommitments, CandidateHash, CoreIndex, GroupIndex, Id as ParaId, ValidatorIndex, }; use scale_info::TypeInfo; @@ -218,7 +219,8 @@ mod tests { use frame_support::traits::UncheckedOnRuntimeUpgrade; use polkadot_primitives::{AvailabilityBitfield, Id as ParaId}; use polkadot_primitives_test_helpers::{ - dummy_candidate_commitments, dummy_candidate_descriptor, dummy_hash, + dummy_candidate_commitments, dummy_candidate_descriptor, dummy_candidate_descriptor_v2, + dummy_hash, }; #[test] @@ -234,7 +236,7 @@ mod tests { let mut expected = vec![]; for i in 1..5 { - let descriptor = dummy_candidate_descriptor(dummy_hash()); + let descriptor = dummy_candidate_descriptor_v2(dummy_hash()); v0::PendingAvailability::::insert( ParaId::from(i), v0::CandidatePendingAvailability { @@ -284,7 +286,7 @@ mod tests { ParaId::from(6), v0::CandidatePendingAvailability { core: CoreIndex(6), - descriptor: dummy_candidate_descriptor(dummy_hash()), + descriptor: dummy_candidate_descriptor_v2(dummy_hash()), relay_parent_number: 6, hash: CandidateHash(dummy_hash()), availability_votes: Default::default(), diff --git a/polkadot/runtime/parachains/src/inclusion/mod.rs b/polkadot/runtime/parachains/src/inclusion/mod.rs index 75b80d06fc18..d3edad7b3d6c 100644 --- a/polkadot/runtime/parachains/src/inclusion/mod.rs +++ b/polkadot/runtime/parachains/src/inclusion/mod.rs @@ -45,11 +45,15 @@ use frame_support::{ use frame_system::pallet_prelude::*; use pallet_message_queue::OnQueueChanged; use polkadot_primitives::{ - effective_minimum_backing_votes, supermajority_threshold, well_known_keys, vstaging::BackedCandidate, - CandidateCommitments, vstaging::CandidateDescriptorV2 as CandidateDescriptor, CandidateHash, vstaging::CandidateReceiptV2 as CandidateReceipt, - vstaging::CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreIndex, GroupIndex, Hash, HeadData, Id as ParaId, - SignedAvailabilityBitfields, SigningContext, UpwardMessage, ValidatorId, ValidatorIndex, - ValidityAttestation, + effective_minimum_backing_votes, supermajority_threshold, + vstaging::{ + BackedCandidate, CandidateDescriptorV2 as CandidateDescriptor, + CandidateReceiptV2 as CandidateReceipt, + CommittedCandidateReceiptV2 as CommittedCandidateReceipt, + }, + well_known_keys, CandidateCommitments, CandidateHash, CoreIndex, GroupIndex, Hash, HeadData, + Id as ParaId, SignedAvailabilityBitfields, SigningContext, UpwardMessage, ValidatorId, + ValidatorIndex, ValidityAttestation, }; use scale_info::TypeInfo; use sp_runtime::{traits::One, DispatchError, SaturatedConversion, Saturating}; diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index 3ead456cde5a..bfb2f228c452 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -26,21 +26,19 @@ use crate::{ shared::AllowedRelayParentsTracker, }; use polkadot_primitives::{ - effective_minimum_backing_votes, AvailabilityBitfield, SignedAvailabilityBitfields, - UncheckedSignedAvailabilityBitfields, + effective_minimum_backing_votes, vstaging::CandidateDescriptorV2, AvailabilityBitfield, + SignedAvailabilityBitfields, UncheckedSignedAvailabilityBitfields, }; use assert_matches::assert_matches; use codec::DecodeAll; use frame_support::assert_noop; use polkadot_primitives::{ - BlockNumber, CandidateCommitments, CandidateDescriptor, CollatorId, - CompactStatement as Statement, Hash, SignedAvailabilityBitfield, SignedStatement, - ValidationCode, ValidatorId, ValidityAttestation, PARACHAIN_KEY_TYPE_ID, -}; -use polkadot_primitives_test_helpers::{ - dummy_collator, dummy_collator_signature, dummy_validation_code, + BlockNumber, CandidateCommitments, CollatorId, CompactStatement as Statement, Hash, + SignedAvailabilityBitfield, SignedStatement, ValidationCode, ValidatorId, ValidityAttestation, + PARACHAIN_KEY_TYPE_ID, }; +use polkadot_primitives_test_helpers::dummy_validation_code; use sc_keystore::LocalKeystore; use sp_keyring::Sr25519Keyring; use sp_keystore::{Keystore, KeystorePtr}; @@ -96,24 +94,6 @@ pub(crate) enum BackingKind { Lacking, } -pub(crate) fn collator_sign_candidate( - collator: Sr25519Keyring, - candidate: &mut CommittedCandidateReceipt, -) { - candidate.descriptor.collator = collator.public().into(); - - let payload = polkadot_primitives::collator_signature_payload( - &candidate.descriptor.relay_parent, - &candidate.descriptor.para_id, - &candidate.descriptor.persisted_validation_data_hash, - &candidate.descriptor.pov_hash, - &candidate.descriptor.validation_code_hash, - ); - - candidate.descriptor.signature = collator.sign(&payload[..]).into(); - assert!(candidate.descriptor().check_collator_signature().is_ok()); -} - pub(crate) fn back_candidate( candidate: CommittedCandidateReceipt, validators: &[Sr25519Keyring], @@ -303,17 +283,17 @@ impl std::default::Default for TestCandidateBuilder { impl TestCandidateBuilder { pub(crate) fn build(self) -> CommittedCandidateReceipt { CommittedCandidateReceipt { - descriptor: CandidateDescriptor { - para_id: self.para_id, - pov_hash: self.pov_hash, - relay_parent: self.relay_parent, - persisted_validation_data_hash: self.persisted_validation_data_hash, - validation_code_hash: self.validation_code.hash(), - para_head: self.para_head_hash.unwrap_or_else(|| self.head_data.hash()), - erasure_root: Default::default(), - signature: dummy_collator_signature(), - collator: dummy_collator(), - }, + descriptor: CandidateDescriptorV2::new( + self.para_id, + self.relay_parent, + CoreIndex(0), + 1, + self.persisted_validation_data_hash, + self.pov_hash, + Default::default(), + self.para_head_hash.unwrap_or_else(|| self.head_data.hash()), + self.validation_code.hash(), + ), commitments: CandidateCommitments { head_data: self.head_data, new_validation_code: self.new_validation_code, @@ -1280,10 +1260,6 @@ fn candidate_checks() { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); - collator_sign_candidate(Sr25519Keyring::Two, &mut candidate_b_1); - collator_sign_candidate(Sr25519Keyring::Two, &mut candidate_b_2); - let backed_a = back_candidate( candidate_a, &validators, @@ -1370,7 +1346,6 @@ fn candidate_checks() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::Two, &mut candidate_b_3); let backed_b_3 = back_candidate( candidate_b_3, @@ -1404,7 +1379,6 @@ fn candidate_checks() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); // Insufficient backing. let backed = back_candidate( @@ -1477,10 +1451,6 @@ fn candidate_checks() { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); - - collator_sign_candidate(Sr25519Keyring::Two, &mut candidate_b); - let backed_a = back_candidate( candidate_a, &validators, @@ -1530,10 +1500,9 @@ fn candidate_checks() { .build(); assert_eq!(CollatorId::from(Sr25519Keyring::Two.public()), thread_collator); - collator_sign_candidate(Sr25519Keyring::Two, &mut candidate); // change the candidate after signing. - candidate.descriptor.pov_hash = Hash::repeat_byte(2); + candidate.descriptor.set_pov_hash(Hash::repeat_byte(2)); let backed = back_candidate( candidate, @@ -1607,8 +1576,6 @@ fn candidate_checks() { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -1657,8 +1624,6 @@ fn candidate_checks() { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -1695,8 +1660,6 @@ fn candidate_checks() { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -1733,8 +1696,6 @@ fn candidate_checks() { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -1834,7 +1795,6 @@ fn backing_works() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); let mut candidate_b = TestCandidateBuilder { para_id: chain_b, @@ -1845,7 +1805,6 @@ fn backing_works() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_b); let mut candidate_c = TestCandidateBuilder { para_id: thread_a, @@ -1856,7 +1815,6 @@ fn backing_works() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::Two, &mut candidate_c); let backed_a = back_candidate( candidate_a.clone(), @@ -1977,7 +1935,7 @@ fn backing_works() { Vec<(ValidatorIndex, ValidityAttestation)>, )>| { candidate_receipts_with_backers.sort_by(|(cr1, _), (cr2, _)| { - cr1.descriptor().para_id.cmp(&cr2.descriptor().para_id) + cr1.descriptor().para_id().cmp(&cr2.descriptor().para_id()) }); candidate_receipts_with_backers }; @@ -2129,7 +2087,6 @@ fn backing_works_with_elastic_scaling_mvp() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); let mut candidate_b_1 = TestCandidateBuilder { para_id: chain_b, @@ -2140,7 +2097,6 @@ fn backing_works_with_elastic_scaling_mvp() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_b_1); // Make candidate b2 a child of b1. let mut candidate_b_2 = TestCandidateBuilder { @@ -2157,7 +2113,6 @@ fn backing_works_with_elastic_scaling_mvp() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_b_2); let backed_a = back_candidate( candidate_a.clone(), @@ -2405,7 +2360,6 @@ fn can_include_candidate_with_ok_code_upgrade() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); let backed_a = back_candidate( candidate_a.clone(), @@ -2568,7 +2522,6 @@ fn check_allowed_relay_parents() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); let signing_context_a = SigningContext { parent_hash: relay_parent_a.1, session_index: 5 }; let mut candidate_b = TestCandidateBuilder { @@ -2584,7 +2537,6 @@ fn check_allowed_relay_parents() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_b); let signing_context_b = SigningContext { parent_hash: relay_parent_b.1, session_index: 5 }; let mut candidate_c = TestCandidateBuilder { @@ -2600,7 +2552,6 @@ fn check_allowed_relay_parents() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::Two, &mut candidate_c); let signing_context_c = SigningContext { parent_hash: relay_parent_c.1, session_index: 5 }; let backed_a = back_candidate( @@ -2832,7 +2783,6 @@ fn para_upgrade_delay_scheduled_from_inclusion() { ..Default::default() } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); let backed_a = back_candidate( candidate_a.clone(), diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index de616ea763b9..66330674898b 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -48,12 +48,17 @@ use frame_support::{ use frame_system::pallet_prelude::*; use pallet_babe::{self, ParentBlockRandomness}; use polkadot_primitives::{ - effective_minimum_backing_votes, node_features::FeatureIndex, vstaging::BackedCandidate, CandidateHash, - vstaging::CandidateReceiptV2 as CandidateReceipt, CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, CoreIndex, - DisputeStatementSet, HeadData, vstaging::InherentData as ParachainsInherentData, - MultiDisputeStatementSet, vstaging::ScrapedOnChainVotes, SessionIndex, SignedAvailabilityBitfields, - SigningContext, UncheckedSignedAvailabilityBitfield, UncheckedSignedAvailabilityBitfields, - ValidatorId, ValidatorIndex, ValidityAttestation, PARACHAINS_INHERENT_IDENTIFIER, + effective_minimum_backing_votes, + node_features::FeatureIndex, + vstaging::{ + BackedCandidate, CandidateReceiptV2 as CandidateReceipt, + InherentData as ParachainsInherentData, ScrapedOnChainVotes, + }, + CandidateHash, CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, CoreIndex, + DisputeStatementSet, HeadData, MultiDisputeStatementSet, SessionIndex, + SignedAvailabilityBitfields, SigningContext, UncheckedSignedAvailabilityBitfield, + UncheckedSignedAvailabilityBitfields, ValidatorId, ValidatorIndex, ValidityAttestation, + PARACHAINS_INHERENT_IDENTIFIER, }; use rand::{seq::SliceRandom, SeedableRng}; use scale_info::TypeInfo; diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 59fbb0948373..eae61f3c57b0 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -1381,7 +1381,7 @@ mod enter { // The chained candidates are not picked, instead a single other candidate is picked assert_eq!(backed_candidates.len(), 1); - assert_ne!(backed_candidates[0].descriptor().para_id, ParaId::from(1000)); + assert_ne!(backed_candidates[0].descriptor().para_id(), ParaId::from(1000)); // All bitfields are kept. assert_eq!(bitfields.len(), 150); @@ -1402,9 +1402,9 @@ mod enter { // Only the chained candidates should pass filter. assert_eq!(backed_candidates.len(), 3); // Check the actual candidates - assert_eq!(backed_candidates[0].descriptor().para_id, ParaId::from(1000)); - assert_eq!(backed_candidates[1].descriptor().para_id, ParaId::from(1000)); - assert_eq!(backed_candidates[2].descriptor().para_id, ParaId::from(1000)); + assert_eq!(backed_candidates[0].descriptor().para_id(), ParaId::from(1000)); + assert_eq!(backed_candidates[1].descriptor().para_id(), ParaId::from(1000)); + assert_eq!(backed_candidates[2].descriptor().para_id(), ParaId::from(1000)); // All bitfields are kept. assert_eq!(bitfields.len(), 150); @@ -1481,9 +1481,7 @@ mod sanitizers { use super::*; use crate::{ - inclusion::tests::{ - back_candidate, collator_sign_candidate, BackingKind, TestCandidateBuilder, - }, + inclusion::tests::{back_candidate, BackingKind, TestCandidateBuilder}, mock::new_test_ext, }; use bitvec::order::Lsb0; @@ -1898,8 +1896,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -1932,7 +1928,7 @@ mod sanitizers { let mut expected_backed_candidates_with_core = BTreeMap::new(); for candidate in backed_candidates.iter() { - let para_id = candidate.descriptor().para_id; + let para_id = candidate.descriptor().para_id(); expected_backed_candidates_with_core.entry(para_id).or_insert(vec![]).push(( candidate.clone(), @@ -2136,8 +2132,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let prev_candidate = candidate.clone(); let backed: BackedCandidate = back_candidate( candidate, @@ -2174,8 +2168,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -2213,8 +2205,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -2252,8 +2242,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -2289,8 +2277,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let prev_candidate = candidate.clone(); let backed = back_candidate( candidate, @@ -2325,8 +2311,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -2360,8 +2344,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -2393,8 +2375,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -2426,8 +2406,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -2666,8 +2644,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let prev_candidate = candidate.clone(); let prev_backed: BackedCandidate = back_candidate( candidate, @@ -2697,8 +2673,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -2732,8 +2706,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_1); - let backed_1: BackedCandidate = back_candidate( candidate_1, &validators, @@ -2770,8 +2742,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_2); - let backed_2 = back_candidate( candidate_2.clone(), &validators, @@ -2801,8 +2771,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_3); - let backed_3 = back_candidate( candidate_3, &validators, @@ -2835,8 +2803,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let prev_candidate = candidate.clone(); let backed: BackedCandidate = back_candidate( candidate, @@ -2873,8 +2839,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -2913,8 +2877,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed: BackedCandidate = back_candidate( candidate.clone(), &validators, @@ -3169,8 +3131,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let prev_candidate = candidate.clone(); let backed: BackedCandidate = back_candidate( candidate, @@ -3208,8 +3168,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let prev_candidate = candidate.clone(); let backed = back_candidate( candidate, @@ -3241,8 +3199,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -3275,8 +3231,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let prev_candidate = candidate.clone(); let backed: BackedCandidate = back_candidate( candidate, @@ -3314,8 +3268,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let prev_candidate = candidate.clone(); let backed = back_candidate( candidate, @@ -3353,8 +3305,6 @@ mod sanitizers { } .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed = back_candidate( candidate, &validators, @@ -3714,9 +3664,9 @@ mod sanitizers { let mut invalid_set = std::collections::BTreeSet::new(); for (idx, backed_candidate) in backed_candidates.iter().enumerate() { - if backed_candidate.descriptor().para_id == ParaId::from(1) && idx == 0 { + if backed_candidate.descriptor().para_id() == ParaId::from(1) && idx == 0 { invalid_set.insert(backed_candidate.hash()); - } else if backed_candidate.descriptor().para_id == ParaId::from(3) { + } else if backed_candidate.descriptor().para_id() == ParaId::from(3) { invalid_set.insert(backed_candidate.hash()); } } @@ -3752,7 +3702,7 @@ mod sanitizers { let mut invalid_set = std::collections::BTreeSet::new(); for (idx, backed_candidate) in backed_candidates.iter().enumerate() { - if backed_candidate.descriptor().para_id == ParaId::from(1) && idx == 1 { + if backed_candidate.descriptor().para_id() == ParaId::from(1) && idx == 1 { invalid_set.insert(backed_candidate.hash()); } } From 5efab68c7d848e09fdcbffc92b1ef385148c1cf0 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 12 Aug 2024 15:06:11 +0300 Subject: [PATCH 010/103] update rococo and westend Signed-off-by: Andrei Sandu --- polkadot/runtime/rococo/src/lib.rs | 18 +++++++++++------- polkadot/runtime/westend/src/lib.rs | 18 +++++++++++------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 2f23b889916b..5534b86a564d 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -35,12 +35,16 @@ use frame_support::{ }; use pallet_nis::WithMaximumOf; use polkadot_primitives::{ - slashing, AccountId, AccountIndex, ApprovalVotingParams, Balance, BlockNumber, CandidateEvent, - CandidateHash, CommittedCandidateReceipt, CoreIndex, CoreState, DisputeState, ExecutorParams, - GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, - NodeFeatures, Nonce, OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, - SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, - PARACHAIN_KEY_TYPE_ID, + slashing, + vstaging::{ + CandidateEvent, CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreState, + ScrapedOnChainVotes, + }, + AccountId, AccountIndex, ApprovalVotingParams, Balance, BlockNumber, CandidateHash, CoreIndex, + DisputeState, ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, NodeFeatures, Nonce, OccupiedCoreAssumption, + PersistedValidationData, SessionInfo, Signature, ValidationCode, ValidationCodeHash, + ValidatorId, ValidatorIndex, PARACHAIN_KEY_TYPE_ID, }; use polkadot_runtime_common::{ assigned_slots, auctions, claims, crowdloan, identity_migrator, impl_runtime_weights, @@ -2025,7 +2029,7 @@ sp_api::impl_runtime_apis! { parachains_runtime_api_impl::minimum_backing_votes::() } - fn para_backing_state(para_id: ParaId) -> Option { + fn para_backing_state(para_id: ParaId) -> Option { parachains_runtime_api_impl::backing_state::(para_id) } diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 838ba17e5613..67c43de8bae7 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -49,12 +49,16 @@ use pallet_identity::legacy::IdentityInfo; use pallet_session::historical as session_historical; use pallet_transaction_payment::{FeeDetails, FungibleAdapter, RuntimeDispatchInfo}; use polkadot_primitives::{ - slashing, AccountId, AccountIndex, ApprovalVotingParams, Balance, BlockNumber, CandidateEvent, - CandidateHash, CommittedCandidateReceipt, CoreIndex, CoreState, DisputeState, ExecutorParams, - GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, Moment, - NodeFeatures, Nonce, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, - ScrapedOnChainVotes, SessionInfo, Signature, ValidationCode, ValidationCodeHash, ValidatorId, - ValidatorIndex, ValidatorSignature, PARACHAIN_KEY_TYPE_ID, + slashing, + vstaging::{ + CandidateEvent, CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreState, + ScrapedOnChainVotes, + }, + AccountId, AccountIndex, ApprovalVotingParams, Balance, BlockNumber, CandidateHash, CoreIndex, + DisputeState, ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, NodeFeatures, Nonce, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, SessionInfo, Signature, ValidationCode, + ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, PARACHAIN_KEY_TYPE_ID, }; use polkadot_runtime_common::{ assigned_slots, auctions, crowdloan, @@ -2085,7 +2089,7 @@ sp_api::impl_runtime_apis! { parachains_runtime_api_impl::minimum_backing_votes::() } - fn para_backing_state(para_id: ParaId) -> Option { + fn para_backing_state(para_id: ParaId) -> Option { parachains_runtime_api_impl::backing_state::(para_id) } From c2232e425d5b2f8d0a4131963bfe4233193979d3 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 12 Aug 2024 15:06:57 +0300 Subject: [PATCH 011/103] client keeps using the old primitives Signed-off-by: Andrei Sandu --- .../src/lib.rs | 22 ++++++++++-- polkadot/node/service/src/fake_runtime_api.rs | 16 +++++---- .../subsystem-types/src/runtime_client.rs | 34 +++++++++++++++---- .../node/test/client/src/block_builder.rs | 2 +- 4 files changed, 58 insertions(+), 16 deletions(-) diff --git a/cumulus/client/relay-chain-inprocess-interface/src/lib.rs b/cumulus/client/relay-chain-inprocess-interface/src/lib.rs index 38ba84748c1e..65c0592ff5e5 100644 --- a/cumulus/client/relay-chain-inprocess-interface/src/lib.rs +++ b/cumulus/client/relay-chain-inprocess-interface/src/lib.rs @@ -137,7 +137,11 @@ impl RelayChainInterface for RelayChainInProcessInterface { hash: PHash, para_id: ParaId, ) -> RelayChainResult> { - Ok(self.full_client.runtime_api().candidate_pending_availability(hash, para_id)?) + Ok(self + .full_client + .runtime_api() + .candidate_pending_availability(hash, para_id)? + .map(|receipt| receipt.into())) } async fn session_index_for_child(&self, hash: PHash) -> RelayChainResult { @@ -260,7 +264,13 @@ impl RelayChainInterface for RelayChainInProcessInterface { &self, relay_parent: PHash, ) -> RelayChainResult>> { - Ok(self.full_client.runtime_api().availability_cores(relay_parent)?) + Ok(self + .full_client + .runtime_api() + .availability_cores(relay_parent)? + .into_iter() + .map(|core_state| core_state.into()) + .collect::>()) } async fn candidates_pending_availability( @@ -268,7 +278,13 @@ impl RelayChainInterface for RelayChainInProcessInterface { hash: PHash, para_id: ParaId, ) -> RelayChainResult> { - Ok(self.full_client.runtime_api().candidates_pending_availability(hash, para_id)?) + Ok(self + .full_client + .runtime_api() + .candidates_pending_availability(hash, para_id)? + .into_iter() + .map(|receipt| receipt.into()) + .collect::>()) } } diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs index cdef39d5bdf1..1f2efdbbb5b3 100644 --- a/polkadot/node/service/src/fake_runtime_api.rs +++ b/polkadot/node/service/src/fake_runtime_api.rs @@ -21,12 +21,16 @@ use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use polkadot_primitives::{ - runtime_api, slashing, AccountId, AuthorityDiscoveryId, Balance, Block, BlockNumber, - CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, - DisputeState, ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, - InboundHrmpMessage, Nonce, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, - ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, - ValidatorId, ValidatorIndex, ValidatorSignature, + runtime_api, slashing, + vstaging::{ + CandidateEvent, CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreState, + ScrapedOnChainVotes, + }, + AccountId, AuthorityDiscoveryId, Balance, Block, BlockNumber, CandidateCommitments, + CandidateHash, DisputeState, ExecutorParams, GroupRotationInfo, Hash, Id as ParaId, + InboundDownwardMessage, InboundHrmpMessage, Nonce, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, SessionIndex, SessionInfo, ValidationCode, + ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_consensus_beefy::ecdsa_crypto::{AuthorityId as BeefyId, Signature as BeefySignature}; use sp_consensus_grandpa::AuthorityId as GrandpaId; diff --git a/polkadot/node/subsystem-types/src/runtime_client.rs b/polkadot/node/subsystem-types/src/runtime_client.rs index e5e1e4d24ef9..7938223df23b 100644 --- a/polkadot/node/subsystem-types/src/runtime_client.rs +++ b/polkadot/node/subsystem-types/src/runtime_client.rs @@ -380,7 +380,10 @@ where &self, at: Hash, ) -> Result>, ApiError> { - self.client.runtime_api().availability_cores(at) + self.client + .runtime_api() + .availability_cores(at) + .map(|cores| cores.into_iter().map(|core| core.into()).collect::>()) } async fn persisted_validation_data( @@ -433,7 +436,10 @@ where at: Hash, para_id: Id, ) -> Result>, ApiError> { - self.client.runtime_api().candidate_pending_availability(at, para_id) + self.client + .runtime_api() + .candidate_pending_availability(at, para_id) + .map(|maybe_candidate| maybe_candidate.map(|candidate| candidate.into())) } async fn candidates_pending_availability( @@ -441,11 +447,19 @@ where at: Hash, para_id: Id, ) -> Result>, ApiError> { - self.client.runtime_api().candidates_pending_availability(at, para_id) + self.client + .runtime_api() + .candidates_pending_availability(at, para_id) + .map(|candidates| { + candidates.into_iter().map(|candidate| candidate.into()).collect::>() + }) } async fn candidate_events(&self, at: Hash) -> Result>, ApiError> { - self.client.runtime_api().candidate_events(at) + self.client + .runtime_api() + .candidate_events(at) + .map(|events| events.into_iter().map(|event| event.into()).collect::>()) } async fn dmq_contents( @@ -476,7 +490,10 @@ where &self, at: Hash, ) -> Result>, ApiError> { - self.client.runtime_api().on_chain_votes(at) + self.client + .runtime_api() + .on_chain_votes(at) + .map(|maybe_votes| maybe_votes.map(|votes| votes.into())) } async fn session_executor_params( @@ -588,7 +605,12 @@ where at: Hash, para_id: Id, ) -> Result, ApiError> { - self.client.runtime_api().para_backing_state(at, para_id) + self.client + .runtime_api() + .para_backing_state(at, para_id) + .map(|maybe_backing_state| { + maybe_backing_state.map(|backing_state| backing_state.into()) + }) } async fn async_backing_params( diff --git a/polkadot/node/test/client/src/block_builder.rs b/polkadot/node/test/client/src/block_builder.rs index ed24cd55d084..9375aca6ed73 100644 --- a/polkadot/node/test/client/src/block_builder.rs +++ b/polkadot/node/test/client/src/block_builder.rs @@ -16,7 +16,7 @@ use crate::Client; use codec::{Decode, Encode}; -use polkadot_primitives::{Block, vstaging::InherentData as ParachainsInherentData}; +use polkadot_primitives::{vstaging::InherentData as ParachainsInherentData, Block}; use polkadot_test_runtime::UncheckedExtrinsic; use polkadot_test_service::GetLastTimestamp; use sc_block_builder::{BlockBuilder, BlockBuilderBuilder}; From 87b079f82219d62e4aca3076840583fcabb3e661 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 12 Aug 2024 15:30:16 +0300 Subject: [PATCH 012/103] no unsafe pls Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 9541484d5028..b1f5a8387260 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -32,7 +32,7 @@ use alloc::{ vec, vec::Vec, }; -use codec::{Decode, Encode, WrapperTypeDecode}; +use codec::{Decode, Encode}; use scale_info::TypeInfo; use sp_arithmetic::Perbill; use sp_core::RuntimeDebug; @@ -447,8 +447,8 @@ impl CandidateDescriptorV2 { fn rebuild_collator_field(&self) -> CollatorId { let mut collator_id = vec![self.version.0]; - let core_index: [u8; 2] = unsafe { core::mem::transmute(self.core_index) }; - let session_index: [u8; 4] = unsafe { core::mem::transmute(self.session_index) }; + let core_index: [u8; 2] = self.core_index.to_ne_bytes(); + let session_index: [u8; 4] = self.session_index.to_ne_bytes(); collator_id.append(&mut core_index.as_slice().to_vec()); collator_id.append(&mut session_index.as_slice().to_vec()); From 00e8c13133007f1e68879e626bc5e569ff368a3d Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 12 Aug 2024 17:28:50 +0300 Subject: [PATCH 013/103] move async backing primtiives to own file Signed-off-by: Andrei Sandu --- polkadot/primitives/src/runtime_api.rs | 2 +- .../primitives/src/vstaging/async_backing.rs | 76 +++++++++++++++++++ polkadot/primitives/src/vstaging/mod.rs | 55 +------------- .../parachains/src/runtime_api_impl/v10.rs | 2 +- 4 files changed, 80 insertions(+), 55 deletions(-) create mode 100644 polkadot/primitives/src/vstaging/async_backing.rs diff --git a/polkadot/primitives/src/runtime_api.rs b/polkadot/primitives/src/runtime_api.rs index 6412a67b460d..ddebe99e6214 100644 --- a/polkadot/primitives/src/runtime_api.rs +++ b/polkadot/primitives/src/runtime_api.rs @@ -264,7 +264,7 @@ sp_api::decl_runtime_apis! { /// Returns the state of parachain backing for a given para. #[api_version(7)] - fn para_backing_state(_: ppp::Id) -> Option>; + fn para_backing_state(_: ppp::Id) -> Option>; /// Returns candidate's acceptance limitations for asynchronous backing for a relay parent. #[api_version(7)] diff --git a/polkadot/primitives/src/vstaging/async_backing.rs b/polkadot/primitives/src/vstaging/async_backing.rs new file mode 100644 index 000000000000..bdf94e0f00db --- /dev/null +++ b/polkadot/primitives/src/vstaging/async_backing.rs @@ -0,0 +1,76 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; + +use alloc::vec::Vec; +use codec::{Decode, Encode}; +use scale_info::TypeInfo; +use sp_core::RuntimeDebug; + +/// A candidate pending availability. +#[derive(RuntimeDebug, Clone, PartialEq, Encode, Decode, TypeInfo)] +pub struct CandidatePendingAvailability { + /// The hash of the candidate. + pub candidate_hash: CandidateHash, + /// The candidate's descriptor. + pub descriptor: CandidateDescriptorV2, + /// The commitments of the candidate. + pub commitments: CandidateCommitments, + /// The candidate's relay parent's number. + pub relay_parent_number: N, + /// The maximum Proof-of-Validity size allowed, in bytes. + pub max_pov_size: u32, +} + +impl From> + for crate::v7::async_backing::CandidatePendingAvailability +{ + fn from(value: CandidatePendingAvailability) -> Self { + Self { + candidate_hash: value.candidate_hash, + descriptor: value.descriptor.into(), + commitments: value.commitments, + relay_parent_number: value.relay_parent_number, + max_pov_size: value.max_pov_size, + } + } +} + +/// The per-parachain state of the backing system, including +/// state-machine constraints and candidates pending availability. +#[derive(RuntimeDebug, Clone, PartialEq, Encode, Decode, TypeInfo)] +pub struct BackingState { + /// The state-machine constraints of the parachain. + pub constraints: Constraints, + /// The candidates pending availability. These should be ordered, i.e. they should form + /// a sub-chain, where the first candidate builds on top of the required parent of the + /// constraints and each subsequent builds on top of the previous head-data. + pub pending_availability: Vec>, +} + +impl From> for crate::v7::async_backing::BackingState { + fn from(value: BackingState) -> Self { + Self { + constraints: value.constraints, + pending_availability: value + .pending_availability + .into_iter() + .map(|candidate| candidate.into()) + .collect::>(), + } + } +} diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index b1f5a8387260..e62e8aa3d760 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -39,6 +39,8 @@ use sp_core::RuntimeDebug; use sp_runtime::traits::Header as HeaderT; use sp_staking::SessionIndex; +pub mod async_backing; + /// Scheduler configuration parameters. All coretime/ondemand parameters are here. #[derive( RuntimeDebug, @@ -715,59 +717,6 @@ impl From> for super::v7::Scra } } -/// A candidate pending availability. -#[derive(RuntimeDebug, Clone, PartialEq, Encode, Decode, TypeInfo)] -pub struct CandidatePendingAvailability { - /// The hash of the candidate. - pub candidate_hash: CandidateHash, - /// The candidate's descriptor. - pub descriptor: CandidateDescriptorV2, - /// The commitments of the candidate. - pub commitments: CandidateCommitments, - /// The candidate's relay parent's number. - pub relay_parent_number: N, - /// The maximum Proof-of-Validity size allowed, in bytes. - pub max_pov_size: u32, -} - -impl From> - for super::async_backing::CandidatePendingAvailability -{ - fn from(value: CandidatePendingAvailability) -> Self { - Self { - candidate_hash: value.candidate_hash, - descriptor: value.descriptor.into(), - commitments: value.commitments, - relay_parent_number: value.relay_parent_number, - max_pov_size: value.max_pov_size, - } - } -} - -/// The per-parachain state of the backing system, including -/// state-machine constraints and candidates pending availability. -#[derive(RuntimeDebug, Clone, PartialEq, Encode, Decode, TypeInfo)] -pub struct BackingState { - /// The state-machine constraints of the parachain. - pub constraints: Constraints, - /// The candidates pending availability. These should be ordered, i.e. they should form - /// a sub-chain, where the first candidate builds on top of the required parent of the - /// constraints and each subsequent builds on top of the previous head-data. - pub pending_availability: Vec>, -} - -impl From> for super::async_backing::BackingState { - fn from(value: BackingState) -> Self { - Self { - constraints: value.constraints, - pending_availability: value - .pending_availability - .into_iter() - .map(|candidate| candidate.into()) - .collect::>(), - } - } -} /// Information about a core which is currently occupied. #[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] #[cfg_attr(feature = "std", derive(PartialEq))] diff --git a/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs b/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs index 608db7434b87..b72b1b550a0b 100644 --- a/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs +++ b/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs @@ -31,7 +31,7 @@ use polkadot_primitives::{ }, slashing, vstaging::{ - BackingState, CandidateEvent, CandidatePendingAvailability, + async_backing::BackingState, CandidateEvent, async_backing::CandidatePendingAvailability, CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreState, OccupiedCore, ScrapedOnChainVotes, }, From cd4d02f963aed56fe41969191966772be17c353d Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 12 Aug 2024 18:00:54 +0300 Subject: [PATCH 014/103] fix Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 7 +++++-- polkadot/runtime/parachains/src/runtime_api_impl/v10.rs | 6 +++--- polkadot/runtime/rococo/src/lib.rs | 2 +- polkadot/runtime/westend/src/lib.rs | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index e62e8aa3d760..4f69073648fd 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -39,6 +39,7 @@ use sp_core::RuntimeDebug; use sp_runtime::traits::Header as HeaderT; use sp_staking::SessionIndex; +/// Async backing primitives pub mod async_backing; /// Scheduler configuration parameters. All coretime/ondemand parameters are here. @@ -209,6 +210,7 @@ impl CandidateDescriptorV2 { } } + /// Set the PoV size in the descriptor. Only for tests. #[cfg(feature = "test")] pub fn set_pov_hash(&mut self, pov_hash: Hash) { self.pov_hash = pov_hash; @@ -456,7 +458,8 @@ impl CandidateDescriptorV2 { collator_id.append(&mut session_index.as_slice().to_vec()); collator_id.append(&mut self.reserved25b.as_slice().to_vec()); - CollatorId::from_slice(&collator_id.as_slice()).expect("Slice size is exactly 32 bytes") + CollatorId::from_slice(&collator_id.as_slice()) + .expect("Slice size is exactly 32 bytes; qed") } /// Returns the collator id if this is a v1 `CandidateDescriptor` @@ -470,7 +473,7 @@ impl CandidateDescriptorV2 { fn rebuild_signature_field(&self) -> CollatorSignature { CollatorSignature::from_slice(self.reserved64b.as_slice()) - .expect("Slice size is exactly 64 bytes") + .expect("Slice size is exactly 64 bytes; qed") } /// Returns the collator signature of `V1` candidate descriptors, `None` otherwise. diff --git a/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs b/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs index b72b1b550a0b..ead825b38f07 100644 --- a/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs +++ b/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs @@ -31,9 +31,9 @@ use polkadot_primitives::{ }, slashing, vstaging::{ - async_backing::BackingState, CandidateEvent, async_backing::CandidatePendingAvailability, - CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreState, OccupiedCore, - ScrapedOnChainVotes, + async_backing::{BackingState, CandidatePendingAvailability}, + CandidateEvent, CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreState, + OccupiedCore, ScrapedOnChainVotes, }, ApprovalVotingParams, AuthorityDiscoveryId, CandidateHash, CoreIndex, DisputeState, ExecutorParams, GroupIndex, GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 5534b86a564d..688fe6317cc4 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -2029,7 +2029,7 @@ sp_api::impl_runtime_apis! { parachains_runtime_api_impl::minimum_backing_votes::() } - fn para_backing_state(para_id: ParaId) -> Option { + fn para_backing_state(para_id: ParaId) -> Option { parachains_runtime_api_impl::backing_state::(para_id) } diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 67c43de8bae7..6f487960b810 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -2089,7 +2089,7 @@ sp_api::impl_runtime_apis! { parachains_runtime_api_impl::minimum_backing_votes::() } - fn para_backing_state(para_id: ParaId) -> Option { + fn para_backing_state(para_id: ParaId) -> Option { parachains_runtime_api_impl::backing_state::(para_id) } From 5509e3340810973ea4344466bebb7e2cb5953574 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 12 Aug 2024 18:33:04 +0300 Subject: [PATCH 015/103] fix test build Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/builder.rs | 22 +++------ .../parachains/src/inclusion/migration.rs | 3 +- .../runtime/parachains/src/inclusion/tests.rs | 44 ++++++++--------- .../parachains/src/paras_inherent/tests.rs | 49 +++++++++---------- 4 files changed, 53 insertions(+), 65 deletions(-) diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index c1e7844de470..ebc956a08e67 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -36,29 +36,19 @@ use polkadot_primitives::{ CommittedCandidateReceiptV2 as CommittedCandidateReceipt, InherentData as ParachainsInherentData, }, - AvailabilityBitfield, CandidateCommitments, CandidateHash, CollatorId, CollatorSignature, - CompactStatement, CoreIndex, DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, - Id as ParaId, IndexedVec, InvalidDisputeStatementKind, PersistedValidationData, SessionIndex, - SigningContext, UncheckedSigned, ValidDisputeStatementKind, ValidationCode, ValidatorId, - ValidatorIndex, ValidityAttestation, + AvailabilityBitfield, CandidateCommitments, CandidateHash, CompactStatement, CoreIndex, + DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, Id as ParaId, IndexedVec, + InvalidDisputeStatementKind, PersistedValidationData, SessionIndex, SigningContext, + UncheckedSigned, ValidDisputeStatementKind, ValidationCode, ValidatorId, ValidatorIndex, + ValidityAttestation, }; -use sp_core::{sr25519, ByteArray, H256}; +use sp_core::H256; use sp_runtime::{ generic::Digest, traits::{Header as HeaderT, One, TrailingZeroInput, Zero}, RuntimeAppPublic, }; -/// Create a null collator id. -pub fn dummy_collator() -> CollatorId { - CollatorId::from_slice(&vec![0u8; 32]).expect("32 bytes; qed") -} - -/// Create a null collator signature. -pub fn dummy_collator_signature() -> CollatorSignature { - CollatorSignature::from_slice(&vec![0u8; 64]).expect("64 bytes; qed") -} - fn mock_validation_code() -> ValidationCode { ValidationCode(vec![1, 2, 3]) } diff --git a/polkadot/runtime/parachains/src/inclusion/migration.rs b/polkadot/runtime/parachains/src/inclusion/migration.rs index 143c4e7d629a..2a215d5d595c 100644 --- a/polkadot/runtime/parachains/src/inclusion/migration.rs +++ b/polkadot/runtime/parachains/src/inclusion/migration.rs @@ -219,8 +219,7 @@ mod tests { use frame_support::traits::UncheckedOnRuntimeUpgrade; use polkadot_primitives::{AvailabilityBitfield, Id as ParaId}; use polkadot_primitives_test_helpers::{ - dummy_candidate_commitments, dummy_candidate_descriptor, dummy_candidate_descriptor_v2, - dummy_hash, + dummy_candidate_commitments, dummy_candidate_descriptor_v2, dummy_hash, }; #[test] diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index bfb2f228c452..2125dffe2592 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -1223,7 +1223,7 @@ fn candidate_checks() { // Check candidate ordering { - let mut candidate_a = TestCandidateBuilder { + let candidate_a = TestCandidateBuilder { para_id: chain_a, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(1), @@ -1232,7 +1232,7 @@ fn candidate_checks() { ..Default::default() } .build(); - let mut candidate_b_1 = TestCandidateBuilder { + let candidate_b_1 = TestCandidateBuilder { para_id: chain_b, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(2), @@ -1244,7 +1244,7 @@ fn candidate_checks() { .build(); // Make candidate b2 a child of b1. - let mut candidate_b_2 = TestCandidateBuilder { + let candidate_b_2 = TestCandidateBuilder { para_id: chain_b, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(3), @@ -1331,7 +1331,7 @@ fn candidate_checks() { // candidate does not build on top of the latest unincluded head - let mut candidate_b_3 = TestCandidateBuilder { + let candidate_b_3 = TestCandidateBuilder { para_id: chain_b, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(4), @@ -1370,7 +1370,7 @@ fn candidate_checks() { // candidate not backed. { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: chain_a, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(1), @@ -1432,7 +1432,7 @@ fn candidate_checks() { let wrong_parent_hash = Hash::repeat_byte(222); assert!(System::parent_hash() != wrong_parent_hash); - let mut candidate_a = TestCandidateBuilder { + let candidate_a = TestCandidateBuilder { para_id: chain_a, relay_parent: wrong_parent_hash, pov_hash: Hash::repeat_byte(1), @@ -1441,7 +1441,7 @@ fn candidate_checks() { } .build(); - let mut candidate_b = TestCandidateBuilder { + let candidate_b = TestCandidateBuilder { para_id: chain_b, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(2), @@ -1565,7 +1565,7 @@ fn candidate_checks() { // interfering code upgrade - reject { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: chain_a, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(1), @@ -1614,7 +1614,7 @@ fn candidate_checks() { // Bad validation data hash - reject { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: chain_a, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(1), @@ -1649,7 +1649,7 @@ fn candidate_checks() { // bad validation code hash { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: chain_a, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(1), @@ -1685,7 +1685,7 @@ fn candidate_checks() { // Para head hash in descriptor doesn't match head data { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: chain_a, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(1), @@ -1786,7 +1786,7 @@ fn backing_works() { let chain_b_assignment = (chain_b, CoreIndex::from(1)); let thread_a_assignment = (thread_a, CoreIndex::from(2)); - let mut candidate_a = TestCandidateBuilder { + let candidate_a = TestCandidateBuilder { para_id: chain_a, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(1), @@ -1796,7 +1796,7 @@ fn backing_works() { } .build(); - let mut candidate_b = TestCandidateBuilder { + let candidate_b = TestCandidateBuilder { para_id: chain_b, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(2), @@ -1806,7 +1806,7 @@ fn backing_works() { } .build(); - let mut candidate_c = TestCandidateBuilder { + let candidate_c = TestCandidateBuilder { para_id: thread_a, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(3), @@ -2078,7 +2078,7 @@ fn backing_works_with_elastic_scaling_mvp() { let allowed_relay_parents = default_allowed_relay_parent_tracker(); - let mut candidate_a = TestCandidateBuilder { + let candidate_a = TestCandidateBuilder { para_id: chain_a, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(1), @@ -2088,7 +2088,7 @@ fn backing_works_with_elastic_scaling_mvp() { } .build(); - let mut candidate_b_1 = TestCandidateBuilder { + let candidate_b_1 = TestCandidateBuilder { para_id: chain_b, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(2), @@ -2099,7 +2099,7 @@ fn backing_works_with_elastic_scaling_mvp() { .build(); // Make candidate b2 a child of b1. - let mut candidate_b_2 = TestCandidateBuilder { + let candidate_b_2 = TestCandidateBuilder { para_id: chain_b, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(3), @@ -2350,7 +2350,7 @@ fn can_include_candidate_with_ok_code_upgrade() { let allowed_relay_parents = default_allowed_relay_parent_tracker(); let chain_a_assignment = (chain_a, CoreIndex::from(0)); - let mut candidate_a = TestCandidateBuilder { + let candidate_a = TestCandidateBuilder { para_id: chain_a, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(1), @@ -2509,7 +2509,7 @@ fn check_allowed_relay_parents() { let chain_b_assignment = (chain_b, CoreIndex::from(1)); let thread_a_assignment = (thread_a, CoreIndex::from(2)); - let mut candidate_a = TestCandidateBuilder { + let candidate_a = TestCandidateBuilder { para_id: chain_a, relay_parent: relay_parent_a.1, pov_hash: Hash::repeat_byte(1), @@ -2524,7 +2524,7 @@ fn check_allowed_relay_parents() { .build(); let signing_context_a = SigningContext { parent_hash: relay_parent_a.1, session_index: 5 }; - let mut candidate_b = TestCandidateBuilder { + let candidate_b = TestCandidateBuilder { para_id: chain_b, relay_parent: relay_parent_b.1, pov_hash: Hash::repeat_byte(2), @@ -2539,7 +2539,7 @@ fn check_allowed_relay_parents() { .build(); let signing_context_b = SigningContext { parent_hash: relay_parent_b.1, session_index: 5 }; - let mut candidate_c = TestCandidateBuilder { + let candidate_c = TestCandidateBuilder { para_id: thread_a, relay_parent: relay_parent_c.1, pov_hash: Hash::repeat_byte(3), @@ -2773,7 +2773,7 @@ fn para_upgrade_delay_scheduled_from_inclusion() { let allowed_relay_parents = default_allowed_relay_parent_tracker(); let chain_a_assignment = (chain_a, CoreIndex::from(0)); - let mut candidate_a = TestCandidateBuilder { + let candidate_a = TestCandidateBuilder { para_id: chain_a, relay_parent: System::parent_hash(), pov_hash: Hash::repeat_byte(1), diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index eae61f3c57b0..6c82233e0820 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -1495,7 +1495,6 @@ mod sanitizers { use crate::mock::Test; use polkadot_primitives::PARACHAIN_KEY_TYPE_ID; use sc_keystore::LocalKeystore; - use sp_keyring::Sr25519Keyring; use sp_keystore::{Keystore, KeystorePtr}; use std::sync::Arc; @@ -1879,7 +1878,7 @@ mod sanitizers { .into_iter() .map(|idx0| { let idx1 = idx0 + 1; - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(idx1), relay_parent, pov_hash: Hash::repeat_byte(idx1 as u8), @@ -2114,7 +2113,7 @@ mod sanitizers { // Para 1 { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(1), relay_parent, pov_hash: Hash::repeat_byte(1 as u8), @@ -2150,7 +2149,7 @@ mod sanitizers { .push((backed, CoreIndex(0))); } - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(1), relay_parent, pov_hash: Hash::repeat_byte(2 as u8), @@ -2188,7 +2187,7 @@ mod sanitizers { // Para 2 { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(2), relay_parent, pov_hash: Hash::repeat_byte(3 as u8), @@ -2225,7 +2224,7 @@ mod sanitizers { // Para 3 { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(3), relay_parent, pov_hash: Hash::repeat_byte(4 as u8), @@ -2260,7 +2259,7 @@ mod sanitizers { // Para 4 { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(4), relay_parent, pov_hash: Hash::repeat_byte(5 as u8), @@ -2293,7 +2292,7 @@ mod sanitizers { .or_insert(vec![]) .push((backed, CoreIndex(5))); - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(4), relay_parent, pov_hash: Hash::repeat_byte(6 as u8), @@ -2327,7 +2326,7 @@ mod sanitizers { // Para 6. { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(6), relay_parent, pov_hash: Hash::repeat_byte(3 as u8), @@ -2358,7 +2357,7 @@ mod sanitizers { // Para 7. { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(7), relay_parent, pov_hash: Hash::repeat_byte(3 as u8), @@ -2389,7 +2388,7 @@ mod sanitizers { // Para 8. { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(8), relay_parent, pov_hash: Hash::repeat_byte(3 as u8), @@ -2626,7 +2625,7 @@ mod sanitizers { // Para 1 { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(1), relay_parent, pov_hash: Hash::repeat_byte(1 as u8), @@ -2655,7 +2654,7 @@ mod sanitizers { core_index_enabled.then_some(CoreIndex(0 as u32)), ); - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(1), relay_parent, pov_hash: Hash::repeat_byte(2 as u8), @@ -2688,7 +2687,7 @@ mod sanitizers { // Para 2. { - let mut candidate_1 = TestCandidateBuilder { + let candidate_1 = TestCandidateBuilder { para_id: ParaId::from(2), relay_parent, pov_hash: Hash::repeat_byte(3 as u8), @@ -2724,7 +2723,7 @@ mod sanitizers { .push((backed_1, CoreIndex(2))); } - let mut candidate_2 = TestCandidateBuilder { + let candidate_2 = TestCandidateBuilder { para_id: ParaId::from(2), relay_parent, pov_hash: Hash::repeat_byte(4 as u8), @@ -2753,7 +2752,7 @@ mod sanitizers { ); backed_candidates.push(backed_2.clone()); - let mut candidate_3 = TestCandidateBuilder { + let candidate_3 = TestCandidateBuilder { para_id: ParaId::from(2), relay_parent, pov_hash: Hash::repeat_byte(5 as u8), @@ -2785,7 +2784,7 @@ mod sanitizers { // Para 3 { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(3), relay_parent, pov_hash: Hash::repeat_byte(6 as u8), @@ -2821,7 +2820,7 @@ mod sanitizers { .push((backed, CoreIndex(5))); } - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(3), relay_parent, pov_hash: Hash::repeat_byte(6 as u8), @@ -2859,7 +2858,7 @@ mod sanitizers { // Para 4 { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(4), relay_parent, pov_hash: Hash::repeat_byte(8 as u8), @@ -3113,7 +3112,7 @@ mod sanitizers { // Para 1 { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(1), relay_parent, pov_hash: Hash::repeat_byte(1 as u8), @@ -3149,7 +3148,7 @@ mod sanitizers { .push((backed, CoreIndex(0))); } - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(1), relay_parent: prev_relay_parent, pov_hash: Hash::repeat_byte(1 as u8), @@ -3180,7 +3179,7 @@ mod sanitizers { ); backed_candidates.push(backed.clone()); - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(1), relay_parent, pov_hash: Hash::repeat_byte(1 as u8), @@ -3213,7 +3212,7 @@ mod sanitizers { // Para 2 { - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(2), relay_parent: prev_relay_parent, pov_hash: Hash::repeat_byte(2 as u8), @@ -3249,7 +3248,7 @@ mod sanitizers { .push((backed, CoreIndex(3))); } - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(2), relay_parent, pov_hash: Hash::repeat_byte(2 as u8), @@ -3286,7 +3285,7 @@ mod sanitizers { .push((backed, CoreIndex(4))); } - let mut candidate = TestCandidateBuilder { + let candidate = TestCandidateBuilder { para_id: ParaId::from(2), relay_parent, pov_hash: Hash::repeat_byte(2 as u8), From f8b86d26c687a447120122277966dd4322f4b06f Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 12 Aug 2024 23:03:31 +0300 Subject: [PATCH 016/103] fix test-runtime Signed-off-by: Andrei Sandu --- polkadot/runtime/test-runtime/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs index 8e34320d38f2..26ddf8c90baf 100644 --- a/polkadot/runtime/test-runtime/src/lib.rs +++ b/polkadot/runtime/test-runtime/src/lib.rs @@ -59,10 +59,10 @@ use pallet_session::historical as session_historical; use pallet_timestamp::Now; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use polkadot_primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, CandidateEvent, CandidateHash, - CommittedCandidateReceipt, CoreIndex, CoreState, DisputeState, ExecutorParams, + slashing, AccountId, AccountIndex, Balance, BlockNumber, vstaging::CandidateEvent, CandidateHash, + vstaging::CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreIndex, vstaging::CoreState, DisputeState, ExecutorParams, GroupRotationInfo, Hash as HashT, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, - Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, ScrapedOnChainVotes, + Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, vstaging::ScrapedOnChainVotes, SessionInfo as SessionInfoData, Signature, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, PARACHAIN_KEY_TYPE_ID, }; @@ -978,7 +978,7 @@ sp_api::impl_runtime_apis! { runtime_impl::minimum_backing_votes::() } - fn para_backing_state(para_id: ParaId) -> Option { + fn para_backing_state(para_id: ParaId) -> Option { runtime_impl::backing_state::(para_id) } From fe2fbfb6c3b759cfaed13f1ed50425cb4bd14827 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 13 Aug 2024 11:07:39 +0300 Subject: [PATCH 017/103] self review feedback Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 38 ++++++++++++++---------- polkadot/runtime/test-runtime/src/lib.rs | 12 +++++--- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 4f69073648fd..74d75fcc46eb 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -350,9 +350,10 @@ impl From> for super::v7::CandidateReceipt { } } -/// A strictly increasing sequence number, tipically this would be the parachain block number. +/// A strictly increasing sequence number, tipically this would be the least significant byte of the +/// block number. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] -pub struct CoreSelector(pub BlockNumber); +pub struct CoreSelector(pub u8); /// An offset in the relay chain claim queue. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] @@ -375,7 +376,7 @@ pub const UMP_SEPARATOR: Vec = vec![]; impl CandidateCommitments { /// Returns the core selector and claim queue offset the candidate has commited to, if any. pub fn selected_core(&self) -> Option<(CoreSelector, ClaimQueueOffset)> { - // We need at least 2 messages for the separator and core index + // We need at least 2 messages for the separator and core selector if self.upward_messages.len() < 2 { return None } @@ -388,7 +389,7 @@ impl CandidateCommitments { .take_while(|message| message != &UMP_SEPARATOR) .collect::>(); - // We didn't find the separator, no core index commitment. + // Check for UMP separator if upward_commitments.len() == self.upward_messages.len() || upward_commitments.is_empty() { return None } @@ -510,32 +511,43 @@ impl CommittedCandidateReceiptV2 { /// Returns error if: /// - descriptor core index is different than the core selected /// by the commitments - /// - the core index is out of bounds wrt `n_cores`. + /// - the core index is out of bounds pub fn check( &self, n_cores: u32, // TODO: consider promoting `ClaimQueueSnapshot` as primitive claim_queue: &BTreeMap>, ) -> Result<(), CandidateReceiptError> { + // Don't check v1 descriptors. + if self.descriptor.version() == CandidateDescriptorVersion::V1 { + return Ok(()) + } + if claim_queue.is_empty() { return Err(CandidateReceiptError::NoAssignment) } + let descriptor_core_index = CoreIndex(self.descriptor.core_index as u32); + if descriptor_core_index.0 > n_cores - 1 { + return Err(CandidateReceiptError::InvalidCoreIndex) + } + + let (core_selector, cq_offset) = + self.commitments.selected_core().ok_or(CandidateReceiptError::NoCoreSelected)?; + + // TODO: double check the invariant of claim queue len == scheduling lookahead. + // What happens when on-demand is used and there is only 1 claim on a core. let claim_queue_depth = claim_queue .first_key_value() .ok_or(CandidateReceiptError::NoAssignment)? .1 .len(); - - let descriptor_core_index = CoreIndex(self.descriptor.core_index as u32); - let (core_selector, cq_offset) = - self.commitments.selected_core().ok_or(CandidateReceiptError::NoCoreSelected)?; - let para_id = self.descriptor.para_id; - if cq_offset.0 as usize >= claim_queue_depth { return Err(CandidateReceiptError::InvalidSelectedCore) } + let para_id = self.descriptor.para_id; + // Get a vec of the core indices the parachain is assigned to at `cq_offset`. let assigned_cores = claim_queue .iter() @@ -562,10 +574,6 @@ impl CommittedCandidateReceiptV2 { return Err(CandidateReceiptError::CoreIndexMismatch) } - if descriptor_core_index.0 > n_cores - 1 { - return Err(CandidateReceiptError::InvalidCoreIndex) - } - Ok(()) } } diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs index 26ddf8c90baf..72d024e9a878 100644 --- a/polkadot/runtime/test-runtime/src/lib.rs +++ b/polkadot/runtime/test-runtime/src/lib.rs @@ -59,10 +59,14 @@ use pallet_session::historical as session_historical; use pallet_timestamp::Now; use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo}; use polkadot_primitives::{ - slashing, AccountId, AccountIndex, Balance, BlockNumber, vstaging::CandidateEvent, CandidateHash, - vstaging::CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreIndex, vstaging::CoreState, DisputeState, ExecutorParams, - GroupRotationInfo, Hash as HashT, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, - Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, vstaging::ScrapedOnChainVotes, + slashing, + vstaging::{ + CandidateEvent, CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreState, + ScrapedOnChainVotes, + }, + AccountId, AccountIndex, Balance, BlockNumber, CandidateHash, CoreIndex, DisputeState, + ExecutorParams, GroupRotationInfo, Hash as HashT, Id as ParaId, InboundDownwardMessage, + InboundHrmpMessage, Moment, Nonce, OccupiedCoreAssumption, PersistedValidationData, SessionInfo as SessionInfoData, Signature, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, PARACHAIN_KEY_TYPE_ID, }; From 975e13bafcbaf2c27c5ac5bcb2dafe5f897ca421 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 13 Aug 2024 13:22:27 +0300 Subject: [PATCH 018/103] review feedback Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 9 +++++---- polkadot/runtime/parachains/Cargo.toml | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 74d75fcc46eb..7f438be9434d 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -451,13 +451,14 @@ impl CandidateDescriptorV2 { } fn rebuild_collator_field(&self) -> CollatorId { - let mut collator_id = vec![self.version.0]; + let mut collator_id = Vec::with_capacity(32); let core_index: [u8; 2] = self.core_index.to_ne_bytes(); let session_index: [u8; 4] = self.session_index.to_ne_bytes(); - collator_id.append(&mut core_index.as_slice().to_vec()); - collator_id.append(&mut session_index.as_slice().to_vec()); - collator_id.append(&mut self.reserved25b.as_slice().to_vec()); + collator_id.push(self.version.0); + collator_id.extend_from_slice(core_index.as_slice()); + collator_id.extend_from_slice(session_index.as_slice()); + collator_id.extend_from_slice(self.reserved25b.as_slice()); CollatorId::from_slice(&collator_id.as_slice()) .expect("Slice size is exactly 32 bytes; qed") diff --git a/polkadot/runtime/parachains/Cargo.toml b/polkadot/runtime/parachains/Cargo.toml index 59af889fe6e7..f3123fa554ca 100644 --- a/polkadot/runtime/parachains/Cargo.toml +++ b/polkadot/runtime/parachains/Cargo.toml @@ -49,7 +49,7 @@ frame-system = { workspace = true } xcm = { workspace = true } xcm-executor = { workspace = true } -polkadot-primitives = { workspace = true, features=["test"] } +polkadot-primitives = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } @@ -59,7 +59,7 @@ polkadot-runtime-metrics = { workspace = true } polkadot-core-primitives = { workspace = true } [dev-dependencies] -polkadot-primitives = { workspace = true } +polkadot-primitives = { workspace = true, features=["test"] } futures = { workspace = true } hex-literal = { workspace = true, default-features = true } From 1c7ac55b5dce45a2f0a63e55381e0e18d58ac5e1 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 13 Aug 2024 13:39:40 +0300 Subject: [PATCH 019/103] feedback Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 7f438be9434d..8251a32c4aa7 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -122,14 +122,13 @@ impl> Default for SchedulerParams #[cfg_attr(feature = "std", derive(Hash))] pub struct InternalVersion(pub u8); -/// A type representing the version of the candidate descriptor and internal version number. -/// For `V1`` the internal version number stores the first byte of the `CollatorId`. +/// A type representing the version of the candidate descriptor. #[derive(PartialEq, Eq, Clone, TypeInfo, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Hash))] pub enum CandidateDescriptorVersion { /// The old candidate descriptor version. V1, - /// Introduced with `CandidateDescriptorV2` + /// The new `CandidateDescriptorV2`. V2, } @@ -141,7 +140,8 @@ pub struct CandidateDescriptorV2 { para_id: ParaId, /// The hash of the relay-chain block this is executed in the context of. relay_parent: H, - /// Version field + /// Version field. The raw value here is not exposed, instead it is used + /// to determine the `CandidateDescriptorVersion`, see `fn version()` version: InternalVersion, /// The core index where the candidate is backed. core_index: u16, @@ -237,7 +237,7 @@ pub struct CommittedCandidateReceiptV2 { pub commitments: CandidateCommitments, } -/// An even concerning a candidate. +/// An event concerning a candidate. #[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] #[cfg_attr(feature = "std", derive(PartialEq))] pub enum CandidateEvent { @@ -350,7 +350,7 @@ impl From> for super::v7::CandidateReceipt { } } -/// A strictly increasing sequence number, tipically this would be the least significant byte of the +/// A strictly increasing sequence number, typically this would be the least significant byte of the /// block number. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] pub struct CoreSelector(pub u8); @@ -374,7 +374,7 @@ pub enum UMPSignal { pub const UMP_SEPARATOR: Vec = vec![]; impl CandidateCommitments { - /// Returns the core selector and claim queue offset the candidate has commited to, if any. + /// Returns the core selector and claim queue offset the candidate has committed to, if any. pub fn selected_core(&self) -> Option<(CoreSelector, ClaimQueueOffset)> { // We need at least 2 messages for the separator and core selector if self.upward_messages.len() < 2 { From 653873b15819df3cdedb92161cfb19bbbf814420 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 13 Aug 2024 13:51:08 +0300 Subject: [PATCH 020/103] feedback Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 8251a32c4aa7..56e4dcb7054a 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -381,21 +381,11 @@ impl CandidateCommitments { return None } - let upward_commitments = self - .upward_messages - .iter() - .cloned() - .rev() - .take_while(|message| message != &UMP_SEPARATOR) - .collect::>(); - - // Check for UMP separator - if upward_commitments.len() == self.upward_messages.len() || upward_commitments.is_empty() { - return None - } + let separator_pos = + self.upward_messages.iter().rposition(|message| message == &UMP_SEPARATOR)?; // Use first commitment - let Some(message) = upward_commitments.into_iter().rev().next() else { return None }; + let message = self.upward_messages.get(separator_pos + 1)?; match UMPSignal::decode(&mut message.as_slice()).ok()? { UMPSignal::SelectCore(core_selector, cq_offset) => Some((core_selector, cq_offset)), From dc98149924a59162fda79f93eae2836fb545b502 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 13 Aug 2024 13:56:51 +0300 Subject: [PATCH 021/103] clippy Signed-off-by: Andrei Sandu --- .../runtime/parachains/src/paras_inherent/benchmarking.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/benchmarking.rs b/polkadot/runtime/parachains/src/paras_inherent/benchmarking.rs index c5284ba1dd1f..07bdf3d5489c 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/benchmarking.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/benchmarking.rs @@ -141,8 +141,8 @@ benchmarks! { // Traverse candidates and assert descriptors are as expected for (para_id, backing_validators) in vote.backing_validators_per_candidate.iter().enumerate() { let descriptor = backing_validators.0.descriptor(); - assert_eq!(ParaId::from(para_id), descriptor.para_id); - assert_eq!(header.hash(), descriptor.relay_parent); + assert_eq!(ParaId::from(para_id), descriptor.para_id()); + assert_eq!(header.hash(), descriptor.relay_parent()); assert_eq!(backing_validators.1.len(), votes); } @@ -198,8 +198,8 @@ benchmarks! { for (para_id, backing_validators) in vote.backing_validators_per_candidate.iter().enumerate() { let descriptor = backing_validators.0.descriptor(); - assert_eq!(ParaId::from(para_id), descriptor.para_id); - assert_eq!(header.hash(), descriptor.relay_parent); + assert_eq!(ParaId::from(para_id), descriptor.para_id()); + assert_eq!(header.hash(), descriptor.relay_parent()); assert_eq!( backing_validators.1.len(), votes, From 0a6bce3404f3ff91e67ccabf0356c60e19321e4b Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 13 Aug 2024 23:08:03 +0300 Subject: [PATCH 022/103] chores Signed-off-by: Andrei Sandu --- polkadot/primitives/src/v7/mod.rs | 2 +- polkadot/primitives/src/vstaging/mod.rs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/polkadot/primitives/src/v7/mod.rs b/polkadot/primitives/src/v7/mod.rs index 4b7abdb311f5..7d5d6e093c8f 100644 --- a/polkadot/primitives/src/v7/mod.rs +++ b/polkadot/primitives/src/v7/mod.rs @@ -1153,7 +1153,7 @@ pub enum OccupiedCoreAssumption { Free, } -/// An even concerning a candidate. +/// An event concerning a candidate. #[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] #[cfg_attr(feature = "std", derive(PartialEq))] pub enum CandidateEvent { diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 56e4dcb7054a..b089fb422f3c 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -539,13 +539,11 @@ impl CommittedCandidateReceiptV2 { let para_id = self.descriptor.para_id; - // Get a vec of the core indices the parachain is assigned to at `cq_offset`. + // Get a sorted vec of the core indices the parachain is assigned to at `cq_offset`. let assigned_cores = claim_queue .iter() .filter_map(|(core_index, queue)| { - let queued_para = queue.get(cq_offset.0 as usize)?; - - if queued_para == ¶_id { + if queue.get(cq_offset.0 as usize)? == ¶_id { Some(core_index) } else { None From 5e4dac2bc9939ddaa759e44738c6a3c5ad4f7ef8 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 14 Aug 2024 15:18:15 +0300 Subject: [PATCH 023/103] Filter v2 candidate descriptors Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 7 ++ polkadot/primitives/test-helpers/src/lib.rs | 8 +- polkadot/runtime/parachains/src/builder.rs | 75 +++++++++++----- .../runtime/parachains/src/inclusion/tests.rs | 29 +++--- .../parachains/src/paras_inherent/mod.rs | 16 +++- .../parachains/src/paras_inherent/tests.rs | 89 ++++++++++++++++++- 6 files changed, 181 insertions(+), 43 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index b089fb422f3c..aec3c7aeb612 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -181,6 +181,13 @@ impl From> for CandidateDescriptor { } } +#[cfg(feature = "test")] +impl From> for CandidateDescriptorV2 { + fn from(value: CandidateDescriptor) -> Self { + Decode::decode(&mut value.encode().as_slice()).unwrap() + } +} + impl CandidateDescriptorV2 { /// Constructor pub fn new( diff --git a/polkadot/primitives/test-helpers/src/lib.rs b/polkadot/primitives/test-helpers/src/lib.rs index cb644372f758..1adf86624cf8 100644 --- a/polkadot/primitives/test-helpers/src/lib.rs +++ b/polkadot/primitives/test-helpers/src/lib.rs @@ -29,7 +29,7 @@ use polkadot_primitives::{ SessionIndex, ValidationCode, ValidationCodeHash, ValidatorId, }; pub use rand; -use sp_application_crypto::sr25519; +use sp_application_crypto::{sr25519, ByteArray}; use sp_keyring::Sr25519Keyring; use sp_runtime::generic::Digest; @@ -172,7 +172,8 @@ pub fn dummy_head_data() -> HeadData { /// Create a meaningless collator id. pub fn dummy_collator() -> CollatorId { - CollatorId::from(sr25519::Public::default()) + CollatorId::from_slice(&mut (0..32).into_iter().collect::>().as_slice()) + .expect("32 bytes; qed") } /// Create a meaningless validator id. @@ -182,7 +183,8 @@ pub fn dummy_validator() -> ValidatorId { /// Create a meaningless collator signature. pub fn dummy_collator_signature() -> CollatorSignature { - CollatorSignature::from(sr25519::Signature::default()) + CollatorSignature::from_slice(&mut (0..64).into_iter().collect::>().as_slice()) + .expect("64 bytes; qed") } /// Create a meaningless persisted validation data. diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index ebc956a08e67..0b64201390a6 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -26,29 +26,30 @@ use alloc::{ vec, vec::Vec, }; + use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; use polkadot_primitives::{ node_features::FeatureIndex, vstaging::{ - BackedCandidate, CandidateDescriptorV2 as CandidateDescriptor, + BackedCandidate, CandidateDescriptorV2, CommittedCandidateReceiptV2 as CommittedCandidateReceipt, InherentData as ParachainsInherentData, }, - AvailabilityBitfield, CandidateCommitments, CandidateHash, CompactStatement, CoreIndex, - DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, Id as ParaId, IndexedVec, - InvalidDisputeStatementKind, PersistedValidationData, SessionIndex, SigningContext, - UncheckedSigned, ValidDisputeStatementKind, ValidationCode, ValidatorId, ValidatorIndex, - ValidityAttestation, + AvailabilityBitfield, CandidateCommitments, CandidateDescriptor, CandidateHash, + CompactStatement, CoreIndex, DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, + Id as ParaId, IndexedVec, InvalidDisputeStatementKind, PersistedValidationData, SessionIndex, + SigningContext, UncheckedSigned, ValidDisputeStatementKind, ValidationCode, ValidatorId, + ValidatorIndex, ValidityAttestation, }; +use polkadot_primitives_test_helpers::{dummy_collator, dummy_collator_signature}; use sp_core::H256; use sp_runtime::{ generic::Digest, traits::{Header as HeaderT, One, TrailingZeroInput, Zero}, RuntimeAppPublic, }; - fn mock_validation_code() -> ValidationCode { ValidationCode(vec![1, 2, 3]) } @@ -116,6 +117,8 @@ pub(crate) struct BenchBuilder { fill_claimqueue: bool, /// Cores which should not be available when being populated with pending candidates. unavailable_cores: Vec, + /// Use v2 candidate descriptor. + candidate_descriptor_v2: bool, _phantom: core::marker::PhantomData, } @@ -147,6 +150,7 @@ impl BenchBuilder { code_upgrade: None, fill_claimqueue: true, unavailable_cores: vec![], + candidate_descriptor_v2: false, _phantom: core::marker::PhantomData::, } } @@ -254,6 +258,12 @@ impl BenchBuilder { self } + /// Toggle usage of v2 candidate descriptors. + pub(crate) fn set_candidate_descriptor_v2(mut self, enable: bool) -> Self { + self.candidate_descriptor_v2 = enable; + self + } + /// Get the maximum number of validators per core. fn max_validators_per_core(&self) -> u32 { self.max_validators_per_core.unwrap_or(Self::fallback_max_validators_per_core()) @@ -289,18 +299,20 @@ impl BenchBuilder { HeadData(vec![0xFF; max_head_size as usize]) } - fn candidate_descriptor_mock() -> CandidateDescriptor { - CandidateDescriptor::::new( - 0.into(), - Default::default(), - CoreIndex(0), - 1, - Default::default(), - Default::default(), - Default::default(), - Default::default(), - mock_validation_code().hash(), - ) + fn candidate_descriptor_mock() -> CandidateDescriptorV2 { + // Use a v1 descriptor. + CandidateDescriptor:: { + para_id: 0.into(), + relay_parent: Default::default(), + collator: dummy_collator(), + persisted_validation_data_hash: Default::default(), + pov_hash: Default::default(), + erasure_root: Default::default(), + signature: dummy_collator_signature(), + para_head: Default::default(), + validation_code_hash: mock_validation_code().hash(), + } + .into() } /// Create a mock of `CandidatePendingAvailability`. @@ -625,18 +637,35 @@ impl BenchBuilder { let group_validators = scheduler::Pallet::::group_validators(group_idx).unwrap(); - let candidate = CommittedCandidateReceipt:: { - descriptor: CandidateDescriptor::::new( + let descriptor = if self.candidate_descriptor_v2 { + CandidateDescriptorV2::new( para_id, relay_parent, - CoreIndex(0), + core_idx, 1, persisted_validation_data_hash, pov_hash, Default::default(), head_data.hash(), validation_code_hash, - ), + ) + } else { + CandidateDescriptor:: { + para_id, + relay_parent, + collator: dummy_collator(), + persisted_validation_data_hash, + pov_hash, + erasure_root: Default::default(), + signature: dummy_collator_signature(), + para_head: head_data.hash(), + validation_code_hash, + } + .into() + }; + + let candidate = CommittedCandidateReceipt:: { + descriptor, commitments: CandidateCommitments:: { upward_messages: Default::default(), horizontal_messages: Default::default(), diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index 2125dffe2592..7f892a195068 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -26,7 +26,7 @@ use crate::{ shared::AllowedRelayParentsTracker, }; use polkadot_primitives::{ - effective_minimum_backing_votes, vstaging::CandidateDescriptorV2, AvailabilityBitfield, + effective_minimum_backing_votes, AvailabilityBitfield, CandidateDescriptor, SignedAvailabilityBitfields, UncheckedSignedAvailabilityBitfields, }; @@ -38,7 +38,9 @@ use polkadot_primitives::{ SignedAvailabilityBitfield, SignedStatement, ValidationCode, ValidatorId, ValidityAttestation, PARACHAIN_KEY_TYPE_ID, }; -use polkadot_primitives_test_helpers::dummy_validation_code; +use polkadot_primitives_test_helpers::{ + dummy_collator, dummy_collator_signature, dummy_validation_code, +}; use sc_keystore::LocalKeystore; use sp_keyring::Sr25519Keyring; use sp_keystore::{Keystore, KeystorePtr}; @@ -283,17 +285,18 @@ impl std::default::Default for TestCandidateBuilder { impl TestCandidateBuilder { pub(crate) fn build(self) -> CommittedCandidateReceipt { CommittedCandidateReceipt { - descriptor: CandidateDescriptorV2::new( - self.para_id, - self.relay_parent, - CoreIndex(0), - 1, - self.persisted_validation_data_hash, - self.pov_hash, - Default::default(), - self.para_head_hash.unwrap_or_else(|| self.head_data.hash()), - self.validation_code.hash(), - ), + descriptor: CandidateDescriptor { + para_id: self.para_id, + pov_hash: self.pov_hash, + relay_parent: self.relay_parent, + persisted_validation_data_hash: self.persisted_validation_data_hash, + validation_code_hash: self.validation_code.hash(), + para_head: self.para_head_hash.unwrap_or_else(|| self.head_data.hash()), + erasure_root: Default::default(), + signature: dummy_collator_signature(), + collator: dummy_collator(), + } + .into(), commitments: CandidateCommitments { head_data: self.head_data, new_validation_code: self.new_validation_code, diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 66330674898b..c00c3dc23130 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -51,7 +51,7 @@ use polkadot_primitives::{ effective_minimum_backing_votes, node_features::FeatureIndex, vstaging::{ - BackedCandidate, CandidateReceiptV2 as CandidateReceipt, + BackedCandidate, CandidateDescriptorVersion, CandidateReceiptV2 as CandidateReceipt, InherentData as ParachainsInherentData, ScrapedOnChainVotes, }, CandidateHash, CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, CoreIndex, @@ -575,6 +575,12 @@ impl Pallet { .map(|b| *b) .unwrap_or(false); + let allow_v2_receipts = configuration::ActiveConfig::::get() + .node_features + .get(FeatureIndex::CandidateReceiptV2 as usize) + .map(|b| *b) + .unwrap_or(false); + let mut eligible: BTreeMap> = BTreeMap::new(); let mut total_eligible_cores = 0; @@ -591,6 +597,7 @@ impl Pallet { concluded_invalid_hashes, eligible, core_index_enabled, + allow_v2_receipts, ); let count = count_backed_candidates(&backed_candidates_with_core); @@ -964,11 +971,18 @@ fn sanitize_backed_candidates( concluded_invalid_with_descendants: BTreeSet, scheduled: BTreeMap>, core_index_enabled: bool, + allow_v2_receipts: bool, ) -> BTreeMap, CoreIndex)>> { // Map the candidates to the right paraids, while making sure that the order between candidates // of the same para is preserved. let mut candidates_per_para: BTreeMap> = BTreeMap::new(); for candidate in backed_candidates { + // Drop any v2 candidate receipts if nodes are not allowed to use them. + if !allow_v2_receipts && candidate.descriptor().version() == CandidateDescriptorVersion::V2 + { + continue + } + candidates_per_para .entry(candidate.descriptor().para_id()) .or_default() diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 6c82233e0820..db8e2d9c3d70 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -70,6 +70,7 @@ mod enter { fill_claimqueue: bool, elastic_paras: BTreeMap, unavailable_cores: Vec, + v2_descriptor: bool, } fn make_inherent_data( @@ -82,6 +83,7 @@ mod enter { fill_claimqueue, elastic_paras, unavailable_cores, + v2_descriptor, }: TestConfig, ) -> Bench { let extra_cores = elastic_paras @@ -99,7 +101,8 @@ mod enter { .set_backed_and_concluding_paras(backed_and_concluding.clone()) .set_dispute_sessions(&dispute_sessions[..]) .set_fill_claimqueue(fill_claimqueue) - .set_unavailable_cores(unavailable_cores); + .set_unavailable_cores(unavailable_cores) + .set_candidate_descriptor_v2(v2_descriptor); // Setup some assignments as needed: mock_assigner::Pallet::::set_core_count(builder.max_cores()); @@ -145,6 +148,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); // We expect the scenario to have cores 0 & 1 with pending availability. The backed @@ -240,6 +244,7 @@ mod enter { fill_claimqueue: false, elastic_paras: [(2, 3)].into_iter().collect(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -344,6 +349,7 @@ mod enter { fill_claimqueue: true, elastic_paras: [(2, 4)].into_iter().collect(), unavailable_cores: unavailable_cores.clone(), + v2_descriptor: false, }); let mut expected_para_inherent_data = scenario.data.clone(); @@ -600,6 +606,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -673,6 +680,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -744,6 +752,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -831,6 +840,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -918,6 +928,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1004,6 +1015,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1111,6 +1123,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1179,6 +1192,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1245,6 +1259,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1348,6 +1363,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let mut para_inherent_data = scenario.data.clone(); @@ -1437,6 +1453,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1465,6 +1482,62 @@ mod enter { assert_eq!(dispatch_error, Error::::InherentOverweight.into()); }); } + + #[test] + fn v2_descriptors_are_filtered() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + let mut backed_and_concluding = BTreeMap::new(); + + for i in 0..10 { + backed_and_concluding.insert(i, i); + } + + let scenario = make_inherent_data(TestConfig { + dispute_statements: BTreeMap::new(), + dispute_sessions: vec![], + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, + fill_claimqueue: false, + elastic_paras: BTreeMap::new(), + unavailable_cores: vec![], + v2_descriptor: true, + }); + + let unfiltered_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 10 backed candidates) + assert_eq!(unfiltered_para_inherent_data.bitfields.len(), 50); + // * 10 v2 candidate descriptors. + assert_eq!(unfiltered_para_inherent_data.backed_candidates.len(), 10); + + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &unfiltered_para_inherent_data) + .unwrap(); + + // We expect all candidates to be filtered out (including the backed candidates.) + let filtered_para_inherend_data = + Pallet::::create_inherent_inner(&inherent_data).unwrap(); + + assert_eq!(filtered_para_inherend_data.backed_candidates.len(), 0); + + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &unfiltered_para_inherent_data) + .unwrap(); + let dispatch_error = Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + ) + .unwrap_err() + .error; + + // We expect `enter` to fail because it contains backed candidates with v2 descriptors. + assert_eq!(dispatch_error, Error::::CandidatesFilteredDuringExecution.into()); + }); + } } fn default_header() -> polkadot_primitives::Header { @@ -3376,7 +3449,8 @@ mod sanitizers { &shared::AllowedRelayParents::::get(), BTreeSet::new(), scheduled, - core_index_enabled + core_index_enabled, + false, ), expected_backed_candidates_with_core, ); @@ -3400,7 +3474,8 @@ mod sanitizers { &shared::AllowedRelayParents::::get(), BTreeSet::new(), scheduled, - core_index_enabled + core_index_enabled, + false, ), expected_backed_candidates_with_core, ); @@ -3425,6 +3500,7 @@ mod sanitizers { BTreeSet::new(), scheduled, core_index_enabled, + false, ), expected_backed_candidates_with_core ); @@ -3457,6 +3533,7 @@ mod sanitizers { BTreeSet::new(), scheduled, core_index_enabled, + false, ), expected_backed_candidates_with_core ); @@ -3497,6 +3574,7 @@ mod sanitizers { BTreeSet::new(), scheduled, core_index_enabled, + false, ); if core_index_enabled { @@ -3567,6 +3645,7 @@ mod sanitizers { BTreeSet::new(), scheduled, core_index_enabled, + false, ); if core_index_enabled { @@ -3605,6 +3684,7 @@ mod sanitizers { BTreeSet::new(), scheduled, core_index_enabled, + false, ); assert!(sanitized_backed_candidates.is_empty()); @@ -3641,6 +3721,7 @@ mod sanitizers { set, scheduled, core_index_enabled, + false, ); assert_eq!(sanitized_backed_candidates.len(), backed_candidates.len() / 2); @@ -3678,6 +3759,7 @@ mod sanitizers { invalid_set, scheduled, true, + false, ); // We'll be left with candidates from paraid 2 and 4. @@ -3714,6 +3796,7 @@ mod sanitizers { invalid_set, scheduled, true, + false, ); // Only the second candidate of paraid 1 should be removed. From f12ca7a502edbf0d8849e69919ec8fe174fd3bcf Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 14 Aug 2024 16:52:30 +0300 Subject: [PATCH 024/103] fix Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/paras_inherent/tests.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index db8e2d9c3d70..3e04394d9827 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -1523,13 +1523,9 @@ mod enter { assert_eq!(filtered_para_inherend_data.backed_candidates.len(), 0); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &unfiltered_para_inherent_data) - .unwrap(); let dispatch_error = Pallet::::enter( frame_system::RawOrigin::None.into(), - expected_para_inherent_data, + unfiltered_para_inherent_data, ) .unwrap_err() .error; From 13734dedfec81602238793901a5614a08c41c1e8 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 14 Aug 2024 19:05:15 +0300 Subject: [PATCH 025/103] fix prospective parachains tests Signed-off-by: Andrei Sandu --- polkadot/primitives/test-helpers/src/lib.rs | 24 +++++++++++++------ polkadot/runtime/parachains/src/builder.rs | 10 ++++---- .../runtime/parachains/src/inclusion/tests.rs | 6 ++--- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/polkadot/primitives/test-helpers/src/lib.rs b/polkadot/primitives/test-helpers/src/lib.rs index 1adf86624cf8..7aaa8f06160b 100644 --- a/polkadot/primitives/test-helpers/src/lib.rs +++ b/polkadot/primitives/test-helpers/src/lib.rs @@ -170,21 +170,31 @@ pub fn dummy_head_data() -> HeadData { HeadData(vec![]) } -/// Create a meaningless collator id. -pub fn dummy_collator() -> CollatorId { +/// Create a meaningless validator id. +pub fn dummy_validator() -> ValidatorId { + ValidatorId::from(sr25519::Public::default()) +} + +/// Create junk non-zeroed collator id. +pub fn junk_collator() -> CollatorId { CollatorId::from_slice(&mut (0..32).into_iter().collect::>().as_slice()) .expect("32 bytes; qed") } -/// Create a meaningless validator id. -pub fn dummy_validator() -> ValidatorId { - ValidatorId::from(sr25519::Public::default()) +/// Create junk non-zeroed collator signature. +pub fn junk_collator_signature() -> CollatorSignature { + CollatorSignature::from_slice(&mut (0..64).into_iter().collect::>().as_slice()) + .expect("64 bytes; qed") +} + +/// Create a meaningless collator id. +pub fn dummy_collator() -> CollatorId { + CollatorId::from(sr25519::Public::default()) } /// Create a meaningless collator signature. pub fn dummy_collator_signature() -> CollatorSignature { - CollatorSignature::from_slice(&mut (0..64).into_iter().collect::>().as_slice()) - .expect("64 bytes; qed") + CollatorSignature::from(sr25519::Signature::default()) } /// Create a meaningless persisted validation data. diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 0b64201390a6..432b7777aa9e 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -43,7 +43,7 @@ use polkadot_primitives::{ SigningContext, UncheckedSigned, ValidDisputeStatementKind, ValidationCode, ValidatorId, ValidatorIndex, ValidityAttestation, }; -use polkadot_primitives_test_helpers::{dummy_collator, dummy_collator_signature}; +use polkadot_primitives_test_helpers::{junk_collator, junk_collator_signature}; use sp_core::H256; use sp_runtime::{ generic::Digest, @@ -304,11 +304,11 @@ impl BenchBuilder { CandidateDescriptor:: { para_id: 0.into(), relay_parent: Default::default(), - collator: dummy_collator(), + collator: junk_collator(), persisted_validation_data_hash: Default::default(), pov_hash: Default::default(), erasure_root: Default::default(), - signature: dummy_collator_signature(), + signature: junk_collator_signature(), para_head: Default::default(), validation_code_hash: mock_validation_code().hash(), } @@ -653,11 +653,11 @@ impl BenchBuilder { CandidateDescriptor:: { para_id, relay_parent, - collator: dummy_collator(), + collator: junk_collator(), persisted_validation_data_hash, pov_hash, erasure_root: Default::default(), - signature: dummy_collator_signature(), + signature: junk_collator_signature(), para_head: head_data.hash(), validation_code_hash, } diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index 7f892a195068..e3c1a9bdefa9 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -39,7 +39,7 @@ use polkadot_primitives::{ PARACHAIN_KEY_TYPE_ID, }; use polkadot_primitives_test_helpers::{ - dummy_collator, dummy_collator_signature, dummy_validation_code, + dummy_validation_code, junk_collator, junk_collator_signature, }; use sc_keystore::LocalKeystore; use sp_keyring::Sr25519Keyring; @@ -293,8 +293,8 @@ impl TestCandidateBuilder { validation_code_hash: self.validation_code.hash(), para_head: self.para_head_hash.unwrap_or_else(|| self.head_data.hash()), erasure_root: Default::default(), - signature: dummy_collator_signature(), - collator: dummy_collator(), + signature: junk_collator_signature(), + collator: junk_collator(), } .into(), commitments: CandidateCommitments { From effb1cc57e040e43724e0027a2b95f4571b74ac7 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 14 Aug 2024 23:06:10 +0300 Subject: [PATCH 026/103] fix fix Signed-off-by: Andrei Sandu --- polkadot/primitives/test-helpers/src/lib.rs | 14 +------------- polkadot/runtime/parachains/src/builder.rs | 17 ++++++++++++++--- .../runtime/parachains/src/inclusion/tests.rs | 13 +++++++------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/polkadot/primitives/test-helpers/src/lib.rs b/polkadot/primitives/test-helpers/src/lib.rs index 7aaa8f06160b..b0f78717dd97 100644 --- a/polkadot/primitives/test-helpers/src/lib.rs +++ b/polkadot/primitives/test-helpers/src/lib.rs @@ -29,7 +29,7 @@ use polkadot_primitives::{ SessionIndex, ValidationCode, ValidationCodeHash, ValidatorId, }; pub use rand; -use sp_application_crypto::{sr25519, ByteArray}; +use sp_application_crypto::sr25519; use sp_keyring::Sr25519Keyring; use sp_runtime::generic::Digest; @@ -175,18 +175,6 @@ pub fn dummy_validator() -> ValidatorId { ValidatorId::from(sr25519::Public::default()) } -/// Create junk non-zeroed collator id. -pub fn junk_collator() -> CollatorId { - CollatorId::from_slice(&mut (0..32).into_iter().collect::>().as_slice()) - .expect("32 bytes; qed") -} - -/// Create junk non-zeroed collator signature. -pub fn junk_collator_signature() -> CollatorSignature { - CollatorSignature::from_slice(&mut (0..64).into_iter().collect::>().as_slice()) - .expect("64 bytes; qed") -} - /// Create a meaningless collator id. pub fn dummy_collator() -> CollatorId { CollatorId::from(sr25519::Public::default()) diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 432b7777aa9e..eaef0d99dccc 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -26,7 +26,7 @@ use alloc::{ vec, vec::Vec, }; - +use sp_core::ByteArray; use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; @@ -41,9 +41,8 @@ use polkadot_primitives::{ CompactStatement, CoreIndex, DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, Id as ParaId, IndexedVec, InvalidDisputeStatementKind, PersistedValidationData, SessionIndex, SigningContext, UncheckedSigned, ValidDisputeStatementKind, ValidationCode, ValidatorId, - ValidatorIndex, ValidityAttestation, + ValidatorIndex, ValidityAttestation, CollatorId, CollatorSignature, }; -use polkadot_primitives_test_helpers::{junk_collator, junk_collator_signature}; use sp_core::H256; use sp_runtime::{ generic::Digest, @@ -54,6 +53,18 @@ fn mock_validation_code() -> ValidationCode { ValidationCode(vec![1, 2, 3]) } +// Create a dummy collator id suitable to be used in a V1 candidate descriptor. +fn junk_collator() -> CollatorId { + CollatorId::from_slice(&mut (0..32).into_iter().collect::>().as_slice()) + .expect("32 bytes; qed") +} + +// Creates a dummy collator signature suitable to be used in a V1 candidate descriptor. +fn junk_collator_signature() -> CollatorSignature { + CollatorSignature::from_slice(&mut (0..64).into_iter().collect::>().as_slice()) + .expect("64 bytes; qed") +} + /// Grab an account, seeded by a name and index. /// /// This is directly from frame-benchmarking. Copy/pasted so we can use it when not compiling with diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index e3c1a9bdefa9..80d6ed752b84 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -36,11 +36,10 @@ use frame_support::assert_noop; use polkadot_primitives::{ BlockNumber, CandidateCommitments, CollatorId, CompactStatement as Statement, Hash, SignedAvailabilityBitfield, SignedStatement, ValidationCode, ValidatorId, ValidityAttestation, - PARACHAIN_KEY_TYPE_ID, -}; -use polkadot_primitives_test_helpers::{ - dummy_validation_code, junk_collator, junk_collator_signature, + PARACHAIN_KEY_TYPE_ID, CollatorSignature, }; +use polkadot_primitives_test_helpers::dummy_validation_code; +use sp_core::ByteArray; use sc_keystore::LocalKeystore; use sp_keyring::Sr25519Keyring; use sp_keystore::{Keystore, KeystorePtr}; @@ -293,8 +292,10 @@ impl TestCandidateBuilder { validation_code_hash: self.validation_code.hash(), para_head: self.para_head_hash.unwrap_or_else(|| self.head_data.hash()), erasure_root: Default::default(), - signature: junk_collator_signature(), - collator: junk_collator(), + signature: CollatorSignature::from_slice(&mut (0..64).into_iter().collect::>().as_slice()) + .expect("64 bytes; qed"), + collator: CollatorId::from_slice(&mut (0..32).into_iter().collect::>().as_slice()) + .expect("32 bytes; qed"), } .into(), commitments: CandidateCommitments { From 3f75cba787c806cc4e15be5bf076d1e83ac32da5 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 14 Aug 2024 23:07:26 +0300 Subject: [PATCH 027/103] fmt Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/builder.rs | 13 ++++++------ .../runtime/parachains/src/inclusion/tests.rs | 20 +++++++++++-------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index eaef0d99dccc..35835793ce9d 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -26,7 +26,6 @@ use alloc::{ vec, vec::Vec, }; -use sp_core::ByteArray; use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; @@ -37,13 +36,13 @@ use polkadot_primitives::{ CommittedCandidateReceiptV2 as CommittedCandidateReceipt, InherentData as ParachainsInherentData, }, - AvailabilityBitfield, CandidateCommitments, CandidateDescriptor, CandidateHash, - CompactStatement, CoreIndex, DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, - Id as ParaId, IndexedVec, InvalidDisputeStatementKind, PersistedValidationData, SessionIndex, - SigningContext, UncheckedSigned, ValidDisputeStatementKind, ValidationCode, ValidatorId, - ValidatorIndex, ValidityAttestation, CollatorId, CollatorSignature, + AvailabilityBitfield, CandidateCommitments, CandidateDescriptor, CandidateHash, CollatorId, + CollatorSignature, CompactStatement, CoreIndex, DisputeStatement, DisputeStatementSet, + GroupIndex, HeadData, Id as ParaId, IndexedVec, InvalidDisputeStatementKind, + PersistedValidationData, SessionIndex, SigningContext, UncheckedSigned, + ValidDisputeStatementKind, ValidationCode, ValidatorId, ValidatorIndex, ValidityAttestation, }; -use sp_core::H256; +use sp_core::{ByteArray, H256}; use sp_runtime::{ generic::Digest, traits::{Header as HeaderT, One, TrailingZeroInput, Zero}, diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index 80d6ed752b84..f17b8ed2e0e9 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -34,13 +34,13 @@ use assert_matches::assert_matches; use codec::DecodeAll; use frame_support::assert_noop; use polkadot_primitives::{ - BlockNumber, CandidateCommitments, CollatorId, CompactStatement as Statement, Hash, - SignedAvailabilityBitfield, SignedStatement, ValidationCode, ValidatorId, ValidityAttestation, - PARACHAIN_KEY_TYPE_ID, CollatorSignature, + BlockNumber, CandidateCommitments, CollatorId, CollatorSignature, + CompactStatement as Statement, Hash, SignedAvailabilityBitfield, SignedStatement, + ValidationCode, ValidatorId, ValidityAttestation, PARACHAIN_KEY_TYPE_ID, }; use polkadot_primitives_test_helpers::dummy_validation_code; -use sp_core::ByteArray; use sc_keystore::LocalKeystore; +use sp_core::ByteArray; use sp_keyring::Sr25519Keyring; use sp_keystore::{Keystore, KeystorePtr}; use std::sync::Arc; @@ -292,10 +292,14 @@ impl TestCandidateBuilder { validation_code_hash: self.validation_code.hash(), para_head: self.para_head_hash.unwrap_or_else(|| self.head_data.hash()), erasure_root: Default::default(), - signature: CollatorSignature::from_slice(&mut (0..64).into_iter().collect::>().as_slice()) - .expect("64 bytes; qed"), - collator: CollatorId::from_slice(&mut (0..32).into_iter().collect::>().as_slice()) - .expect("32 bytes; qed"), + signature: CollatorSignature::from_slice( + &mut (0..64).into_iter().collect::>().as_slice(), + ) + .expect("64 bytes; qed"), + collator: CollatorId::from_slice( + &mut (0..32).into_iter().collect::>().as_slice(), + ) + .expect("32 bytes; qed"), } .into(), commitments: CandidateCommitments { From 75a47bb1859e325dda0cee1a1a8e902c5627f972 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 14 Aug 2024 23:10:24 +0300 Subject: [PATCH 028/103] fix comment Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/paras_inherent/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 3e04394d9827..e3b4d8579055 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -1517,7 +1517,7 @@ mod enter { .put_data(PARACHAINS_INHERENT_IDENTIFIER, &unfiltered_para_inherent_data) .unwrap(); - // We expect all candidates to be filtered out (including the backed candidates.) + // We expect all backed candidates to be filtered out. let filtered_para_inherend_data = Pallet::::create_inherent_inner(&inherent_data).unwrap(); From 12ed853d0641295f1b6262174636da8be3ad97bb Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 14 Aug 2024 23:12:57 +0300 Subject: [PATCH 029/103] another one Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/paras_inherent/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index e3b4d8579055..a5923d68a12a 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -1530,7 +1530,7 @@ mod enter { .unwrap_err() .error; - // We expect `enter` to fail because it contains backed candidates with v2 descriptors. + // We expect `enter` to fail because the inherent data contains backed candidates with v2 descriptors. assert_eq!(dispatch_error, Error::::CandidatesFilteredDuringExecution.into()); }); } From f2c0882c8e573166f2ab0f859a2bd0db69dc47d7 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 15 Aug 2024 13:07:41 +0300 Subject: [PATCH 030/103] fix build Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index aec3c7aeb612..8ea628bdc28a 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -181,7 +181,6 @@ impl From> for CandidateDescriptor { } } -#[cfg(feature = "test")] impl From> for CandidateDescriptorV2 { fn from(value: CandidateDescriptor) -> Self { Decode::decode(&mut value.encode().as_slice()).unwrap() From 768e03446a39a454337a158326d1482ba48228af Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 15 Aug 2024 14:05:09 +0300 Subject: [PATCH 031/103] . Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 2 +- polkadot/runtime/parachains/src/paras_inherent/tests.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 8ea628bdc28a..cc97f4a6666c 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -38,7 +38,6 @@ use sp_arithmetic::Perbill; use sp_core::RuntimeDebug; use sp_runtime::traits::Header as HeaderT; use sp_staking::SessionIndex; - /// Async backing primitives pub mod async_backing; @@ -181,6 +180,7 @@ impl From> for CandidateDescriptor { } } +#[cfg(any(feature = "runtime-benchmarks", feature = "test"))] impl From> for CandidateDescriptorV2 { fn from(value: CandidateDescriptor) -> Self { Decode::decode(&mut value.encode().as_slice()).unwrap() diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index a5923d68a12a..39c9fc52d685 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -1530,7 +1530,8 @@ mod enter { .unwrap_err() .error; - // We expect `enter` to fail because the inherent data contains backed candidates with v2 descriptors. + // We expect `enter` to fail because the inherent data contains backed candidates with + // v2 descriptors. assert_eq!(dispatch_error, Error::::CandidatesFilteredDuringExecution.into()); }); } From 4bf0706876422e84e80f32fc62b44e23e6fc3c9a Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 15 Aug 2024 17:43:57 +0300 Subject: [PATCH 032/103] improve test and add comment Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 12 ++++++ .../parachains/src/paras_inherent/mod.rs | 13 +++--- .../parachains/src/paras_inherent/tests.rs | 40 ++++++++++++++----- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index cc97f4a6666c..d5e39979ab34 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -221,6 +221,12 @@ impl CandidateDescriptorV2 { pub fn set_pov_hash(&mut self, pov_hash: Hash) { self.pov_hash = pov_hash; } + + /// Set the version in the descriptor. Only for tests. + #[cfg(feature = "test")] + pub fn set_version(&mut self, version: InternalVersion) { + self.version = version; + } } /// A candidate-receipt at version 2. @@ -624,6 +630,12 @@ impl BackedCandidate { &self.candidate.descriptor } + /// Get a mutable reference to the descriptor of the candidate. Only for testing. + #[cfg(feature = "test")] + pub fn descriptor_mut(&mut self) -> &mut CandidateDescriptorV2 { + &mut self.candidate.descriptor + } + /// Get a reference to the validity votes of the candidate. pub fn validity_votes(&self) -> &[ValidityAttestation] { &self.validity_votes diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index c00c3dc23130..7c1b7455de38 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -954,14 +954,15 @@ pub(crate) fn sanitize_bitfields( /// subsequent candidates after the filtered one. /// /// Filter out: -/// 1. any candidates which don't form a chain with the other candidates of the paraid (even if they +/// 1. Candidates that have v2 descriptors if the node `CandidateReceiptV2` feature is not enabled. +/// 2. any candidates which don't form a chain with the other candidates of the paraid (even if they /// do form a chain but are not in the right order). -/// 2. any candidates that have a concluded invalid dispute or who are descendants of a concluded +/// 3. any candidates that have a concluded invalid dispute or who are descendants of a concluded /// invalid candidate. -/// 3. any unscheduled candidates, as well as candidates whose paraid has multiple cores assigned +/// 4. any unscheduled candidates, as well as candidates whose paraid has multiple cores assigned /// but have no injected core index. -/// 4. all backing votes from disabled validators -/// 5. any candidates that end up with less than `effective_minimum_backing_votes` backing votes +/// 5. all backing votes from disabled validators +/// 6. any candidates that end up with less than `effective_minimum_backing_votes` backing votes /// /// Returns the scheduled /// backed candidates which passed filtering, mapped by para id and in the right dependency order. @@ -978,6 +979,8 @@ fn sanitize_backed_candidates( let mut candidates_per_para: BTreeMap> = BTreeMap::new(); for candidate in backed_candidates { // Drop any v2 candidate receipts if nodes are not allowed to use them. + // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure + // any v1 descendants of v2 candidates are dropped. if !allow_v2_receipts && candidate.descriptor().version() == CandidateDescriptorVersion::V2 { continue diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 39c9fc52d685..c0e1636569a8 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -58,7 +58,10 @@ mod enter { use core::panic; use frame_support::assert_ok; use frame_system::limits; - use polkadot_primitives::{vstaging::SchedulerParams, AvailabilityBitfield, UncheckedSigned}; + use polkadot_primitives::{ + vstaging::{InternalVersion, SchedulerParams}, + AvailabilityBitfield, UncheckedSigned, + }; use sp_runtime::Perbill; struct TestConfig { @@ -1485,26 +1488,38 @@ mod enter { #[test] fn v2_descriptors_are_filtered() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { + let config = default_config(); + assert!(config.configuration.config.scheduler_params.lookahead > 0); + new_test_ext(config).execute_with(|| { + // Set the elastic scaling MVP feature. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::ElasticScalingMVP as u8, + true, + ) + .unwrap(); + let mut backed_and_concluding = BTreeMap::new(); + backed_and_concluding.insert(0, 1); + backed_and_concluding.insert(1, 1); + backed_and_concluding.insert(2, 1); - for i in 0..10 { - backed_and_concluding.insert(i, i); - } + let unavailable_cores = vec![]; let scenario = make_inherent_data(TestConfig { dispute_statements: BTreeMap::new(), - dispute_sessions: vec![], + dispute_sessions: vec![], // No disputes backed_and_concluding, num_validators_per_core: 5, code_upgrade: None, - fill_claimqueue: false, - elastic_paras: BTreeMap::new(), - unavailable_cores: vec![], + fill_claimqueue: true, + // 8 cores ! + elastic_paras: [(2, 8)].into_iter().collect(), + unavailable_cores: unavailable_cores.clone(), v2_descriptor: true, }); - let unfiltered_para_inherent_data = scenario.data.clone(); + let mut unfiltered_para_inherent_data = scenario.data.clone(); // Check the para inherent data is as expected: // * 1 bitfield per validator (5 validators per core, 10 backed candidates) @@ -1512,6 +1527,11 @@ mod enter { // * 10 v2 candidate descriptors. assert_eq!(unfiltered_para_inherent_data.backed_candidates.len(), 10); + // Make the last candidate look like v1, by using an unknown version. + unfiltered_para_inherent_data.backed_candidates[9] + .descriptor_mut() + .set_version(InternalVersion(123)); + let mut inherent_data = InherentData::new(); inherent_data .put_data(PARACHAINS_INHERENT_IDENTIFIER, &unfiltered_para_inherent_data) From 0c83201531b3bc9d40d8f6c1d14cfdc1589cfaa9 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 15 Aug 2024 17:47:48 +0300 Subject: [PATCH 033/103] add log Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/paras_inherent/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 7c1b7455de38..2bcc2d873131 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -983,6 +983,12 @@ fn sanitize_backed_candidates( // any v1 descendants of v2 candidates are dropped. if !allow_v2_receipts && candidate.descriptor().version() == CandidateDescriptorVersion::V2 { + log::debug!( + target: LOG_TARGET, + "V2 candidate descriptors not allowed. Dropping candidate {:?} for paraid {:?}.", + candidate.candidate().hash(), + candidate.descriptor().para_id() + ); continue } From 429694215f8ffba2ac3e8639da79df3ec3e32887 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 20 Aug 2024 00:58:10 +0300 Subject: [PATCH 034/103] simplify check() Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 86 ++++--------------------- 1 file changed, 12 insertions(+), 74 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index b089fb422f3c..e2dcf124b545 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -27,11 +27,7 @@ use super::{ use bitvec::prelude::*; use sp_application_crypto::ByteArray; -use alloc::{ - collections::{btree_map::BTreeMap, vec_deque::VecDeque}, - vec, - vec::Vec, -}; +use alloc::{vec, vec::Vec}; use codec::{Decode, Encode}; use scale_info::TypeInfo; use sp_arithmetic::Perbill; @@ -497,69 +493,29 @@ impl CandidateDescriptorV2 { } impl CommittedCandidateReceiptV2 { - /// Performs a sanity check of the descriptor and commitment. - /// - /// Returns error if: - /// - descriptor core index is different than the core selected - /// by the commitments - /// - the core index is out of bounds - pub fn check( - &self, - n_cores: u32, - // TODO: consider promoting `ClaimQueueSnapshot` as primitive - claim_queue: &BTreeMap>, - ) -> Result<(), CandidateReceiptError> { + /// Checks if descriptor core index is equal to the commited core index. + /// Input `assigned_cores` must contain the sorted cores assigned to the para at + /// the committed claim queue offset. + pub fn check(&self, assigned_cores: &Vec) -> Result<(), CandidateReceiptError> { // Don't check v1 descriptors. if self.descriptor.version() == CandidateDescriptorVersion::V1 { return Ok(()) } - if claim_queue.is_empty() { + if assigned_cores.is_empty() { return Err(CandidateReceiptError::NoAssignment) } let descriptor_core_index = CoreIndex(self.descriptor.core_index as u32); - if descriptor_core_index.0 > n_cores - 1 { - return Err(CandidateReceiptError::InvalidCoreIndex) - } - let (core_selector, cq_offset) = + let (core_selector, _cq_offset) = self.commitments.selected_core().ok_or(CandidateReceiptError::NoCoreSelected)?; - // TODO: double check the invariant of claim queue len == scheduling lookahead. - // What happens when on-demand is used and there is only 1 claim on a core. - let claim_queue_depth = claim_queue - .first_key_value() - .ok_or(CandidateReceiptError::NoAssignment)? - .1 - .len(); - if cq_offset.0 as usize >= claim_queue_depth { - return Err(CandidateReceiptError::InvalidSelectedCore) - } - - let para_id = self.descriptor.para_id; - - // Get a sorted vec of the core indices the parachain is assigned to at `cq_offset`. - let assigned_cores = claim_queue - .iter() - .filter_map(|(core_index, queue)| { - if queue.get(cq_offset.0 as usize)? == ¶_id { - Some(core_index) - } else { - None - } - }) - .collect::>(); - - if assigned_cores.is_empty() { - return Err(CandidateReceiptError::NoAssignment) - } - let core_index = assigned_cores .get(core_selector.0 as usize % assigned_cores.len()) - .expect("provided index is always less than queue len; qed"); + .ok_or(CandidateReceiptError::InvalidCoreIndex)?; - if **core_index != descriptor_core_index { + if *core_index != descriptor_core_index { return Err(CandidateReceiptError::CoreIndexMismatch) } @@ -869,13 +825,7 @@ mod tests { .upward_messages .force_push(UMPSignal::SelectCore(CoreSelector(0), ClaimQueueOffset(1)).encode()); - let mut claim_queue = BTreeMap::new(); - claim_queue.insert( - new_ccr.descriptor.core_index().unwrap(), - vec![2.into(), new_ccr.descriptor.para_id, 3.into()].into(), - ); - - assert_eq!(new_ccr.check(200, &claim_queue), Ok(())); + assert_eq!(new_ccr.check(&vec![CoreIndex(123)]), Ok(())); } #[test] @@ -897,12 +847,6 @@ mod tests { .upward_messages .force_push(UMPSignal::SelectCore(CoreSelector(0), ClaimQueueOffset(1)).encode()); - let mut claim_queue = BTreeMap::new(); - claim_queue.insert( - new_ccr.descriptor.core_index().unwrap(), - vec![2.into(), new_ccr.descriptor.para_id, 3.into()].into(), - ); - let encoded_ccr = new_ccr.encode(); let decoded_ccr: CommittedCandidateReceipt = Decode::decode(&mut encoded_ccr.as_slice()).unwrap(); @@ -918,7 +862,7 @@ mod tests { Decode::decode(&mut encoded_ccr.as_slice()).unwrap(); assert_eq!(v2_ccr.descriptor.core_index(), Some(CoreIndex(123))); - assert_eq!(new_ccr.check(200, &claim_queue), Ok(())); + assert_eq!(new_ccr.check(&vec![CoreIndex(123)]), Ok(())); assert_eq!(new_ccr.hash(), v2_ccr.hash()); } @@ -945,15 +889,9 @@ mod tests { // Since collator sig and id are zeroed, it means that the descriptor uses format // version 2. - let mut claim_queue = BTreeMap::new(); - claim_queue.insert( - new_ccr.descriptor.core_index().unwrap(), - vec![2.into(), new_ccr.descriptor.para_id(), 3.into()].into(), - ); - // We expect the check to fail in such case because there will be no `SelectCore` // commitment. - assert_eq!(new_ccr.check(200, &claim_queue), Err(CandidateReceiptError::NoCoreSelected)); + assert_eq!(new_ccr.check(&vec![CoreIndex(0)]), Err(CandidateReceiptError::NoCoreSelected)); // Adding collator signature should make it decode as v1. old_ccr.descriptor.signature = dummy_collator_signature(); From 6fb7790eb1aa7aa88acb7f355149f3f70f25b19b Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 20 Aug 2024 18:02:12 +0300 Subject: [PATCH 035/103] impl Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index e2dcf124b545..fe828c02142e 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -492,11 +492,11 @@ impl CandidateDescriptorV2 { } } -impl CommittedCandidateReceiptV2 { +impl CommittedCandidateReceiptV2 { /// Checks if descriptor core index is equal to the commited core index. /// Input `assigned_cores` must contain the sorted cores assigned to the para at /// the committed claim queue offset. - pub fn check(&self, assigned_cores: &Vec) -> Result<(), CandidateReceiptError> { + pub fn check(&self, assigned_cores: &[CoreIndex]) -> Result<(), CandidateReceiptError> { // Don't check v1 descriptors. if self.descriptor.version() == CandidateDescriptorVersion::V1 { return Ok(()) From d0b3961e3c1ab98d9d5bb4506a6af5e9273a1e2c Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 20 Aug 2024 19:19:28 +0300 Subject: [PATCH 036/103] comment Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index fe828c02142e..8503fab2ba70 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -137,7 +137,9 @@ pub struct CandidateDescriptorV2 { /// The hash of the relay-chain block this is executed in the context of. relay_parent: H, /// Version field. The raw value here is not exposed, instead it is used - /// to determine the `CandidateDescriptorVersion`, see `fn version()` + /// to determine the `CandidateDescriptorVersion`, see `fn version()`. + /// For the current version this field is set to `0` and will be incremented + /// by next versions. version: InternalVersion, /// The core index where the candidate is backed. core_index: u16, From 66f7a96d5f6eddb402873b513b4a8ab8b3d57c70 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 21 Aug 2024 02:21:16 +0300 Subject: [PATCH 037/103] add some tests Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 40 +++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 8503fab2ba70..aa47d9cbc3e2 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -808,6 +808,18 @@ mod tests { assert_eq!(old_ccr.hash(), new_ccr.hash()); } + #[test] + fn is_invalid_version_decodes_as_v1() { + let mut new_ccr = dummy_committed_candidate_receipt_v2(); + new_ccr.descriptor.version = InternalVersion(100); + + // Deserialize as V1. + let new_ccr: CommittedCandidateReceiptV2 = + Decode::decode(&mut new_ccr.encode().as_slice()).unwrap(); + + assert_eq!(new_ccr.descriptor.version(), CandidateDescriptorVersion::V1); + } + #[test] fn test_ump_commitment() { let mut new_ccr = dummy_committed_candidate_receipt_v2(); @@ -830,6 +842,34 @@ mod tests { assert_eq!(new_ccr.check(&vec![CoreIndex(123)]), Ok(())); } + #[test] + fn test_invalid_ump_commitment() { + let mut new_ccr = dummy_committed_candidate_receipt_v2(); + new_ccr.descriptor.core_index = 0; + new_ccr.descriptor.para_id = ParaId::new(1000); + + // separator + new_ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); + + + // The check should fail because no `SelectCore` signal was sent. + assert_eq!(new_ccr.check(&vec![CoreIndex(0), CoreIndex(100)]), Err(CandidateReceiptError::NoCoreSelected)); + + new_ccr + .commitments + .upward_messages + .force_push(UMPSignal::SelectCore(CoreSelector(0), ClaimQueueOffset(1)).encode()); + // Duplicate + new_ccr + .commitments + .upward_messages + .force_push(UMPSignal::SelectCore(CoreSelector(1), ClaimQueueOffset(1)).encode()); + + // Duplicate doesn't override first signal. + assert_eq!(new_ccr.check(&vec![CoreIndex(0), CoreIndex(100)]), Ok(())); + + } + #[test] fn test_version2_receipts_decoded_as_v1() { let mut new_ccr = dummy_committed_candidate_receipt_v2(); From 5c0c919114afbc0062e7c2f76af1a47a91faba92 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 21 Aug 2024 02:43:46 +0300 Subject: [PATCH 038/103] update Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 48 +++++++++++++++++-------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index aa47d9cbc3e2..623b0a1a3044 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -761,6 +761,16 @@ mod tests { vstaging::{CandidateDescriptorV2, CommittedCandidateReceiptV2}, }; + fn dummy_collator_signature() -> CollatorSignature { + CollatorSignature::from_slice(&mut (0..64).into_iter().collect::>().as_slice()) + .expect("64 bytes; qed") + } + + fn dummy_collator_id() -> CollatorId { + CollatorId::from_slice(&mut (0..32).into_iter().collect::>().as_slice()) + .expect("32 bytes; qed") + } + pub fn dummy_committed_candidate_receipt_v2() -> CommittedCandidateReceiptV2 { let zeros = Hash::zero(); let reserved64b = [0; 64]; @@ -811,6 +821,7 @@ mod tests { #[test] fn is_invalid_version_decodes_as_v1() { let mut new_ccr = dummy_committed_candidate_receipt_v2(); + // Put some unknown version. new_ccr.descriptor.version = InternalVersion(100); // Deserialize as V1. @@ -848,17 +859,35 @@ mod tests { new_ccr.descriptor.core_index = 0; new_ccr.descriptor.para_id = ParaId::new(1000); - // separator new_ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); - + new_ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); // The check should fail because no `SelectCore` signal was sent. - assert_eq!(new_ccr.check(&vec![CoreIndex(0), CoreIndex(100)]), Err(CandidateReceiptError::NoCoreSelected)); + assert_eq!( + new_ccr.check(&vec![CoreIndex(0), CoreIndex(100)]), + Err(CandidateReceiptError::NoCoreSelected) + ); + + // Garbage message. + new_ccr.commitments.upward_messages.force_push(vec![0, 13, 200].encode()); + + // No `SelectCore` can be decoded. + assert_eq!(new_ccr.commitments.selected_core(), None); + + // Failure is expected. + assert_eq!( + new_ccr.check(&vec![CoreIndex(0), CoreIndex(100)]), + Err(CandidateReceiptError::NoCoreSelected) + ); + + new_ccr.commitments.upward_messages.clear(); + new_ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); new_ccr .commitments .upward_messages .force_push(UMPSignal::SelectCore(CoreSelector(0), ClaimQueueOffset(1)).encode()); + // Duplicate new_ccr .commitments @@ -867,7 +896,6 @@ mod tests { // Duplicate doesn't override first signal. assert_eq!(new_ccr.check(&vec![CoreIndex(0), CoreIndex(100)]), Ok(())); - } #[test] @@ -898,7 +926,7 @@ mod tests { assert_eq!(new_ccr.hash(), decoded_ccr.hash()); - // // // Encode v1 and decode as V2 + // Encode v1 and decode as V2 let encoded_ccr = new_ccr.encode(); let v2_ccr: CommittedCandidateReceiptV2 = Decode::decode(&mut encoded_ccr.as_slice()).unwrap(); @@ -909,16 +937,6 @@ mod tests { assert_eq!(new_ccr.hash(), v2_ccr.hash()); } - fn dummy_collator_signature() -> CollatorSignature { - CollatorSignature::from_slice(&mut (0..64).into_iter().collect::>().as_slice()) - .expect("64 bytes; qed") - } - - fn dummy_collator_id() -> CollatorId { - CollatorId::from_slice(&mut (0..32).into_iter().collect::>().as_slice()) - .expect("32 bytes; qed") - } - #[test] fn test_core_select_is_mandatory() { // Testing edge case when collators provide zeroed signature and collator id. From 38ce5899bf7c58a02183b0fb227db514609d0d02 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 21 Aug 2024 16:34:56 +0300 Subject: [PATCH 039/103] prdoc Signed-off-by: Andrei Sandu --- prdoc/pr_5322.prdoc | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 prdoc/pr_5322.prdoc diff --git a/prdoc/pr_5322.prdoc b/prdoc/pr_5322.prdoc new file mode 100644 index 000000000000..0e262f18bf92 --- /dev/null +++ b/prdoc/pr_5322.prdoc @@ -0,0 +1,33 @@ +title: Elastic scaling: introduce new candidate receipt primitive + +doc: + - audience: [Runtime Dev, Node Dev] + description: | + Introduces `CandidateDescriptorV2` primitive as described in [RFC 103](https://github.com/polkadot-fellows/RFCs/pull/103). + Updates parachains runtime, Westend, Rococo and test runtimes to use the new primitives. + This change does not implement the functionality of the new candidate receipts. + +crates: +- name: polkadot-primitives + bump: minor +- name: polkadot-primitives-test-helpers + bump: minor +- name: polkadot-runtime-parachains + bump: major +- name: rococo-runtime + bump: major +- name: westend-runtime + bump: major +- name: polkadot-test-runtime + bump: major +- name: polkadot-service + bump: patch +- name: polkadot-node-subsystem-types + bump: patch +- name: polkadot-test-client + bump: major +- name: cumulus-relay-chain-inprocess-interface + bump: patch + + + From 9f1d611ee48726993d71b17a473a5ce835e4584e Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 21 Aug 2024 17:04:55 +0300 Subject: [PATCH 040/103] can't be happy if CI is sad Signed-off-by: Andrei Sandu --- polkadot/primitives/Cargo.toml | 1 + polkadot/primitives/src/vstaging/mod.rs | 2 +- polkadot/runtime/parachains/Cargo.toml | 2 +- prdoc/pr_5322.prdoc | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/polkadot/primitives/Cargo.toml b/polkadot/primitives/Cargo.toml index 311fc210d42b..a8cd6cb5f4e0 100644 --- a/polkadot/primitives/Cargo.toml +++ b/polkadot/primitives/Cargo.toml @@ -58,6 +58,7 @@ std = [ "sp-keystore?/std", "sp-runtime/std", "sp-staking/std", + "sp-std/std", ] runtime-benchmarks = [ "polkadot-parachain-primitives/runtime-benchmarks", diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 623b0a1a3044..61a3ddb408b6 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -312,7 +312,7 @@ impl CommittedCandidateReceiptV2 { self.to_plain().hash() } - /// Does this committed candidate receipt corresponds to the given [`CandidateReceipt`]? + /// Does this committed candidate receipt corresponds to the given [`CandidateReceiptV2`]? pub fn corresponds_to(&self, receipt: &CandidateReceiptV2) -> bool where H: PartialEq, diff --git a/polkadot/runtime/parachains/Cargo.toml b/polkadot/runtime/parachains/Cargo.toml index f3123fa554ca..a3eec3f9d961 100644 --- a/polkadot/runtime/parachains/Cargo.toml +++ b/polkadot/runtime/parachains/Cargo.toml @@ -59,7 +59,7 @@ polkadot-runtime-metrics = { workspace = true } polkadot-core-primitives = { workspace = true } [dev-dependencies] -polkadot-primitives = { workspace = true, features=["test"] } +polkadot-primitives = { workspace = true, features = ["test"] } futures = { workspace = true } hex-literal = { workspace = true, default-features = true } diff --git a/prdoc/pr_5322.prdoc b/prdoc/pr_5322.prdoc index 0e262f18bf92..68991ad02c93 100644 --- a/prdoc/pr_5322.prdoc +++ b/prdoc/pr_5322.prdoc @@ -1,4 +1,4 @@ -title: Elastic scaling: introduce new candidate receipt primitive +title: Elastic scaling - introduce new candidate receipt primitive doc: - audience: [Runtime Dev, Node Dev] From 663817dd49bcba63772544b07afbddf79ccf1c54 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 21 Aug 2024 17:08:29 +0300 Subject: [PATCH 041/103] remove newlines Signed-off-by: Andrei Sandu --- prdoc/pr_5322.prdoc | 3 --- 1 file changed, 3 deletions(-) diff --git a/prdoc/pr_5322.prdoc b/prdoc/pr_5322.prdoc index 68991ad02c93..b4cf261f33a4 100644 --- a/prdoc/pr_5322.prdoc +++ b/prdoc/pr_5322.prdoc @@ -28,6 +28,3 @@ crates: bump: major - name: cumulus-relay-chain-inprocess-interface bump: patch - - - From a1dacc1eb24a28c06c67ef8189b202df08717ff0 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 21 Aug 2024 17:10:28 +0300 Subject: [PATCH 042/103] match rfc 103 reserved field naming Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 61a3ddb408b6..af2e17d17ddf 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -146,7 +146,7 @@ pub struct CandidateDescriptorV2 { /// The session index of the candidate relay parent. session_index: SessionIndex, /// Reserved bytes. - reserved25b: [u8; 25], + reserved1: [u8; 25], /// The blake2-256 hash of the persisted validation data. This is extra data derived from /// relay-chain state which may vary based on bitfields included before the candidate. /// Thus it cannot be derived entirely from the relay-parent. @@ -156,7 +156,7 @@ pub struct CandidateDescriptorV2 { /// The root of a block's erasure encoding Merkle tree. erasure_root: Hash, /// Reserved bytes. - reserved64b: [u8; 64], + reserved2: [u8; 64], /// Hash of the para header that is being generated by this candidate. para_head: Hash, /// The blake2-256 hash of the validation code bytes. @@ -198,11 +198,11 @@ impl CandidateDescriptorV2 { version: InternalVersion(0), core_index: core_index.0 as u16, session_index, - reserved25b: [0; 25], + reserved1: [0; 25], persisted_validation_data_hash, pov_hash, erasure_root, - reserved64b: [0; 64], + reserved2: [0; 64], para_head, validation_code_hash, } @@ -428,7 +428,7 @@ impl CandidateDescriptorV2 { /// The candidate is at version 2 if the reserved fields are zeroed out /// and the internal `version` field is 0. pub fn version(&self) -> CandidateDescriptorVersion { - if self.reserved64b != [0u8; 64] || self.reserved25b != [0u8; 25] { + if self.reserved2 != [0u8; 64] || self.reserved1 != [0u8; 25] { return CandidateDescriptorVersion::V1 } @@ -446,7 +446,7 @@ impl CandidateDescriptorV2 { collator_id.push(self.version.0); collator_id.extend_from_slice(core_index.as_slice()); collator_id.extend_from_slice(session_index.as_slice()); - collator_id.extend_from_slice(self.reserved25b.as_slice()); + collator_id.extend_from_slice(self.reserved1.as_slice()); CollatorId::from_slice(&collator_id.as_slice()) .expect("Slice size is exactly 32 bytes; qed") @@ -462,7 +462,7 @@ impl CandidateDescriptorV2 { } fn rebuild_signature_field(&self) -> CollatorSignature { - CollatorSignature::from_slice(self.reserved64b.as_slice()) + CollatorSignature::from_slice(self.reserved2.as_slice()) .expect("Slice size is exactly 64 bytes; qed") } @@ -773,7 +773,7 @@ mod tests { pub fn dummy_committed_candidate_receipt_v2() -> CommittedCandidateReceiptV2 { let zeros = Hash::zero(); - let reserved64b = [0; 64]; + let reserved2 = [0; 64]; CommittedCandidateReceiptV2 { descriptor: CandidateDescriptorV2 { @@ -782,11 +782,11 @@ mod tests { version: InternalVersion(0), core_index: 123, session_index: 1, - reserved25b: Default::default(), + reserved1: Default::default(), persisted_validation_data_hash: zeros, pov_hash: zeros, erasure_root: zeros, - reserved64b, + reserved2, para_head: zeros, validation_code_hash: ValidationCode(vec![1, 2, 3, 4, 5, 6, 7, 8, 9]).hash(), }, From 33b80ea99d4e53d9dd897a90fa3a6e373379825f Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 21 Aug 2024 17:14:39 +0300 Subject: [PATCH 043/103] remove default cq offset Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index af2e17d17ddf..962c9a467fe7 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -357,9 +357,6 @@ pub struct CoreSelector(pub u8); #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] pub struct ClaimQueueOffset(pub u8); -/// Default claim queue offset -pub const DEFAULT_CLAIM_QUEUE_OFFSET: ClaimQueueOffset = ClaimQueueOffset(1); - /// Signals that a parachain can send to the relay chain via the UMP queue. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] pub enum UMPSignal { From 29e4b47355408ff0712f190ee67a0fb0b19a079c Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 16 Aug 2024 18:36:11 +0300 Subject: [PATCH 044/103] Ignore UMP signals when checking and processing UMP queue Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/inclusion/mod.rs | 10 ++++++++++ polkadot/runtime/parachains/src/ump_tests.rs | 12 ++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/polkadot/runtime/parachains/src/inclusion/mod.rs b/polkadot/runtime/parachains/src/inclusion/mod.rs index d3edad7b3d6c..4dbd0ea638ef 100644 --- a/polkadot/runtime/parachains/src/inclusion/mod.rs +++ b/polkadot/runtime/parachains/src/inclusion/mod.rs @@ -925,6 +925,15 @@ impl Pallet { para: ParaId, upward_messages: &[UpwardMessage], ) -> Result<(), UmpAcceptanceCheckErr> { + // Filter any pending UMP signals and the separator. + let upward_messages = if let Some(separator_index) = + upward_messages.iter().rposition(|message| message.is_empty()) + { + upward_messages.split_at(separator_index).0 + } else { + upward_messages + }; + // Cannot send UMP messages while off-boarding. if paras::Pallet::::is_offboarding(para) { ensure!(upward_messages.is_empty(), UmpAcceptanceCheckErr::IsOffboarding); @@ -979,6 +988,7 @@ impl Pallet { pub(crate) fn receive_upward_messages(para: ParaId, upward_messages: &[Vec]) -> Weight { let bounded = upward_messages .iter() + .take_while(|message| !message.is_empty()) .filter_map(|d| { BoundedSlice::try_from(&d[..]) .map_err(|e| { diff --git a/polkadot/runtime/parachains/src/ump_tests.rs b/polkadot/runtime/parachains/src/ump_tests.rs index d914bf8b6661..71494a3faf67 100644 --- a/polkadot/runtime/parachains/src/ump_tests.rs +++ b/polkadot/runtime/parachains/src/ump_tests.rs @@ -141,12 +141,12 @@ mod check_upward_messages { configuration::ActiveConfig::::get().max_upward_message_num_per_candidate; for sent in 0..permitted + 1 { - check(P_0, vec![msg(""); sent as usize], None); + check(P_0, vec![msg("a"); sent as usize], None); } for sent in permitted + 1..permitted + 10 { check( P_0, - vec![msg(""); sent as usize], + vec![msg("a"); sent as usize], Some(UmpAcceptanceCheckErr::MoreMessagesThanPermitted { sent, permitted }), ); } @@ -185,18 +185,18 @@ mod check_upward_messages { let limit = configuration::ActiveConfig::::get().max_upward_queue_count as u64; for _ in 0..limit { - check(P_0, vec![msg("")], None); - queue(P_0, vec![msg("")]); + check(P_0, vec![msg("a")], None); + queue(P_0, vec![msg("a")]); } check( P_0, - vec![msg("")], + vec![msg("a")], Some(UmpAcceptanceCheckErr::CapacityExceeded { count: limit + 1, limit }), ); check( P_0, - vec![msg(""); 2], + vec![msg("a"); 2], Some(UmpAcceptanceCheckErr::CapacityExceeded { count: limit + 2, limit }), ); }); From ab85fe346f94af7927441cadeed3b763f324d92f Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 20 Aug 2024 18:00:34 +0300 Subject: [PATCH 045/103] wip Signed-off-by: Andrei Sandu --- .../runtime/parachains/src/inclusion/mod.rs | 1 + .../parachains/src/paras_inherent/mod.rs | 66 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/polkadot/runtime/parachains/src/inclusion/mod.rs b/polkadot/runtime/parachains/src/inclusion/mod.rs index 4dbd0ea638ef..f9f54790b570 100644 --- a/polkadot/runtime/parachains/src/inclusion/mod.rs +++ b/polkadot/runtime/parachains/src/inclusion/mod.rs @@ -988,6 +988,7 @@ impl Pallet { pub(crate) fn receive_upward_messages(para: ParaId, upward_messages: &[Vec]) -> Weight { let bounded = upward_messages .iter() + // Stop once we hit the `UMPSignal`` separator. .take_while(|message| !message.is_empty()) .filter_map(|d| { BoundedSlice::try_from(&d[..]) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 2bcc2d873131..f191c690a583 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -45,6 +45,7 @@ use frame_support::{ pallet_prelude::*, traits::Randomness, }; + use frame_system::pallet_prelude::*; use pallet_babe::{self, ParentBlockRandomness}; use polkadot_primitives::{ @@ -977,6 +978,9 @@ fn sanitize_backed_candidates( // Map the candidates to the right paraids, while making sure that the order between candidates // of the same para is preserved. let mut candidates_per_para: BTreeMap> = BTreeMap::new(); + + // Get the paras scheduled next on each core. + let scheduled_paras = scheduler::Pallet::::scheduled_paras().collect::>(); for candidate in backed_candidates { // Drop any v2 candidate receipts if nodes are not allowed to use them. // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure @@ -992,6 +996,68 @@ fn sanitize_backed_candidates( continue } + if candidate.descriptor().version() == CandidateDescriptorVersion::V2 { + let current_block_num = frame_system::Pallet::::block_number(); + let Some((_core_selector, cq_offset)) = + candidate.candidate().commitments.selected_core() + else { + log::debug!( + target: LOG_TARGET, + "Dropping candidate {:?} for paraid {:?}, no `CoreSelector` commitment.", + candidate.candidate().hash(), + candidate.descriptor().para_id() + ); + + continue + }; + + // Get the relay parent number for the candidate: + let Some((_state_root, relay_parent_num)) = + allowed_relay_parents.acquire_info(candidate.descriptor().relay_parent(), None) + else { + log::debug!( + target: LOG_TARGET, + "Relay parent {:?} for candidate {:?} is not in the allowed relay parents.", + candidate.descriptor().relay_parent(), + candidate.candidate().hash(), + ); + continue + }; + + let expected_cq_offset = current_block_num - relay_parent_num - One::one(); + // Drop the v2 candidate receipt if the core claim has not reached the top of the + // claim queue. + if expected_cq_offset == (cq_offset.0 as u32).into() { + log::debug!( + target: LOG_TARGET, + "Dropped candidate {:?} of paraid {:?} because the claimed core is not at top \ + of the claim queue, cq_offset: {:?}, relay_parent_num: {:?}", + candidate.candidate().hash(), + candidate.descriptor().para_id(), + cq_offset, + relay_parent_num + ); + } + + let assigned_cores = scheduled_paras + .iter() + .filter(|(core_index, para)| **para == candidate.descriptor().para_id()) + .collect::>(); + + // Check if core index in descriptoir matches the one in commitments + if let Err(err) = candidate.candidate().check(&assigned_cores) { + log::debug!( + target: LOG_TARGET, + "Dropping candidate {:?} for paraid {:?}, {:?}", + candidate.candidate().hash(), + candidate.descriptor().para_id(), + err, + ); + + continue + } + } + candidates_per_para .entry(candidate.descriptor().para_id()) .or_default() From 7d5636bfbb74ca81970a30149a5fcc4859fe5e61 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 20 Aug 2024 18:14:22 +0300 Subject: [PATCH 046/103] refactor a bit Signed-off-by: Andrei Sandu --- .../parachains/src/paras_inherent/mod.rs | 172 ++++++++++-------- 1 file changed, 99 insertions(+), 73 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index f191c690a583..84d4f4a7a234 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -949,6 +949,99 @@ pub(crate) fn sanitize_bitfields( bitfields } +/// Perform required checks for given version 2 candidate receipt. +/// Returns `true` if candidate has passed all checks. +fn sanitize_backed_candidate_v2( + candidate: &BackedCandidate, + allowed_relay_parents: &AllowedRelayParentsTracker>, + scheduled_paras: &BTreeMap, + allow_v2_receipts: bool, +) -> bool { + // Drop any v2 candidate receipts if nodes are not allowed to use them. + // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure + // any v1 descendants of v2 candidates are dropped. + if !allow_v2_receipts && candidate.descriptor().version() == CandidateDescriptorVersion::V2 { + log::debug!( + target: LOG_TARGET, + "V2 candidate descriptors not allowed. Dropping candidate {:?} for paraid {:?}.", + candidate.candidate().hash(), + candidate.descriptor().para_id() + ); + return false + } + + if candidate.descriptor().version() == CandidateDescriptorVersion::V1 { + return true; + } + + let current_block_num = frame_system::Pallet::::block_number(); + let Some((_core_selector, cq_offset)) = candidate.candidate().commitments.selected_core() + else { + log::debug!( + target: LOG_TARGET, + "Dropping candidate {:?} for paraid {:?}, no `CoreSelector` commitment.", + candidate.candidate().hash(), + candidate.descriptor().para_id() + ); + + return false + }; + + // Get the relay parent number for the candidate: + let Some((_state_root, relay_parent_num)) = + allowed_relay_parents.acquire_info(candidate.descriptor().relay_parent(), None) + else { + log::debug!( + target: LOG_TARGET, + "Relay parent {:?} for candidate {:?} is not in the allowed relay parents.", + candidate.descriptor().relay_parent(), + candidate.candidate().hash(), + ); + return false + }; + + let expected_cq_offset = current_block_num - relay_parent_num - One::one(); + // Drop the v2 candidate receipt if the core claim has not reached the top of the + // claim queue. + if expected_cq_offset != (cq_offset.0 as u32).into() { + log::debug!( + target: LOG_TARGET, + "Dropped candidate {:?} of paraid {:?} because the claimed core is not at top \ + of the claim queue, cq_offset: {:?}, relay_parent_num: {:?}", + candidate.candidate().hash(), + candidate.descriptor().para_id(), + cq_offset, + relay_parent_num + ); + return false + } + + let assigned_cores = scheduled_paras + .iter() + .filter_map(|(core_index, para)| { + if *para == candidate.descriptor().para_id() { + Some(*core_index) + } else { + None + } + }) + .collect::>(); + + // Check if core index in descriptoir matches the one in commitments + if let Err(err) = candidate.candidate().check(&assigned_cores) { + log::debug!( + target: LOG_TARGET, + "Dropping candidate {:?} for paraid {:?}, {:?}", + candidate.candidate().hash(), + candidate.descriptor().para_id(), + err, + ); + + return false + } + true +} + /// Performs various filtering on the backed candidates inherent data. /// Must maintain the invariant that the returned candidate collection contains the candidates /// sorted in dependency order for each para. When doing any filtering, we must therefore drop any @@ -982,82 +1075,15 @@ fn sanitize_backed_candidates( // Get the paras scheduled next on each core. let scheduled_paras = scheduler::Pallet::::scheduled_paras().collect::>(); for candidate in backed_candidates { - // Drop any v2 candidate receipts if nodes are not allowed to use them. - // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure - // any v1 descendants of v2 candidates are dropped. - if !allow_v2_receipts && candidate.descriptor().version() == CandidateDescriptorVersion::V2 - { - log::debug!( - target: LOG_TARGET, - "V2 candidate descriptors not allowed. Dropping candidate {:?} for paraid {:?}.", - candidate.candidate().hash(), - candidate.descriptor().para_id() - ); + if !sanitize_backed_candidate_v2::( + &candidate, + allowed_relay_parents, + &scheduled_paras, + allow_v2_receipts, + ) { continue } - if candidate.descriptor().version() == CandidateDescriptorVersion::V2 { - let current_block_num = frame_system::Pallet::::block_number(); - let Some((_core_selector, cq_offset)) = - candidate.candidate().commitments.selected_core() - else { - log::debug!( - target: LOG_TARGET, - "Dropping candidate {:?} for paraid {:?}, no `CoreSelector` commitment.", - candidate.candidate().hash(), - candidate.descriptor().para_id() - ); - - continue - }; - - // Get the relay parent number for the candidate: - let Some((_state_root, relay_parent_num)) = - allowed_relay_parents.acquire_info(candidate.descriptor().relay_parent(), None) - else { - log::debug!( - target: LOG_TARGET, - "Relay parent {:?} for candidate {:?} is not in the allowed relay parents.", - candidate.descriptor().relay_parent(), - candidate.candidate().hash(), - ); - continue - }; - - let expected_cq_offset = current_block_num - relay_parent_num - One::one(); - // Drop the v2 candidate receipt if the core claim has not reached the top of the - // claim queue. - if expected_cq_offset == (cq_offset.0 as u32).into() { - log::debug!( - target: LOG_TARGET, - "Dropped candidate {:?} of paraid {:?} because the claimed core is not at top \ - of the claim queue, cq_offset: {:?}, relay_parent_num: {:?}", - candidate.candidate().hash(), - candidate.descriptor().para_id(), - cq_offset, - relay_parent_num - ); - } - - let assigned_cores = scheduled_paras - .iter() - .filter(|(core_index, para)| **para == candidate.descriptor().para_id()) - .collect::>(); - - // Check if core index in descriptoir matches the one in commitments - if let Err(err) = candidate.candidate().check(&assigned_cores) { - log::debug!( - target: LOG_TARGET, - "Dropping candidate {:?} for paraid {:?}, {:?}", - candidate.candidate().hash(), - candidate.descriptor().para_id(), - err, - ); - - continue - } - } - candidates_per_para .entry(candidate.descriptor().para_id()) .or_default() From 2954bba27f4de28b8a7d8a760cc14d7e29ed15a5 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 20 Aug 2024 18:45:04 +0300 Subject: [PATCH 047/103] use descriptor core_index in `map_candidates_to_cores` Signed-off-by: Andrei Sandu --- .../parachains/src/paras_inherent/mod.rs | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 84d4f4a7a234..44346519490b 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -957,6 +957,10 @@ fn sanitize_backed_candidate_v2( scheduled_paras: &BTreeMap, allow_v2_receipts: bool, ) -> bool { + if candidate.descriptor().version() == CandidateDescriptorVersion::V1 { + return true; + } + // Drop any v2 candidate receipts if nodes are not allowed to use them. // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure // any v1 descendants of v2 candidates are dropped. @@ -970,10 +974,6 @@ fn sanitize_backed_candidate_v2( return false } - if candidate.descriptor().version() == CandidateDescriptorVersion::V1 { - return true; - } - let current_block_num = frame_system::Pallet::::block_number(); let Some((_core_selector, cq_offset)) = candidate.candidate().commitments.selected_core() else { @@ -1496,10 +1496,18 @@ fn map_candidates_to_cores = - get_injected_core_index::(allowed_relay_parents, &candidate); - if let Some(core_index) = maybe_injected_core_index { + // If candidate is v2, use the descriptor core index. We've already checked the + // core index is valid in `sanitize_backed_candidate_v2`. If candidate is v1, + // expect an injected core index. + let maybe_core_index = + if candidate.descriptor().version() == CandidateDescriptorVersion::V2 { + candidate.descriptor().core_index() + } else { + get_injected_core_index::(allowed_relay_parents, &candidate) + }; + + if let Some(core_index) = maybe_core_index { if scheduled_cores.remove(&core_index) { temp_backed_candidates.push((candidate, core_index)); } else { @@ -1508,7 +1516,7 @@ fn map_candidates_to_cores Date: Wed, 21 Aug 2024 02:52:04 +0300 Subject: [PATCH 048/103] nits Signed-off-by: Andrei Sandu --- .../runtime/parachains/src/paras_inherent/mod.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 44346519490b..763464cb8111 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -949,8 +949,14 @@ pub(crate) fn sanitize_bitfields( bitfields } -/// Perform required checks for given version 2 candidate receipt. -/// Returns `true` if candidate has passed all checks. +/// Perform required checks for given candidate receipt. +/// +/// Returns `true` if candidate descriptor is version 1. +/// +/// For version 2 it returns `false` if: +/// - version 2 descriptors are not allowed +/// - the core index doesn't match the one computed from the commitments +/// - the `SelectCore` signal does not refer to a core at the top of claim queue fn sanitize_backed_candidate_v2( candidate: &BackedCandidate, allowed_relay_parents: &AllowedRelayParentsTracker>, @@ -964,7 +970,7 @@ fn sanitize_backed_candidate_v2( // Drop any v2 candidate receipts if nodes are not allowed to use them. // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure // any v1 descendants of v2 candidates are dropped. - if !allow_v2_receipts && candidate.descriptor().version() == CandidateDescriptorVersion::V2 { + if !allow_v2_receipts { log::debug!( target: LOG_TARGET, "V2 candidate descriptors not allowed. Dropping candidate {:?} for paraid {:?}.", @@ -1027,7 +1033,7 @@ fn sanitize_backed_candidate_v2( }) .collect::>(); - // Check if core index in descriptoir matches the one in commitments + // Check if core index in descriptor matches the one in the commitments if let Err(err) = candidate.candidate().check(&assigned_cores) { log::debug!( target: LOG_TARGET, From 1db5eb0e1c1c8ec45a320f4cc510543e4129efad Mon Sep 17 00:00:00 2001 From: Andrei Sandu <54316454+sandreim@users.noreply.github.com> Date: Thu, 22 Aug 2024 13:23:49 +0300 Subject: [PATCH 049/103] Para Inherent: filter v2 candidate descriptors (#5362) On top of https://github.com/paritytech/polkadot-sdk/pull/5322 This is an additional check to prevent nodes from passing the v2 candidate descriptors in the runtime until it is enabled via the node feature. TODO: - [ ] PRDoc --------- Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 20 +++- polkadot/primitives/test-helpers/src/lib.rs | 10 +- polkadot/runtime/parachains/src/builder.rs | 87 ++++++++++---- .../runtime/parachains/src/inclusion/tests.rs | 38 +++--- .../parachains/src/paras_inherent/mod.rs | 35 +++++- .../parachains/src/paras_inherent/tests.rs | 108 +++++++++++++++++- 6 files changed, 243 insertions(+), 55 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 962c9a467fe7..8d97ddb57fc5 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -34,7 +34,6 @@ use sp_arithmetic::Perbill; use sp_core::RuntimeDebug; use sp_runtime::traits::Header as HeaderT; use sp_staking::SessionIndex; - /// Async backing primitives pub mod async_backing; @@ -179,6 +178,13 @@ impl From> for CandidateDescriptor { } } +#[cfg(any(feature = "runtime-benchmarks", feature = "test"))] +impl From> for CandidateDescriptorV2 { + fn from(value: CandidateDescriptor) -> Self { + Decode::decode(&mut value.encode().as_slice()).unwrap() + } +} + impl CandidateDescriptorV2 { /// Constructor pub fn new( @@ -213,6 +219,12 @@ impl CandidateDescriptorV2 { pub fn set_pov_hash(&mut self, pov_hash: Hash) { self.pov_hash = pov_hash; } + + /// Set the version in the descriptor. Only for tests. + #[cfg(feature = "test")] + pub fn set_version(&mut self, version: InternalVersion) { + self.version = version; + } } /// A candidate-receipt at version 2. @@ -573,6 +585,12 @@ impl BackedCandidate { &self.candidate.descriptor } + /// Get a mutable reference to the descriptor of the candidate. Only for testing. + #[cfg(feature = "test")] + pub fn descriptor_mut(&mut self) -> &mut CandidateDescriptorV2 { + &mut self.candidate.descriptor + } + /// Get a reference to the validity votes of the candidate. pub fn validity_votes(&self) -> &[ValidityAttestation] { &self.validity_votes diff --git a/polkadot/primitives/test-helpers/src/lib.rs b/polkadot/primitives/test-helpers/src/lib.rs index cb644372f758..b0f78717dd97 100644 --- a/polkadot/primitives/test-helpers/src/lib.rs +++ b/polkadot/primitives/test-helpers/src/lib.rs @@ -170,16 +170,16 @@ pub fn dummy_head_data() -> HeadData { HeadData(vec![]) } -/// Create a meaningless collator id. -pub fn dummy_collator() -> CollatorId { - CollatorId::from(sr25519::Public::default()) -} - /// Create a meaningless validator id. pub fn dummy_validator() -> ValidatorId { ValidatorId::from(sr25519::Public::default()) } +/// Create a meaningless collator id. +pub fn dummy_collator() -> CollatorId { + CollatorId::from(sr25519::Public::default()) +} + /// Create a meaningless collator signature. pub fn dummy_collator_signature() -> CollatorSignature { CollatorSignature::from(sr25519::Signature::default()) diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index ebc956a08e67..35835793ce9d 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -32,27 +32,38 @@ use frame_system::pallet_prelude::*; use polkadot_primitives::{ node_features::FeatureIndex, vstaging::{ - BackedCandidate, CandidateDescriptorV2 as CandidateDescriptor, + BackedCandidate, CandidateDescriptorV2, CommittedCandidateReceiptV2 as CommittedCandidateReceipt, InherentData as ParachainsInherentData, }, - AvailabilityBitfield, CandidateCommitments, CandidateHash, CompactStatement, CoreIndex, - DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, Id as ParaId, IndexedVec, - InvalidDisputeStatementKind, PersistedValidationData, SessionIndex, SigningContext, - UncheckedSigned, ValidDisputeStatementKind, ValidationCode, ValidatorId, ValidatorIndex, - ValidityAttestation, + AvailabilityBitfield, CandidateCommitments, CandidateDescriptor, CandidateHash, CollatorId, + CollatorSignature, CompactStatement, CoreIndex, DisputeStatement, DisputeStatementSet, + GroupIndex, HeadData, Id as ParaId, IndexedVec, InvalidDisputeStatementKind, + PersistedValidationData, SessionIndex, SigningContext, UncheckedSigned, + ValidDisputeStatementKind, ValidationCode, ValidatorId, ValidatorIndex, ValidityAttestation, }; -use sp_core::H256; +use sp_core::{ByteArray, H256}; use sp_runtime::{ generic::Digest, traits::{Header as HeaderT, One, TrailingZeroInput, Zero}, RuntimeAppPublic, }; - fn mock_validation_code() -> ValidationCode { ValidationCode(vec![1, 2, 3]) } +// Create a dummy collator id suitable to be used in a V1 candidate descriptor. +fn junk_collator() -> CollatorId { + CollatorId::from_slice(&mut (0..32).into_iter().collect::>().as_slice()) + .expect("32 bytes; qed") +} + +// Creates a dummy collator signature suitable to be used in a V1 candidate descriptor. +fn junk_collator_signature() -> CollatorSignature { + CollatorSignature::from_slice(&mut (0..64).into_iter().collect::>().as_slice()) + .expect("64 bytes; qed") +} + /// Grab an account, seeded by a name and index. /// /// This is directly from frame-benchmarking. Copy/pasted so we can use it when not compiling with @@ -116,6 +127,8 @@ pub(crate) struct BenchBuilder { fill_claimqueue: bool, /// Cores which should not be available when being populated with pending candidates. unavailable_cores: Vec, + /// Use v2 candidate descriptor. + candidate_descriptor_v2: bool, _phantom: core::marker::PhantomData, } @@ -147,6 +160,7 @@ impl BenchBuilder { code_upgrade: None, fill_claimqueue: true, unavailable_cores: vec![], + candidate_descriptor_v2: false, _phantom: core::marker::PhantomData::, } } @@ -254,6 +268,12 @@ impl BenchBuilder { self } + /// Toggle usage of v2 candidate descriptors. + pub(crate) fn set_candidate_descriptor_v2(mut self, enable: bool) -> Self { + self.candidate_descriptor_v2 = enable; + self + } + /// Get the maximum number of validators per core. fn max_validators_per_core(&self) -> u32 { self.max_validators_per_core.unwrap_or(Self::fallback_max_validators_per_core()) @@ -289,18 +309,20 @@ impl BenchBuilder { HeadData(vec![0xFF; max_head_size as usize]) } - fn candidate_descriptor_mock() -> CandidateDescriptor { - CandidateDescriptor::::new( - 0.into(), - Default::default(), - CoreIndex(0), - 1, - Default::default(), - Default::default(), - Default::default(), - Default::default(), - mock_validation_code().hash(), - ) + fn candidate_descriptor_mock() -> CandidateDescriptorV2 { + // Use a v1 descriptor. + CandidateDescriptor:: { + para_id: 0.into(), + relay_parent: Default::default(), + collator: junk_collator(), + persisted_validation_data_hash: Default::default(), + pov_hash: Default::default(), + erasure_root: Default::default(), + signature: junk_collator_signature(), + para_head: Default::default(), + validation_code_hash: mock_validation_code().hash(), + } + .into() } /// Create a mock of `CandidatePendingAvailability`. @@ -625,18 +647,35 @@ impl BenchBuilder { let group_validators = scheduler::Pallet::::group_validators(group_idx).unwrap(); - let candidate = CommittedCandidateReceipt:: { - descriptor: CandidateDescriptor::::new( + let descriptor = if self.candidate_descriptor_v2 { + CandidateDescriptorV2::new( para_id, relay_parent, - CoreIndex(0), + core_idx, 1, persisted_validation_data_hash, pov_hash, Default::default(), head_data.hash(), validation_code_hash, - ), + ) + } else { + CandidateDescriptor:: { + para_id, + relay_parent, + collator: junk_collator(), + persisted_validation_data_hash, + pov_hash, + erasure_root: Default::default(), + signature: junk_collator_signature(), + para_head: head_data.hash(), + validation_code_hash, + } + .into() + }; + + let candidate = CommittedCandidateReceipt:: { + descriptor, commitments: CandidateCommitments:: { upward_messages: Default::default(), horizontal_messages: Default::default(), diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index 2125dffe2592..f17b8ed2e0e9 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -26,7 +26,7 @@ use crate::{ shared::AllowedRelayParentsTracker, }; use polkadot_primitives::{ - effective_minimum_backing_votes, vstaging::CandidateDescriptorV2, AvailabilityBitfield, + effective_minimum_backing_votes, AvailabilityBitfield, CandidateDescriptor, SignedAvailabilityBitfields, UncheckedSignedAvailabilityBitfields, }; @@ -34,12 +34,13 @@ use assert_matches::assert_matches; use codec::DecodeAll; use frame_support::assert_noop; use polkadot_primitives::{ - BlockNumber, CandidateCommitments, CollatorId, CompactStatement as Statement, Hash, - SignedAvailabilityBitfield, SignedStatement, ValidationCode, ValidatorId, ValidityAttestation, - PARACHAIN_KEY_TYPE_ID, + BlockNumber, CandidateCommitments, CollatorId, CollatorSignature, + CompactStatement as Statement, Hash, SignedAvailabilityBitfield, SignedStatement, + ValidationCode, ValidatorId, ValidityAttestation, PARACHAIN_KEY_TYPE_ID, }; use polkadot_primitives_test_helpers::dummy_validation_code; use sc_keystore::LocalKeystore; +use sp_core::ByteArray; use sp_keyring::Sr25519Keyring; use sp_keystore::{Keystore, KeystorePtr}; use std::sync::Arc; @@ -283,17 +284,24 @@ impl std::default::Default for TestCandidateBuilder { impl TestCandidateBuilder { pub(crate) fn build(self) -> CommittedCandidateReceipt { CommittedCandidateReceipt { - descriptor: CandidateDescriptorV2::new( - self.para_id, - self.relay_parent, - CoreIndex(0), - 1, - self.persisted_validation_data_hash, - self.pov_hash, - Default::default(), - self.para_head_hash.unwrap_or_else(|| self.head_data.hash()), - self.validation_code.hash(), - ), + descriptor: CandidateDescriptor { + para_id: self.para_id, + pov_hash: self.pov_hash, + relay_parent: self.relay_parent, + persisted_validation_data_hash: self.persisted_validation_data_hash, + validation_code_hash: self.validation_code.hash(), + para_head: self.para_head_hash.unwrap_or_else(|| self.head_data.hash()), + erasure_root: Default::default(), + signature: CollatorSignature::from_slice( + &mut (0..64).into_iter().collect::>().as_slice(), + ) + .expect("64 bytes; qed"), + collator: CollatorId::from_slice( + &mut (0..32).into_iter().collect::>().as_slice(), + ) + .expect("32 bytes; qed"), + } + .into(), commitments: CandidateCommitments { head_data: self.head_data, new_validation_code: self.new_validation_code, diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 66330674898b..2bcc2d873131 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -51,7 +51,7 @@ use polkadot_primitives::{ effective_minimum_backing_votes, node_features::FeatureIndex, vstaging::{ - BackedCandidate, CandidateReceiptV2 as CandidateReceipt, + BackedCandidate, CandidateDescriptorVersion, CandidateReceiptV2 as CandidateReceipt, InherentData as ParachainsInherentData, ScrapedOnChainVotes, }, CandidateHash, CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, CoreIndex, @@ -575,6 +575,12 @@ impl Pallet { .map(|b| *b) .unwrap_or(false); + let allow_v2_receipts = configuration::ActiveConfig::::get() + .node_features + .get(FeatureIndex::CandidateReceiptV2 as usize) + .map(|b| *b) + .unwrap_or(false); + let mut eligible: BTreeMap> = BTreeMap::new(); let mut total_eligible_cores = 0; @@ -591,6 +597,7 @@ impl Pallet { concluded_invalid_hashes, eligible, core_index_enabled, + allow_v2_receipts, ); let count = count_backed_candidates(&backed_candidates_with_core); @@ -947,14 +954,15 @@ pub(crate) fn sanitize_bitfields( /// subsequent candidates after the filtered one. /// /// Filter out: -/// 1. any candidates which don't form a chain with the other candidates of the paraid (even if they +/// 1. Candidates that have v2 descriptors if the node `CandidateReceiptV2` feature is not enabled. +/// 2. any candidates which don't form a chain with the other candidates of the paraid (even if they /// do form a chain but are not in the right order). -/// 2. any candidates that have a concluded invalid dispute or who are descendants of a concluded +/// 3. any candidates that have a concluded invalid dispute or who are descendants of a concluded /// invalid candidate. -/// 3. any unscheduled candidates, as well as candidates whose paraid has multiple cores assigned +/// 4. any unscheduled candidates, as well as candidates whose paraid has multiple cores assigned /// but have no injected core index. -/// 4. all backing votes from disabled validators -/// 5. any candidates that end up with less than `effective_minimum_backing_votes` backing votes +/// 5. all backing votes from disabled validators +/// 6. any candidates that end up with less than `effective_minimum_backing_votes` backing votes /// /// Returns the scheduled /// backed candidates which passed filtering, mapped by para id and in the right dependency order. @@ -964,11 +972,26 @@ fn sanitize_backed_candidates( concluded_invalid_with_descendants: BTreeSet, scheduled: BTreeMap>, core_index_enabled: bool, + allow_v2_receipts: bool, ) -> BTreeMap, CoreIndex)>> { // Map the candidates to the right paraids, while making sure that the order between candidates // of the same para is preserved. let mut candidates_per_para: BTreeMap> = BTreeMap::new(); for candidate in backed_candidates { + // Drop any v2 candidate receipts if nodes are not allowed to use them. + // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure + // any v1 descendants of v2 candidates are dropped. + if !allow_v2_receipts && candidate.descriptor().version() == CandidateDescriptorVersion::V2 + { + log::debug!( + target: LOG_TARGET, + "V2 candidate descriptors not allowed. Dropping candidate {:?} for paraid {:?}.", + candidate.candidate().hash(), + candidate.descriptor().para_id() + ); + continue + } + candidates_per_para .entry(candidate.descriptor().para_id()) .or_default() diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 6c82233e0820..c0e1636569a8 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -58,7 +58,10 @@ mod enter { use core::panic; use frame_support::assert_ok; use frame_system::limits; - use polkadot_primitives::{vstaging::SchedulerParams, AvailabilityBitfield, UncheckedSigned}; + use polkadot_primitives::{ + vstaging::{InternalVersion, SchedulerParams}, + AvailabilityBitfield, UncheckedSigned, + }; use sp_runtime::Perbill; struct TestConfig { @@ -70,6 +73,7 @@ mod enter { fill_claimqueue: bool, elastic_paras: BTreeMap, unavailable_cores: Vec, + v2_descriptor: bool, } fn make_inherent_data( @@ -82,6 +86,7 @@ mod enter { fill_claimqueue, elastic_paras, unavailable_cores, + v2_descriptor, }: TestConfig, ) -> Bench { let extra_cores = elastic_paras @@ -99,7 +104,8 @@ mod enter { .set_backed_and_concluding_paras(backed_and_concluding.clone()) .set_dispute_sessions(&dispute_sessions[..]) .set_fill_claimqueue(fill_claimqueue) - .set_unavailable_cores(unavailable_cores); + .set_unavailable_cores(unavailable_cores) + .set_candidate_descriptor_v2(v2_descriptor); // Setup some assignments as needed: mock_assigner::Pallet::::set_core_count(builder.max_cores()); @@ -145,6 +151,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); // We expect the scenario to have cores 0 & 1 with pending availability. The backed @@ -240,6 +247,7 @@ mod enter { fill_claimqueue: false, elastic_paras: [(2, 3)].into_iter().collect(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -344,6 +352,7 @@ mod enter { fill_claimqueue: true, elastic_paras: [(2, 4)].into_iter().collect(), unavailable_cores: unavailable_cores.clone(), + v2_descriptor: false, }); let mut expected_para_inherent_data = scenario.data.clone(); @@ -600,6 +609,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -673,6 +683,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -744,6 +755,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -831,6 +843,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -918,6 +931,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1004,6 +1018,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1111,6 +1126,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1179,6 +1195,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1245,6 +1262,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1348,6 +1366,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let mut para_inherent_data = scenario.data.clone(); @@ -1437,6 +1456,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1465,6 +1485,76 @@ mod enter { assert_eq!(dispatch_error, Error::::InherentOverweight.into()); }); } + + #[test] + fn v2_descriptors_are_filtered() { + let config = default_config(); + assert!(config.configuration.config.scheduler_params.lookahead > 0); + new_test_ext(config).execute_with(|| { + // Set the elastic scaling MVP feature. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::ElasticScalingMVP as u8, + true, + ) + .unwrap(); + + let mut backed_and_concluding = BTreeMap::new(); + backed_and_concluding.insert(0, 1); + backed_and_concluding.insert(1, 1); + backed_and_concluding.insert(2, 1); + + let unavailable_cores = vec![]; + + let scenario = make_inherent_data(TestConfig { + dispute_statements: BTreeMap::new(), + dispute_sessions: vec![], // No disputes + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, + fill_claimqueue: true, + // 8 cores ! + elastic_paras: [(2, 8)].into_iter().collect(), + unavailable_cores: unavailable_cores.clone(), + v2_descriptor: true, + }); + + let mut unfiltered_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 10 backed candidates) + assert_eq!(unfiltered_para_inherent_data.bitfields.len(), 50); + // * 10 v2 candidate descriptors. + assert_eq!(unfiltered_para_inherent_data.backed_candidates.len(), 10); + + // Make the last candidate look like v1, by using an unknown version. + unfiltered_para_inherent_data.backed_candidates[9] + .descriptor_mut() + .set_version(InternalVersion(123)); + + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &unfiltered_para_inherent_data) + .unwrap(); + + // We expect all backed candidates to be filtered out. + let filtered_para_inherend_data = + Pallet::::create_inherent_inner(&inherent_data).unwrap(); + + assert_eq!(filtered_para_inherend_data.backed_candidates.len(), 0); + + let dispatch_error = Pallet::::enter( + frame_system::RawOrigin::None.into(), + unfiltered_para_inherent_data, + ) + .unwrap_err() + .error; + + // We expect `enter` to fail because the inherent data contains backed candidates with + // v2 descriptors. + assert_eq!(dispatch_error, Error::::CandidatesFilteredDuringExecution.into()); + }); + } } fn default_header() -> polkadot_primitives::Header { @@ -3376,7 +3466,8 @@ mod sanitizers { &shared::AllowedRelayParents::::get(), BTreeSet::new(), scheduled, - core_index_enabled + core_index_enabled, + false, ), expected_backed_candidates_with_core, ); @@ -3400,7 +3491,8 @@ mod sanitizers { &shared::AllowedRelayParents::::get(), BTreeSet::new(), scheduled, - core_index_enabled + core_index_enabled, + false, ), expected_backed_candidates_with_core, ); @@ -3425,6 +3517,7 @@ mod sanitizers { BTreeSet::new(), scheduled, core_index_enabled, + false, ), expected_backed_candidates_with_core ); @@ -3457,6 +3550,7 @@ mod sanitizers { BTreeSet::new(), scheduled, core_index_enabled, + false, ), expected_backed_candidates_with_core ); @@ -3497,6 +3591,7 @@ mod sanitizers { BTreeSet::new(), scheduled, core_index_enabled, + false, ); if core_index_enabled { @@ -3567,6 +3662,7 @@ mod sanitizers { BTreeSet::new(), scheduled, core_index_enabled, + false, ); if core_index_enabled { @@ -3605,6 +3701,7 @@ mod sanitizers { BTreeSet::new(), scheduled, core_index_enabled, + false, ); assert!(sanitized_backed_candidates.is_empty()); @@ -3641,6 +3738,7 @@ mod sanitizers { set, scheduled, core_index_enabled, + false, ); assert_eq!(sanitized_backed_candidates.len(), backed_candidates.len() / 2); @@ -3678,6 +3776,7 @@ mod sanitizers { invalid_set, scheduled, true, + false, ); // We'll be left with candidates from paraid 2 and 4. @@ -3714,6 +3813,7 @@ mod sanitizers { invalid_set, scheduled, true, + false, ); // Only the second candidate of paraid 1 should be removed. From cdb49a690efe5b1ad4d62138ff160aaa772309f8 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 22 Aug 2024 14:01:21 +0300 Subject: [PATCH 050/103] increase test coverage Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/builder.rs | 102 +++++++++++----- .../parachains/src/paras_inherent/tests.rs | 115 ++++++++++++++++-- 2 files changed, 175 insertions(+), 42 deletions(-) diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 35835793ce9d..72d93f15c499 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -32,9 +32,9 @@ use frame_system::pallet_prelude::*; use polkadot_primitives::{ node_features::FeatureIndex, vstaging::{ - BackedCandidate, CandidateDescriptorV2, - CommittedCandidateReceiptV2 as CommittedCandidateReceipt, - InherentData as ParachainsInherentData, + BackedCandidate, CandidateDescriptorV2, ClaimQueueOffset, + CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CoreSelector, + InherentData as ParachainsInherentData, UMPSignal, UMP_SEPARATOR, }, AvailabilityBitfield, CandidateCommitments, CandidateDescriptor, CandidateHash, CollatorId, CollatorSignature, CompactStatement, CoreIndex, DisputeStatement, DisputeStatementSet, @@ -309,20 +309,34 @@ impl BenchBuilder { HeadData(vec![0xFF; max_head_size as usize]) } - fn candidate_descriptor_mock() -> CandidateDescriptorV2 { - // Use a v1 descriptor. - CandidateDescriptor:: { - para_id: 0.into(), - relay_parent: Default::default(), - collator: junk_collator(), - persisted_validation_data_hash: Default::default(), - pov_hash: Default::default(), - erasure_root: Default::default(), - signature: junk_collator_signature(), - para_head: Default::default(), - validation_code_hash: mock_validation_code().hash(), + fn candidate_descriptor_mock(candidate_descriptor_v2: bool) -> CandidateDescriptorV2 { + if candidate_descriptor_v2 { + CandidateDescriptorV2::new( + 0.into(), + Default::default(), + CoreIndex(200), + 1, + Default::default(), + Default::default(), + Default::default(), + Default::default(), + mock_validation_code().hash(), + ) + } else { + // Convert v1 to v2. + CandidateDescriptor:: { + para_id: 0.into(), + relay_parent: Default::default(), + collator: junk_collator(), + persisted_validation_data_hash: Default::default(), + pov_hash: Default::default(), + erasure_root: Default::default(), + signature: junk_collator_signature(), + para_head: Default::default(), + validation_code_hash: mock_validation_code().hash(), + } + .into() } - .into() } /// Create a mock of `CandidatePendingAvailability`. @@ -332,17 +346,19 @@ impl BenchBuilder { candidate_hash: CandidateHash, availability_votes: BitVec, commitments: CandidateCommitments, + candidate_descriptor_v2: bool, ) -> inclusion::CandidatePendingAvailability> { inclusion::CandidatePendingAvailability::>::new( - core_idx, // core - candidate_hash, // hash - Self::candidate_descriptor_mock(), // candidate descriptor - commitments, // commitments - availability_votes, // availability votes - Default::default(), // backers - Zero::zero(), // relay parent - One::one(), // relay chain block this was backed in - group_idx, // backing group + core_idx, // core + candidate_hash, // hash + Self::candidate_descriptor_mock(candidate_descriptor_v2), // candidate descriptor + commitments, // commitments + availability_votes, // availability votes + Default::default(), // backers + Zero::zero(), // relay parent + One::one(), /* relay chain block this + * was backed in */ + group_idx, // backing group ) } @@ -357,6 +373,7 @@ impl BenchBuilder { group_idx: GroupIndex, availability_votes: BitVec, candidate_hash: CandidateHash, + candidate_descriptor_v2: bool, ) { let commitments = CandidateCommitments:: { upward_messages: Default::default(), @@ -372,6 +389,7 @@ impl BenchBuilder { candidate_hash, availability_votes, commitments, + candidate_descriptor_v2, ); inclusion::PendingAvailability::::mutate(para_id, |maybe_andidates| { if let Some(candidates) = maybe_andidates { @@ -545,6 +563,7 @@ impl BenchBuilder { // No validators have made this candidate available yet. bitvec::bitvec![u8, bitvec::order::Lsb0; 0; validators.len()], CandidateHash(H256::from(byte32_slice_from(current_core_idx))), + self.candidate_descriptor_v2, ); if !self.unavailable_cores.contains(¤t_core_idx) { concluding_cores.insert(current_core_idx); @@ -674,7 +693,7 @@ impl BenchBuilder { .into() }; - let candidate = CommittedCandidateReceipt:: { + let mut candidate = CommittedCandidateReceipt:: { descriptor, commitments: CandidateCommitments:: { upward_messages: Default::default(), @@ -687,6 +706,22 @@ impl BenchBuilder { }, }; + if self.candidate_descriptor_v2 { + // `UMPSignal` separator. + candidate.commitments.upward_messages.force_push(UMP_SEPARATOR); + + // `SelectCore` commitment. + // Claim queue offset must be `0` so this candidate is for the very + // next block. + candidate.commitments.upward_messages.force_push( + UMPSignal::SelectCore( + CoreSelector(chain_idx as u8), + ClaimQueueOffset(0), + ) + .encode(), + ); + } + let candidate_hash = candidate.hash(); let validity_votes: Vec<_> = group_validators @@ -706,12 +741,14 @@ impl BenchBuilder { }) .collect(); - // Check if the elastic scaling bit is set, if so we need to supply the core - // index in the generated candidate. - let core_idx = configuration::ActiveConfig::::get() - .node_features - .get(FeatureIndex::ElasticScalingMVP as usize) - .map(|_the_bit| core_idx); + let core_idx = if self.candidate_descriptor_v2 { + None + } else { + configuration::ActiveConfig::::get() + .node_features + .get(FeatureIndex::ElasticScalingMVP as usize) + .and_then(|the_bit| if *the_bit { Some(core_idx) } else { None }) + }; BackedCandidate::::new( candidate, @@ -764,6 +801,7 @@ impl BenchBuilder { group_idx, Self::validator_availability_votes_yes(validators.len()), candidate_hash, + self.candidate_descriptor_v2, ); let statements_len = diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index c0e1636569a8..ef466491c1ef 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -15,12 +15,15 @@ // along with Polkadot. If not, see . use super::*; +use rstest::rstest; use crate::{ configuration::{self, HostConfiguration}, mock::MockGenesisConfig, }; -use polkadot_primitives::vstaging::SchedulerParams; +use polkadot_primitives::vstaging::{ + ClaimQueueOffset, CoreSelector, SchedulerParams, UMPSignal, UMP_SEPARATOR, +}; fn default_config() -> MockGenesisConfig { MockGenesisConfig { @@ -127,15 +130,25 @@ mod enter { } } - #[test] + #[rstest] + #[case(true)] + #[case(false)] // Validate that if we create 2 backed candidates which are assigned to 2 cores that will be // freed via becoming fully available, the backed candidates will not be filtered out in // `create_inherent` and will not cause `enter` to early. - fn include_backed_candidates() { + fn include_backed_candidates(#[case] v2_descriptor: bool) { let config = MockGenesisConfig::default(); assert!(config.configuration.config.scheduler_params.lookahead > 0); new_test_ext(config).execute_with(|| { + // Enable the v2 receipts. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::CandidateReceiptV2 as u8, + v2_descriptor, + ) + .unwrap(); + let dispute_statements = BTreeMap::new(); let mut backed_and_concluding = BTreeMap::new(); @@ -151,7 +164,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], - v2_descriptor: false, + v2_descriptor, }); // We expect the scenario to have cores 0 & 1 with pending availability. The backed @@ -213,8 +226,10 @@ mod enter { }); } - #[test] - fn include_backed_candidates_elastic_scaling() { + #[rstest] + #[case(true)] + #[case(false)] + fn include_backed_candidates_elastic_scaling(#[case] v2_descriptor: bool) { // ParaId 0 has one pending candidate on core 0. // ParaId 1 has one pending candidate on core 1. // ParaId 2 has three pending candidates on cores 2, 3 and 4. @@ -231,6 +246,14 @@ mod enter { ) .unwrap(); + // Enable the v2 receipts. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::CandidateReceiptV2 as u8, + v2_descriptor, + ) + .unwrap(); + let dispute_statements = BTreeMap::new(); let mut backed_and_concluding = BTreeMap::new(); @@ -247,7 +270,7 @@ mod enter { fill_claimqueue: false, elastic_paras: [(2, 3)].into_iter().collect(), unavailable_cores: vec![], - v2_descriptor: false, + v2_descriptor, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1329,6 +1352,15 @@ mod enter { ccr.commitments.processed_downward_messages = idx as u32; let core_index = start_core_index + idx; + // `UMPSignal` separator. + ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); + + // `SelectCore` commitment. + // Claim queue offset must be `0`` so this candidate is for the very next block. + ccr.commitments.upward_messages.force_push( + UMPSignal::SelectCore(CoreSelector(idx as u8), ClaimQueueOffset(0)).encode(), + ); + BackedCandidate::new( ccr.into(), Default::default(), @@ -1341,8 +1373,10 @@ mod enter { // Ensure that overweight parachain inherents are always rejected by the runtime. // Runtime should panic and return `InherentOverweight` error. - #[test] - fn test_backed_candidates_apply_weight_works_for_elastic_scaling() { + #[rstest] + #[case(true)] + #[case(false)] + fn test_backed_candidates_apply_weight_works_for_elastic_scaling(#[case] v2_descriptor: bool) { new_test_ext(MockGenesisConfig::default()).execute_with(|| { let seed = [ 1, 0, 52, 0, 0, 0, 0, 0, 1, 0, 10, 0, 22, 32, 0, 0, 2, 0, 55, 49, 0, 11, 0, 0, 3, @@ -1353,6 +1387,14 @@ mod enter { // Create an overweight inherent and oversized block let mut backed_and_concluding = BTreeMap::new(); + // Enable the v2 receipts. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::CandidateReceiptV2 as u8, + v2_descriptor, + ) + .unwrap(); + for i in 0..30 { backed_and_concluding.insert(i, i); } @@ -1366,7 +1408,7 @@ mod enter { fill_claimqueue: false, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], - v2_descriptor: false, + v2_descriptor, }); let mut para_inherent_data = scenario.data.clone(); @@ -1555,6 +1597,59 @@ mod enter { assert_eq!(dispatch_error, Error::::CandidatesFilteredDuringExecution.into()); }); } + + #[test] + fn v2_descriptors_are_accepted() { + let config = default_config(); + assert!(config.configuration.config.scheduler_params.lookahead > 0); + new_test_ext(config).execute_with(|| { + // Set the elastic scaling MVP feature. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::ElasticScalingMVP as u8, + true, + ) + .unwrap(); + + // Enable the v2 receipts. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::CandidateReceiptV2 as u8, + true, + ) + .unwrap(); + + let mut backed_and_concluding = BTreeMap::new(); + backed_and_concluding.insert(0, 1); + backed_and_concluding.insert(1, 1); + backed_and_concluding.insert(2, 1); + + let unavailable_cores = vec![]; + + let scenario = make_inherent_data(TestConfig { + dispute_statements: BTreeMap::new(), + dispute_sessions: vec![], // No disputes + backed_and_concluding, + num_validators_per_core: 1, + code_upgrade: None, + fill_claimqueue: true, + // 8 cores ! + elastic_paras: [(2, 3)].into_iter().collect(), + unavailable_cores: unavailable_cores.clone(), + v2_descriptor: false, + }); + + let inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (2 validators per core, 5 backed candidates) + assert_eq!(inherent_data.bitfields.len(), 5); + // * 5 v2 candidate descriptors. + assert_eq!(inherent_data.backed_candidates.len(), 5); + + Pallet::::enter(frame_system::RawOrigin::None.into(), inherent_data).unwrap(); + }); + } } fn default_header() -> polkadot_primitives::Header { From aa925cd40c1cf2af4875acd38a01ee0225e49a72 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 23 Aug 2024 15:26:50 +0300 Subject: [PATCH 051/103] Improve usability of primitives Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 40 ++++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 8d97ddb57fc5..a236f054e1aa 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -398,6 +398,25 @@ impl CandidateCommitments { UMPSignal::SelectCore(core_selector, cq_offset) => Some((core_selector, cq_offset)), } } + + /// Returns the core index determined by `UMPSignal::SelectCore` commitment + /// and `assigned_cores`. + /// + /// Returns `None` if there is no `UMPSignal::SelectCore` commitment or + /// assigned cores is empty. + /// + /// `assigned_cores` must be soted vec of all core indices assigned to a parachain. + pub fn committed_core_index(&self, assigned_cores: &[&CoreIndex]) -> Option { + if assigned_cores.is_empty() { + return None + } + + self.selected_core().and_then(|(core_selector, _cq_offset)| { + let core_index = + **assigned_cores.get(core_selector.0 as usize % assigned_cores.len())?; + Some(core_index) + }) + } } /// CandidateReceipt construction errors. @@ -507,9 +526,11 @@ impl CommittedCandidateReceiptV2 { /// Checks if descriptor core index is equal to the commited core index. /// Input `assigned_cores` must contain the sorted cores assigned to the para at /// the committed claim queue offset. - pub fn check(&self, assigned_cores: &[CoreIndex]) -> Result<(), CandidateReceiptError> { - // Don't check v1 descriptors. - if self.descriptor.version() == CandidateDescriptorVersion::V1 { + pub fn check(&self, assigned_cores: &[&CoreIndex]) -> Result<(), CandidateReceiptError> { + // Don't check v1 descriptors without any `SelectCore` commitments. + if self.descriptor.version() == CandidateDescriptorVersion::V1 && + self.commitments.selected_core().is_none() + { return Ok(()) } @@ -518,15 +539,12 @@ impl CommittedCandidateReceiptV2 { } let descriptor_core_index = CoreIndex(self.descriptor.core_index as u32); + let core_index = self + .commitments + .committed_core_index(assigned_cores) + .ok_or(CandidateReceiptError::NoAssignment)?; - let (core_selector, _cq_offset) = - self.commitments.selected_core().ok_or(CandidateReceiptError::NoCoreSelected)?; - - let core_index = assigned_cores - .get(core_selector.0 as usize % assigned_cores.len()) - .ok_or(CandidateReceiptError::InvalidCoreIndex)?; - - if *core_index != descriptor_core_index { + if core_index != descriptor_core_index { return Err(CandidateReceiptError::CoreIndexMismatch) } From 00d7c710c01ae893617b5dfa6211ad9700caea9d Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 23 Aug 2024 15:28:02 +0300 Subject: [PATCH 052/103] use committed core index if available in v1 receipts Signed-off-by: Andrei Sandu --- .../parachains/src/paras_inherent/mod.rs | 114 +++++++++++------- 1 file changed, 71 insertions(+), 43 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 763464cb8111..99ae574a8c99 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -951,11 +951,12 @@ pub(crate) fn sanitize_bitfields( /// Perform required checks for given candidate receipt. /// -/// Returns `true` if candidate descriptor is version 1. +/// Returns `true` if candidate descriptor is version 1 without `UMPSignal` +/// commitments. /// -/// For version 2 it returns `false` if: +/// Otherwise returns `false` if: /// - version 2 descriptors are not allowed -/// - the core index doesn't match the one computed from the commitments +/// - the core index in descriptor doesn't match the one computed from the commitments /// - the `SelectCore` signal does not refer to a core at the top of claim queue fn sanitize_backed_candidate_v2( candidate: &BackedCandidate, @@ -963,37 +964,43 @@ fn sanitize_backed_candidate_v2( scheduled_paras: &BTreeMap, allow_v2_receipts: bool, ) -> bool { - if candidate.descriptor().version() == CandidateDescriptorVersion::V1 { - return true; - } + // We move forward only if the candidate commits to a core selector and claim queue offset. + // If parachain runtime already makes use of `UMPSignal::SelectCore`, and the collator does not, + // we can expect v1 candidate receipts with `UMPSignal::SelectCore` commitments. + // + // It is important that we use these commitments of v1 receipts aand not rely on injected cores. + // Parachain runtime upgrade should be enough to enable both validators and the runtime + // to also validate the core index the candidate has committed to. + let Some((core_selector, cq_offset)) = candidate.candidate().commitments.selected_core() else { + if candidate.descriptor().version() == CandidateDescriptorVersion::V1 { + return true + } - // Drop any v2 candidate receipts if nodes are not allowed to use them. - // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure - // any v1 descendants of v2 candidates are dropped. - if !allow_v2_receipts { log::debug!( target: LOG_TARGET, - "V2 candidate descriptors not allowed. Dropping candidate {:?} for paraid {:?}.", + "Dropping V2 candidate receipt {:?} for paraid {:?}, no `SelectCore` commitment.", candidate.candidate().hash(), candidate.descriptor().para_id() ); + return false - } + }; - let current_block_num = frame_system::Pallet::::block_number(); - let Some((_core_selector, cq_offset)) = candidate.candidate().commitments.selected_core() - else { + // Drop any candidate receipts with `UMPSignal::SelectCore` commitments if nodes are not allowed + // to use them. + // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure + // any v1 descendants of v2 candidates are dropped. + if !allow_v2_receipts { log::debug!( target: LOG_TARGET, - "Dropping candidate {:?} for paraid {:?}, no `CoreSelector` commitment.", + "V2 candidate descriptors not allowed. Dropping candidate {:?} for paraid {:?}.", candidate.candidate().hash(), candidate.descriptor().para_id() ); - return false - }; + } - // Get the relay parent number for the candidate: + // Get the relay parent number for the candidate let Some((_state_root, relay_parent_num)) = allowed_relay_parents.acquire_info(candidate.descriptor().relay_parent(), None) else { @@ -1006,8 +1013,10 @@ fn sanitize_backed_candidate_v2( return false }; + let current_block_num = frame_system::Pallet::::block_number(); let expected_cq_offset = current_block_num - relay_parent_num - One::one(); - // Drop the v2 candidate receipt if the core claim has not reached the top of the + + // Drop the candidate receipt if the core claim has not reached the top of the // claim queue. if expected_cq_offset != (cq_offset.0 as u32).into() { log::debug!( @@ -1022,25 +1031,30 @@ fn sanitize_backed_candidate_v2( return false } - let assigned_cores = scheduled_paras - .iter() - .filter_map(|(core_index, para)| { - if *para == candidate.descriptor().para_id() { - Some(*core_index) - } else { - None - } - }) - .collect::>(); + let assigned_cores = + scheduled_paras + .iter() + .filter_map(|(core_index, para)| { + if *para == candidate.descriptor().para_id() { + Some(core_index) + } else { + None + } + }) + .collect::>(); + let assigned_cores = assigned_cores.as_slice(); // Check if core index in descriptor matches the one in the commitments - if let Err(err) = candidate.candidate().check(&assigned_cores) { + // No-op for v1 candidate receipts. + if let Err(err) = candidate.candidate().check(assigned_cores) { log::debug!( target: LOG_TARGET, - "Dropping candidate {:?} for paraid {:?}, {:?}", + "Dropping candidate {:?} for paraid {:?}, {:?}, core_selector={:?}, cq_offset={:?}", candidate.candidate().hash(), candidate.descriptor().para_id(), err, + core_selector, + cq_offset, ); return false @@ -1284,6 +1298,7 @@ fn filter_backed_statements_from_disabled_validators< // 1. Core index assigned to the parachain which has produced the candidate // 2. The relay chain block number of the candidate retain_candidates::(backed_candidates_with_core, |para_id, (bc, core_idx)| { + // `CoreIndex` not used, we just need a copy to write it back later. let (validator_indices, maybe_core_index) = bc.validator_indices_and_core_index(core_index_enabled); let mut validator_indices = BitVec::<_>::from(validator_indices); @@ -1491,6 +1506,11 @@ fn map_candidates_to_cores(allowed_relay_parents, &candidate) - }; - - if let Some(core_index) = maybe_core_index { + if let Some(core_index) = + get_core_index::(allowed_relay_parents, &candidate, &assigned_cores) + { if scheduled_cores.remove(&core_index) { temp_backed_candidates.push((candidate, core_index)); } else { @@ -1573,6 +1585,22 @@ fn map_candidates_to_cores( + allowed_relay_parents: &AllowedRelayParentsTracker>, + candidate: &BackedCandidate, + assigned_cores: &BTreeSet, +) -> Option { + // Use the committed core index. It could be available even if descriptor is v1. + // We've already sanitized the candidate so it should be ok to trust it is valid. + // Fall back to injected core index. + candidate + .candidate() + .commitments + .committed_core_index(assigned_cores.iter().collect::>().as_slice()) + .or_else(|| get_injected_core_index::(allowed_relay_parents, &candidate)) +} + fn get_injected_core_index( allowed_relay_parents: &AllowedRelayParentsTracker>, candidate: &BackedCandidate, From af9f5617c40c9380999810f084522cc56e60c302 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 23 Aug 2024 15:29:24 +0300 Subject: [PATCH 053/103] typo Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index a236f054e1aa..9bbc63a31138 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -405,7 +405,7 @@ impl CandidateCommitments { /// Returns `None` if there is no `UMPSignal::SelectCore` commitment or /// assigned cores is empty. /// - /// `assigned_cores` must be soted vec of all core indices assigned to a parachain. + /// `assigned_cores` must be a sorted vec of all core indices assigned to a parachain. pub fn committed_core_index(&self, assigned_cores: &[&CoreIndex]) -> Option { if assigned_cores.is_empty() { return None From fb2cefb31a267f21c27028086bf9ce62fae99837 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 23 Aug 2024 15:33:01 +0300 Subject: [PATCH 054/103] fix check Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 9bbc63a31138..cab5801ade46 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -538,16 +538,20 @@ impl CommittedCandidateReceiptV2 { return Err(CandidateReceiptError::NoAssignment) } - let descriptor_core_index = CoreIndex(self.descriptor.core_index as u32); let core_index = self .commitments .committed_core_index(assigned_cores) .ok_or(CandidateReceiptError::NoAssignment)?; - if core_index != descriptor_core_index { - return Err(CandidateReceiptError::CoreIndexMismatch) + // Only check descriptor `core_index` field of v2 descriptors. If it is v1, that field + // will be garbage. + if self.descriptor.version() == CandidateDescriptorVersion::V2 { + let descriptor_core_index = CoreIndex(self.descriptor.core_index as u32); + if core_index != descriptor_core_index { + return Err(CandidateReceiptError::CoreIndexMismatch) + } } - + Ok(()) } } From b53787d47a1489df0e01c32e0d3ff63bce7d9c2a Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 23 Aug 2024 15:36:00 +0300 Subject: [PATCH 055/103] typo Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/paras_inherent/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 99ae574a8c99..37d7bcb32ea3 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -968,7 +968,7 @@ fn sanitize_backed_candidate_v2( // If parachain runtime already makes use of `UMPSignal::SelectCore`, and the collator does not, // we can expect v1 candidate receipts with `UMPSignal::SelectCore` commitments. // - // It is important that we use these commitments of v1 receipts aand not rely on injected cores. + // It is important that we use these commitments of v1 receipts and not rely on injected cores. // Parachain runtime upgrade should be enough to enable both validators and the runtime // to also validate the core index the candidate has committed to. let Some((core_selector, cq_offset)) = candidate.candidate().commitments.selected_core() else { From e2ef46ebd9aadb645024343b280c01320c6875f2 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 26 Aug 2024 17:43:05 +0300 Subject: [PATCH 056/103] add test for mixed v1 v2 scenario Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 50 +++++++++++-- polkadot/runtime/parachains/src/builder.rs | 8 +- .../parachains/src/paras_inherent/tests.rs | 73 ++++++++++++++++++- 3 files changed, 118 insertions(+), 13 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index cab5801ade46..e8b51d6b2202 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -534,6 +534,10 @@ impl CommittedCandidateReceiptV2 { return Ok(()) } + if self.commitments.selected_core().is_none() { + return Err(CandidateReceiptError::NoCoreSelected) + } + if assigned_cores.is_empty() { return Err(CandidateReceiptError::NoAssignment) } @@ -551,7 +555,7 @@ impl CommittedCandidateReceiptV2 { return Err(CandidateReceiptError::CoreIndexMismatch) } } - + Ok(()) } } @@ -887,7 +891,7 @@ mod tests { .upward_messages .force_push(UMPSignal::SelectCore(CoreSelector(0), ClaimQueueOffset(1)).encode()); - assert_eq!(new_ccr.check(&vec![CoreIndex(123)]), Ok(())); + assert_eq!(new_ccr.check(&vec![&CoreIndex(123)]), Ok(())); } #[test] @@ -901,7 +905,7 @@ mod tests { // The check should fail because no `SelectCore` signal was sent. assert_eq!( - new_ccr.check(&vec![CoreIndex(0), CoreIndex(100)]), + new_ccr.check(&vec![&CoreIndex(0), &CoreIndex(100)]), Err(CandidateReceiptError::NoCoreSelected) ); @@ -913,7 +917,7 @@ mod tests { // Failure is expected. assert_eq!( - new_ccr.check(&vec![CoreIndex(0), CoreIndex(100)]), + new_ccr.check(&vec![&CoreIndex(0), &CoreIndex(100)]), Err(CandidateReceiptError::NoCoreSelected) ); @@ -932,7 +936,7 @@ mod tests { .force_push(UMPSignal::SelectCore(CoreSelector(1), ClaimQueueOffset(1)).encode()); // Duplicate doesn't override first signal. - assert_eq!(new_ccr.check(&vec![CoreIndex(0), CoreIndex(100)]), Ok(())); + assert_eq!(new_ccr.check(&vec![&CoreIndex(0), &CoreIndex(100)]), Ok(())); } #[test] @@ -969,11 +973,43 @@ mod tests { Decode::decode(&mut encoded_ccr.as_slice()).unwrap(); assert_eq!(v2_ccr.descriptor.core_index(), Some(CoreIndex(123))); - assert_eq!(new_ccr.check(&vec![CoreIndex(123)]), Ok(())); + assert_eq!(new_ccr.check(&vec![&CoreIndex(123)]), Ok(())); assert_eq!(new_ccr.hash(), v2_ccr.hash()); } + // Only check descriptor `core_index` field of v2 descriptors. If it is v1, that field + // will be garbage. + #[test] + fn test_v1_descriptors_with_ump_signal() { + let mut ccr = dummy_old_committed_candidate_receipt(); + ccr.descriptor.para_id = ParaId::new(1024); + // Adding collator signature should make it decode as v1. + ccr.descriptor.signature = dummy_collator_signature(); + ccr.descriptor.collator = dummy_collator_id(); + + ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); + ccr.commitments + .upward_messages + .force_push(UMPSignal::SelectCore(CoreSelector(1), ClaimQueueOffset(1)).encode()); + + let encoded_ccr: Vec = ccr.encode(); + + let v1_ccr: CommittedCandidateReceiptV2 = + Decode::decode(&mut encoded_ccr.as_slice()).unwrap(); + + assert_eq!(v1_ccr.descriptor.version(), CandidateDescriptorVersion::V1); + assert!(v1_ccr.commitments.selected_core().is_some()); + assert!(v1_ccr.check(&vec![&CoreIndex(0), &CoreIndex(1)]).is_ok()); + + assert_eq!( + v1_ccr.commitments.committed_core_index(&vec![&CoreIndex(10), &CoreIndex(5)]), + Some(CoreIndex(5)), + ); + + assert_eq!(v1_ccr.descriptor.core_index(), None); + } + #[test] fn test_core_select_is_mandatory() { // Testing edge case when collators provide zeroed signature and collator id. @@ -988,7 +1024,7 @@ mod tests { // version 2. // We expect the check to fail in such case because there will be no `SelectCore` // commitment. - assert_eq!(new_ccr.check(&vec![CoreIndex(0)]), Err(CandidateReceiptError::NoCoreSelected)); + assert_eq!(new_ccr.check(&vec![&CoreIndex(0)]), Err(CandidateReceiptError::NoCoreSelected)); // Adding collator signature should make it decode as v1. old_ccr.descriptor.signature = dummy_collator_signature(); diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 72d93f15c499..7c5e5d7ad307 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -52,14 +52,14 @@ fn mock_validation_code() -> ValidationCode { ValidationCode(vec![1, 2, 3]) } -// Create a dummy collator id suitable to be used in a V1 candidate descriptor. -fn junk_collator() -> CollatorId { +/// Create a dummy collator id suitable to be used in a V1 candidate descriptor. +pub fn junk_collator() -> CollatorId { CollatorId::from_slice(&mut (0..32).into_iter().collect::>().as_slice()) .expect("32 bytes; qed") } -// Creates a dummy collator signature suitable to be used in a V1 candidate descriptor. -fn junk_collator_signature() -> CollatorSignature { +/// Creates a dummy collator signature suitable to be used in a V1 candidate descriptor. +pub fn junk_collator_signature() -> CollatorSignature { CollatorSignature::from_slice(&mut (0..64).into_iter().collect::>().as_slice()) .expect("64 bytes; qed") } diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index ef466491c1ef..6700ab5307bc 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -48,7 +48,7 @@ fn default_config() -> MockGenesisConfig { mod enter { use super::{inclusion::tests::TestCandidateBuilder, *}; use crate::{ - builder::{Bench, BenchBuilder}, + builder::{junk_collator, junk_collator_signature, Bench, BenchBuilder}, mock::{mock_assigner, new_test_ext, BlockLength, BlockWeights, RuntimeOrigin, Test}, scheduler::{ common::{Assignment, AssignmentProvider}, @@ -63,7 +63,7 @@ mod enter { use frame_system::limits; use polkadot_primitives::{ vstaging::{InternalVersion, SchedulerParams}, - AvailabilityBitfield, UncheckedSigned, + AvailabilityBitfield, CandidateDescriptor, UncheckedSigned, }; use sp_runtime::Perbill; @@ -1650,6 +1650,75 @@ mod enter { Pallet::::enter(frame_system::RawOrigin::None.into(), inherent_data).unwrap(); }); } + + // Test when parachain runtime is upgraded to support the new commitments + // but some collators are not and provide v1 descriptors. + #[test] + fn elastic_scaling_mixed_v1_v2_descriptors() { + let config = default_config(); + assert!(config.configuration.config.scheduler_params.lookahead > 0); + new_test_ext(config).execute_with(|| { + // Set the elastic scaling MVP feature. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::ElasticScalingMVP as u8, + true, + ) + .unwrap(); + + // Enable the v2 receipts. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::CandidateReceiptV2 as u8, + true, + ) + .unwrap(); + + let mut backed_and_concluding = BTreeMap::new(); + backed_and_concluding.insert(0, 1); + backed_and_concluding.insert(1, 1); + backed_and_concluding.insert(2, 1); + + let unavailable_cores = vec![]; + + let scenario = make_inherent_data(TestConfig { + dispute_statements: BTreeMap::new(), + dispute_sessions: vec![], // No disputes + backed_and_concluding, + num_validators_per_core: 1, + code_upgrade: None, + fill_claimqueue: true, + // 3 cores + elastic_paras: [(2, 3)].into_iter().collect(), + unavailable_cores: unavailable_cores.clone(), + v2_descriptor: false, + }); + + let mut inherent_data = scenario.data.clone(); + let candidate_count = inherent_data.backed_candidates.len(); + + // Make last 2 candidates v1 + for index in candidate_count - 2..candidate_count { + let encoded = inherent_data.backed_candidates[index].descriptor().encode(); + + let mut decoded: CandidateDescriptor = + Decode::decode(&mut encoded.as_slice()).unwrap(); + decoded.collator = junk_collator(); + decoded.signature = junk_collator_signature(); + + *inherent_data.backed_candidates[index].descriptor_mut() = + Decode::decode(&mut encoded.as_slice()).unwrap(); + } + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (2 validators per core, 5 backed candidates) + assert_eq!(inherent_data.bitfields.len(), 5); + // * 5 v2 candidate descriptors. + assert_eq!(inherent_data.backed_candidates.len(), 5); + + Pallet::::enter(frame_system::RawOrigin::None.into(), inherent_data).unwrap(); + }); + } } fn default_header() -> polkadot_primitives::Header { From 2dfc5420a1510bb875c3a4bf56c3058abc384c29 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 26 Aug 2024 17:47:07 +0300 Subject: [PATCH 057/103] comment Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/paras_inherent/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 37d7bcb32ea3..6e52287e4714 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -969,8 +969,8 @@ fn sanitize_backed_candidate_v2( // we can expect v1 candidate receipts with `UMPSignal::SelectCore` commitments. // // It is important that we use these commitments of v1 receipts and not rely on injected cores. - // Parachain runtime upgrade should be enough to enable both validators and the runtime - // to also validate the core index the candidate has committed to. + // Parachain runtime upgrade is enough to enable both validators and the runtime to validate + // the core index the candidate has committed to. let Some((core_selector, cq_offset)) = candidate.candidate().commitments.selected_core() else { if candidate.descriptor().version() == CandidateDescriptorVersion::V1 { return true From a38a2434ad4448ef38312ae910f46055ed510ae2 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 26 Aug 2024 18:34:19 +0300 Subject: [PATCH 058/103] add ump test Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/ump_tests.rs | 41 +++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/polkadot/runtime/parachains/src/ump_tests.rs b/polkadot/runtime/parachains/src/ump_tests.rs index 71494a3faf67..a687515d330e 100644 --- a/polkadot/runtime/parachains/src/ump_tests.rs +++ b/polkadot/runtime/parachains/src/ump_tests.rs @@ -31,7 +31,10 @@ use frame_support::{ traits::{EnqueueMessage, ExecuteOverweightError, ServiceQueues}, weights::Weight, }; -use polkadot_primitives::{well_known_keys, Id as ParaId, UpwardMessage}; +use polkadot_primitives::{ + vstaging::{ClaimQueueOffset, CoreSelector, UMPSignal, UMP_SEPARATOR}, + well_known_keys, Id as ParaId, UpwardMessage, +}; use sp_crypto_hashing::{blake2_256, twox_64}; use sp_runtime::traits::Bounded; @@ -641,6 +644,42 @@ fn cannot_offboard_while_ump_dispatch_queued() { }); } +/// A para-chain cannot send an UMP to the relay chain while it is offboarding. +#[test] +fn enqueue_ump_signals() { + let para = 100.into(); + + new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { + register_parachain(para); + run_to_block(5, vec![4, 5]); + + let config = configuration::ActiveConfig::::get(); + let mut messages = (0..config.max_upward_message_num_per_candidate) + .into_iter() + .map(|_| "msg".encode()) + .collect::>(); + let expected_messages = messages.iter().cloned().map(|msg| (para, msg)).collect::>(); + + // `UMPSignals` and separator do not count as XCM messages. The below check must pass. + messages.append(&mut vec![ + UMP_SEPARATOR, + UMPSignal::SelectCore(CoreSelector(0), ClaimQueueOffset(0)).encode(), + ]); + + ParaInclusion::check_upward_messages( + &configuration::ActiveConfig::::get(), + para, + &messages, + ) + .unwrap(); + + // We expect that all messages except UMP signal and separator are processed + ParaInclusion::receive_upward_messages(para, &messages); + MessageQueue::service_queues(Weight::max_value()); + assert_eq!(Processed::take(), expected_messages); + }); +} + /// A para-chain cannot send an UMP to the relay chain while it is offboarding. #[test] fn cannot_enqueue_ump_while_offboarding() { From da381da70f5492af8c8dae308b608940358271f4 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 26 Aug 2024 18:58:05 +0300 Subject: [PATCH 059/103] avoid one storage read Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/paras_inherent/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 6e52287e4714..c5fa5f482a3b 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -1092,8 +1092,13 @@ fn sanitize_backed_candidates( // of the same para is preserved. let mut candidates_per_para: BTreeMap> = BTreeMap::new(); - // Get the paras scheduled next on each core. - let scheduled_paras = scheduler::Pallet::::scheduled_paras().collect::>(); + // Get the eligible paras for each core. + let scheduled_paras = scheduled + .iter() + .map(|(para_id, cores)| cores.iter().map(|core| (*core, *para_id))) + .flatten() + .collect(); + for candidate in backed_candidates { if !sanitize_backed_candidate_v2::( &candidate, From ca5c61832b01d58f70faf7743b3c5f4830b15110 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 27 Aug 2024 23:48:11 +0300 Subject: [PATCH 060/103] store claim queue snapshot in allowed relay parent info Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/shared.rs | 34 +++++++++++++------ .../runtime/parachains/src/shared/tests.rs | 18 +++++----- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/polkadot/runtime/parachains/src/shared.rs b/polkadot/runtime/parachains/src/shared.rs index 154b7cfefc3a..7330fcea72c1 100644 --- a/polkadot/runtime/parachains/src/shared.rs +++ b/polkadot/runtime/parachains/src/shared.rs @@ -25,7 +25,7 @@ use alloc::{ }; use frame_support::{pallet_prelude::*, traits::DisabledValidators}; use frame_system::pallet_prelude::BlockNumberFor; -use polkadot_primitives::{SessionIndex, ValidatorId, ValidatorIndex}; +use polkadot_primitives::{CoreIndex, Id, SessionIndex, ValidatorId, ValidatorIndex}; use sp_runtime::traits::AtLeast32BitUnsigned; use rand::{seq::SliceRandom, SeedableRng}; @@ -43,16 +43,25 @@ pub(crate) const SESSION_DELAY: SessionIndex = 2; #[cfg(test)] mod tests; -/// Information about past relay-parents. +/// Information about a relay parent. +#[derive(Encode, Decode, Default, TypeInfo, Debug)] +pub struct RelayParentInfo { + // Relay parent hash + pub relay_parent: Hash, + // The state root at this block + pub state_root: Hash, + // Claim queue snapshot + pub claim_queue: BTreeMap>, +} + +/// Keeps tracks of information about all viable relay parents. #[derive(Encode, Decode, Default, TypeInfo)] pub struct AllowedRelayParentsTracker { - // The past relay parents, paired with state roots, that are viable to build upon. + // Information about past relay parents that are viable to build upon. // // They are in ascending chronologic order, so the newest relay parents are at // the back of the deque. - // - // (relay_parent, state_root) - buffer: VecDeque<(Hash, Hash)>, + buffer: VecDeque>, // The number of the most recent relay-parent, if any. // If the buffer is empty, this value has no meaning and may @@ -70,13 +79,15 @@ impl &mut self, relay_parent: Hash, state_root: Hash, + claim_queue: BTreeMap>, number: BlockNumber, max_ancestry_len: u32, ) { // + 1 for the most recent block, which is always allowed. let buffer_size_limit = max_ancestry_len as usize + 1; - self.buffer.push_back((relay_parent, state_root)); + self.buffer.push_back(RelayParentInfo { relay_parent, state_root, claim_queue }); + self.latest_number = number; while self.buffer.len() > buffer_size_limit { let _ = self.buffer.pop_front(); @@ -96,8 +107,8 @@ impl &self, relay_parent: Hash, prev: Option, - ) -> Option<(Hash, BlockNumber)> { - let pos = self.buffer.iter().position(|(rp, _)| rp == &relay_parent)?; + ) -> Option<(&RelayParentInfo, BlockNumber)> { + let pos = self.buffer.iter().position(|info| info.relay_parent == relay_parent)?; let age = (self.buffer.len() - 1) - pos; let number = self.latest_number - BlockNumber::from(age as u32); @@ -107,7 +118,7 @@ impl } } - Some((self.buffer[pos].1, number)) + Some((&self.buffer[pos], number)) } /// Returns block number of the earliest block the buffer would contain if @@ -263,11 +274,12 @@ impl Pallet { pub(crate) fn add_allowed_relay_parent( relay_parent: T::Hash, state_root: T::Hash, + claim_queue: BTreeMap>, number: BlockNumberFor, max_ancestry_len: u32, ) { AllowedRelayParents::::mutate(|tracker| { - tracker.update(relay_parent, state_root, number, max_ancestry_len) + tracker.update(relay_parent, state_root, claim_queue, number, max_ancestry_len) }) } } diff --git a/polkadot/runtime/parachains/src/shared/tests.rs b/polkadot/runtime/parachains/src/shared/tests.rs index e47d1fd9cfe0..fd4b6555a48c 100644 --- a/polkadot/runtime/parachains/src/shared/tests.rs +++ b/polkadot/runtime/parachains/src/shared/tests.rs @@ -36,19 +36,19 @@ fn tracker_earliest_block_number() { // Push a single block into the tracker, suppose max capacity is 1. let max_ancestry_len = 0; - tracker.update(Hash::zero(), Hash::zero(), 0, max_ancestry_len); + tracker.update(Hash::zero(), Hash::zero(), Default::default(), 0, max_ancestry_len); assert_eq!(tracker.hypothetical_earliest_block_number(now, max_ancestry_len), now); // Test a greater capacity. let max_ancestry_len = 4; let now = 4; for i in 1..now { - tracker.update(Hash::zero(), Hash::zero(), i, max_ancestry_len); + tracker.update(Hash::zero(), Hash::zero(), Default::default(), i, max_ancestry_len); assert_eq!(tracker.hypothetical_earliest_block_number(i + 1, max_ancestry_len), 0); } // Capacity exceeded. - tracker.update(Hash::zero(), Hash::zero(), now, max_ancestry_len); + tracker.update(Hash::zero(), Hash::zero(), Default::default(), now, max_ancestry_len); assert_eq!(tracker.hypothetical_earliest_block_number(now + 1, max_ancestry_len), 1); } @@ -65,20 +65,20 @@ fn tracker_acquire_info() { ]; let (relay_parent, state_root) = blocks[0]; - tracker.update(relay_parent, state_root, 0, max_ancestry_len); + tracker.update(relay_parent, state_root, Default::default(), 0, max_ancestry_len); assert_matches!( tracker.acquire_info(relay_parent, None), - Some((s, b)) if s == state_root && b == 0 + Some((s, b)) if s.state_root == state_root && b == 0 ); let (relay_parent, state_root) = blocks[1]; - tracker.update(relay_parent, state_root, 1u32, max_ancestry_len); + tracker.update(relay_parent, state_root, Default::default(), 1u32, max_ancestry_len); let (relay_parent, state_root) = blocks[2]; - tracker.update(relay_parent, state_root, 2u32, max_ancestry_len); + tracker.update(relay_parent, state_root, Default::default(), 2u32, max_ancestry_len); for (block_num, (rp, state_root)) in blocks.iter().enumerate().take(2) { assert_matches!( tracker.acquire_info(*rp, None), - Some((s, b)) if &s == state_root && b == block_num as u32 + Some((s, b)) if &s.state_root == state_root && b == block_num as u32 ); assert!(tracker.acquire_info(*rp, Some(2)).is_none()); @@ -87,7 +87,7 @@ fn tracker_acquire_info() { for (block_num, (rp, state_root)) in blocks.iter().enumerate().skip(1) { assert_matches!( tracker.acquire_info(*rp, Some(block_num as u32 - 1)), - Some((s, b)) if &s == state_root && b == block_num as u32 + Some((s, b)) if &s.state_root == state_root && b == block_num as u32 ); } } From 4266665728b649853e8dc6b4ebf2cf3f7282d724 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 27 Aug 2024 23:49:07 +0300 Subject: [PATCH 061/103] check v2 receipts using claim queue snapshots Signed-off-by: Andrei Sandu --- .../runtime/parachains/src/inclusion/mod.rs | 6 +- .../runtime/parachains/src/inclusion/tests.rs | 5 +- .../parachains/src/paras_inherent/mod.rs | 101 +++++++----------- .../parachains/src/paras_inherent/tests.rs | 16 +-- 4 files changed, 55 insertions(+), 73 deletions(-) diff --git a/polkadot/runtime/parachains/src/inclusion/mod.rs b/polkadot/runtime/parachains/src/inclusion/mod.rs index f9f54790b570..4cc21a70a024 100644 --- a/polkadot/runtime/parachains/src/inclusion/mod.rs +++ b/polkadot/runtime/parachains/src/inclusion/mod.rs @@ -1261,17 +1261,17 @@ impl CandidateCheckContext { let relay_parent = backed_candidate_receipt.descriptor.relay_parent(); // Check that the relay-parent is one of the allowed relay-parents. - let (relay_parent_storage_root, relay_parent_number) = { + let (state_root, relay_parent_number) = { match allowed_relay_parents.acquire_info(relay_parent, self.prev_context) { None => return Err(Error::::DisallowedRelayParent), - Some(info) => info, + Some((info, relay_parent_number)) => (info.state_root, relay_parent_number), } }; { let persisted_validation_data = make_persisted_validation_data_with_parent::( relay_parent_number, - relay_parent_storage_root, + state_root, parent_head_data, ); diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index f17b8ed2e0e9..ff46ad7fa08e 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -83,7 +83,7 @@ fn default_allowed_relay_parent_tracker() -> AllowedRelayParentsTracker Pallet { let now = frame_system::Pallet::::block_number(); let config = configuration::ActiveConfig::::get(); - // Before anything else, update the allowed relay-parents. - { - let parent_number = now - One::one(); - let parent_storage_root = *parent_header.state_root(); - - shared::AllowedRelayParents::::mutate(|tracker| { - tracker.update( - parent_hash, - parent_storage_root, - parent_number, - config.async_backing_params.allowed_ancestry_len, - ); - }); - } - let allowed_relay_parents = shared::AllowedRelayParents::::get(); - let candidates_weight = backed_candidates_weight::(&backed_candidates); let bitfields_weight = signed_bitfields_weight::(&bitfields); let disputes_weight = multi_dispute_statement_sets_weight::(&disputes); @@ -570,6 +554,29 @@ impl Pallet { METRICS.on_candidates_processed_total(backed_candidates.len() as u64); + // After freeing cores and filling claims, but before processing backed candidates + // we update the allowed relay-parents. + { + let parent_number = now - One::one(); + let parent_storage_root = *parent_header.state_root(); + + shared::AllowedRelayParents::::mutate(|tracker| { + tracker.update( + parent_hash, + parent_storage_root, + scheduler::ClaimQueue::::get() + .into_iter() + .map(|(core_index, paras)| { + (core_index, paras.into_iter().map(|e| e.para_id()).collect()) + }) + .collect(), + parent_number, + config.async_backing_params.allowed_ancestry_len, + ); + }); + } + let allowed_relay_parents = shared::AllowedRelayParents::::get(); + let core_index_enabled = configuration::ActiveConfig::::get() .node_features .get(FeatureIndex::ElasticScalingMVP as usize) @@ -961,7 +968,6 @@ pub(crate) fn sanitize_bitfields( fn sanitize_backed_candidate_v2( candidate: &BackedCandidate, allowed_relay_parents: &AllowedRelayParentsTracker>, - scheduled_paras: &BTreeMap, allow_v2_receipts: bool, ) -> bool { // We move forward only if the candidate commits to a core selector and claim queue offset. @@ -1000,8 +1006,8 @@ fn sanitize_backed_candidate_v2( return false } - // Get the relay parent number for the candidate - let Some((_state_root, relay_parent_num)) = + // Get the claim queue snapshot at the candidate relay paren. + let Some((rp_info, _)) = allowed_relay_parents.acquire_info(candidate.descriptor().relay_parent(), None) else { log::debug!( @@ -1013,35 +1019,19 @@ fn sanitize_backed_candidate_v2( return false }; - let current_block_num = frame_system::Pallet::::block_number(); - let expected_cq_offset = current_block_num - relay_parent_num - One::one(); - - // Drop the candidate receipt if the core claim has not reached the top of the - // claim queue. - if expected_cq_offset != (cq_offset.0 as u32).into() { - log::debug!( - target: LOG_TARGET, - "Dropped candidate {:?} of paraid {:?} because the claimed core is not at top \ - of the claim queue, cq_offset: {:?}, relay_parent_num: {:?}", - candidate.candidate().hash(), - candidate.descriptor().para_id(), - cq_offset, - relay_parent_num - ); - return false - } - - let assigned_cores = - scheduled_paras - .iter() - .filter_map(|(core_index, para)| { - if *para == candidate.descriptor().para_id() { - Some(core_index) - } else { - None - } - }) - .collect::>(); + // The cores assigned to the parachain at the committed claim queue offset. + let assigned_cores = rp_info + .claim_queue + .iter() + .filter_map(move |(core_index, paras)| { + let para_at_offset = *paras.get(cq_offset.0 as usize)?; + if para_at_offset == candidate.descriptor().para_id() { + Some(core_index) + } else { + None + } + }) + .collect::>(); let assigned_cores = assigned_cores.as_slice(); // Check if core index in descriptor matches the one in the commitments @@ -1092,20 +1082,9 @@ fn sanitize_backed_candidates( // of the same para is preserved. let mut candidates_per_para: BTreeMap> = BTreeMap::new(); - // Get the eligible paras for each core. - let scheduled_paras = scheduled - .iter() - .map(|(para_id, cores)| cores.iter().map(|core| (*core, *para_id))) - .flatten() - .collect(); - for candidate in backed_candidates { - if !sanitize_backed_candidate_v2::( - &candidate, - allowed_relay_parents, - &scheduled_paras, - allow_v2_receipts, - ) { + if !sanitize_backed_candidate_v2::(&candidate, allowed_relay_parents, allow_v2_receipts) + { continue } diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 6700ab5307bc..2795130913f1 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -161,7 +161,7 @@ mod enter { backed_and_concluding, num_validators_per_core: 1, code_upgrade: None, - fill_claimqueue: false, + fill_claimqueue: true, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor, @@ -185,9 +185,6 @@ mod enter { .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) .unwrap(); - // The current schedule is empty prior to calling `create_inherent_enter`. - assert!(scheduler::Pallet::::claim_queue_is_empty()); - // Nothing is filtered out (including the backed candidates.) assert_eq!( Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(), @@ -267,7 +264,7 @@ mod enter { backed_and_concluding, num_validators_per_core: 1, code_upgrade: None, - fill_claimqueue: false, + fill_claimqueue: true, elastic_paras: [(2, 3)].into_iter().collect(), unavailable_cores: vec![], v2_descriptor, @@ -287,9 +284,6 @@ mod enter { .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) .unwrap(); - // The current schedule is empty prior to calling `create_inherent_enter`. - assert!(scheduler::Pallet::::claim_queue_is_empty()); - assert!(pallet::OnChainVotes::::get().is_none()); // Nothing is filtered out (including the backed candidates.) @@ -2018,6 +2012,7 @@ mod sanitizers { shared::Pallet::::add_allowed_relay_parent( default_header().hash(), Default::default(), + Default::default(), RELAY_PARENT_NUM, 1, ); @@ -2214,6 +2209,7 @@ mod sanitizers { shared::Pallet::::add_allowed_relay_parent( default_header().hash(), Default::default(), + Default::default(), RELAY_PARENT_NUM, 1, ); @@ -2733,6 +2729,7 @@ mod sanitizers { shared::Pallet::::add_allowed_relay_parent( relay_parent, Default::default(), + Default::default(), RELAY_PARENT_NUM, 1, ); @@ -3233,6 +3230,7 @@ mod sanitizers { shared::Pallet::::add_allowed_relay_parent( prev_relay_parent, Default::default(), + Default::default(), RELAY_PARENT_NUM - 1, 2, ); @@ -3240,6 +3238,7 @@ mod sanitizers { shared::Pallet::::add_allowed_relay_parent( relay_parent, Default::default(), + Default::default(), RELAY_PARENT_NUM, 2, ); @@ -3247,6 +3246,7 @@ mod sanitizers { shared::Pallet::::add_allowed_relay_parent( next_relay_parent, Default::default(), + Default::default(), RELAY_PARENT_NUM + 1, 2, ); From e93b9839a76fcd70b4b3cb0346cf2776ccb94eec Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 27 Aug 2024 23:52:49 +0300 Subject: [PATCH 062/103] typo Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/paras_inherent/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 6e8004101806..b3b7e402f3c4 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -1006,7 +1006,7 @@ fn sanitize_backed_candidate_v2( return false } - // Get the claim queue snapshot at the candidate relay paren. + // Get the claim queue snapshot at the candidate relay parent. let Some((rp_info, _)) = allowed_relay_parents.acquire_info(candidate.descriptor().relay_parent(), None) else { From e01bf53771923338a6a8e4ef72e5742546fb5be9 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 28 Aug 2024 14:03:30 +0300 Subject: [PATCH 063/103] it was a bad idea to process commitments of v1 receipts Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 15 +++----- .../parachains/src/paras_inherent/mod.rs | 38 +++++-------------- 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index e8b51d6b2202..203a0a448bae 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -527,9 +527,8 @@ impl CommittedCandidateReceiptV2 { /// Input `assigned_cores` must contain the sorted cores assigned to the para at /// the committed claim queue offset. pub fn check(&self, assigned_cores: &[&CoreIndex]) -> Result<(), CandidateReceiptError> { - // Don't check v1 descriptors without any `SelectCore` commitments. - if self.descriptor.version() == CandidateDescriptorVersion::V1 && - self.commitments.selected_core().is_none() + // Don't check v1 descriptors. + if self.descriptor.version() == CandidateDescriptorVersion::V1 { return Ok(()) } @@ -547,13 +546,9 @@ impl CommittedCandidateReceiptV2 { .committed_core_index(assigned_cores) .ok_or(CandidateReceiptError::NoAssignment)?; - // Only check descriptor `core_index` field of v2 descriptors. If it is v1, that field - // will be garbage. - if self.descriptor.version() == CandidateDescriptorVersion::V2 { - let descriptor_core_index = CoreIndex(self.descriptor.core_index as u32); - if core_index != descriptor_core_index { - return Err(CandidateReceiptError::CoreIndexMismatch) - } + let descriptor_core_index = CoreIndex(self.descriptor.core_index as u32); + if core_index != descriptor_core_index { + return Err(CandidateReceiptError::CoreIndexMismatch) } Ok(()) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index b3b7e402f3c4..146bb9c3a3d2 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -958,8 +958,7 @@ pub(crate) fn sanitize_bitfields( /// Perform required checks for given candidate receipt. /// -/// Returns `true` if candidate descriptor is version 1 without `UMPSignal` -/// commitments. +/// Returns `true` if candidate descriptor is version 1. /// /// Otherwise returns `false` if: /// - version 2 descriptors are not allowed @@ -970,18 +969,11 @@ fn sanitize_backed_candidate_v2( allowed_relay_parents: &AllowedRelayParentsTracker>, allow_v2_receipts: bool, ) -> bool { - // We move forward only if the candidate commits to a core selector and claim queue offset. - // If parachain runtime already makes use of `UMPSignal::SelectCore`, and the collator does not, - // we can expect v1 candidate receipts with `UMPSignal::SelectCore` commitments. - // - // It is important that we use these commitments of v1 receipts and not rely on injected cores. - // Parachain runtime upgrade is enough to enable both validators and the runtime to validate - // the core index the candidate has committed to. - let Some((core_selector, cq_offset)) = candidate.candidate().commitments.selected_core() else { - if candidate.descriptor().version() == CandidateDescriptorVersion::V1 { - return true - } + if candidate.descriptor().version() == CandidateDescriptorVersion::V1 { + return true + } + let Some((core_selector, cq_offset)) = candidate.candidate().commitments.selected_core() else { log::debug!( target: LOG_TARGET, "Dropping V2 candidate receipt {:?} for paraid {:?}, no `SelectCore` commitment.", @@ -992,8 +984,6 @@ fn sanitize_backed_candidate_v2( return false }; - // Drop any candidate receipts with `UMPSignal::SelectCore` commitments if nodes are not allowed - // to use them. // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure // any v1 descendants of v2 candidates are dropped. if !allow_v2_receipts { @@ -1034,8 +1024,7 @@ fn sanitize_backed_candidate_v2( .collect::>(); let assigned_cores = assigned_cores.as_slice(); - // Check if core index in descriptor matches the one in the commitments - // No-op for v1 candidate receipts. + // Check if core index in descriptor matches the one in the commitments. if let Err(err) = candidate.candidate().check(assigned_cores) { log::debug!( target: LOG_TARGET, @@ -1490,11 +1479,6 @@ fn map_candidates_to_cores(allowed_relay_parents, &candidate, &assigned_cores) + get_core_index::(allowed_relay_parents, &candidate) { if scheduled_cores.remove(&core_index) { temp_backed_candidates.push((candidate, core_index)); @@ -1573,15 +1557,11 @@ fn map_candidates_to_cores( allowed_relay_parents: &AllowedRelayParentsTracker>, candidate: &BackedCandidate, - assigned_cores: &BTreeSet, ) -> Option { - // Use the committed core index. It could be available even if descriptor is v1. - // We've already sanitized the candidate so it should be ok to trust it is valid. - // Fall back to injected core index. candidate .candidate() - .commitments - .committed_core_index(assigned_cores.iter().collect::>().as_slice()) + .descriptor + .core_index() .or_else(|| get_injected_core_index::(allowed_relay_parents, &candidate)) } From fb9fbe6bdaa8e5773f16eb4fa9272aa0e36ab5f3 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 28 Aug 2024 15:07:12 +0300 Subject: [PATCH 064/103] fmt Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 3 +-- polkadot/runtime/parachains/src/paras_inherent/mod.rs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 203a0a448bae..79c7a968fa4d 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -528,8 +528,7 @@ impl CommittedCandidateReceiptV2 { /// the committed claim queue offset. pub fn check(&self, assigned_cores: &[&CoreIndex]) -> Result<(), CandidateReceiptError> { // Don't check v1 descriptors. - if self.descriptor.version() == CandidateDescriptorVersion::V1 - { + if self.descriptor.version() == CandidateDescriptorVersion::V1 { return Ok(()) } diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 146bb9c3a3d2..e984470a33e6 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -1491,8 +1491,7 @@ fn map_candidates_to_cores(allowed_relay_parents, &candidate) + if let Some(core_index) = get_core_index::(allowed_relay_parents, &candidate) { if scheduled_cores.remove(&core_index) { temp_backed_candidates.push((candidate, core_index)); From c50748849771c50f02bbaa491865fb9360e7371a Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 28 Aug 2024 15:08:28 +0300 Subject: [PATCH 065/103] remove unused Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 79c7a968fa4d..64b5f8df8814 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -422,8 +422,6 @@ impl CandidateCommitments { /// CandidateReceipt construction errors. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] pub enum CandidateReceiptError { - /// The specified core index is invalid. - InvalidCoreIndex, /// The core index in commitments doesnt match the one in descriptor CoreIndexMismatch, /// The core selector or claim queue offset is invalid. From 178e2018d9162c634276ac248c513289032ee55c Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 28 Aug 2024 18:30:58 +0300 Subject: [PATCH 066/103] Validate session index Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/builder.rs | 4 +- .../runtime/parachains/src/inclusion/mod.rs | 2 +- .../parachains/src/paras_inherent/mod.rs | 24 +++++ .../parachains/src/paras_inherent/tests.rs | 101 +++++++++++++++++- 4 files changed, 125 insertions(+), 6 deletions(-) diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 7c5e5d7ad307..df806ab5940d 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -315,7 +315,7 @@ impl BenchBuilder { 0.into(), Default::default(), CoreIndex(200), - 1, + 2, Default::default(), Default::default(), Default::default(), @@ -671,7 +671,7 @@ impl BenchBuilder { para_id, relay_parent, core_idx, - 1, + 2, persisted_validation_data_hash, pov_hash, Default::default(), diff --git a/polkadot/runtime/parachains/src/inclusion/mod.rs b/polkadot/runtime/parachains/src/inclusion/mod.rs index 4cc21a70a024..077ac650c559 100644 --- a/polkadot/runtime/parachains/src/inclusion/mod.rs +++ b/polkadot/runtime/parachains/src/inclusion/mod.rs @@ -988,7 +988,7 @@ impl Pallet { pub(crate) fn receive_upward_messages(para: ParaId, upward_messages: &[Vec]) -> Weight { let bounded = upward_messages .iter() - // Stop once we hit the `UMPSignal`` separator. + // Stop once we hit the `UMPSignal` separator. .take_while(|message| !message.is_empty()) .filter_map(|d| { BoundedSlice::try_from(&d[..]) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index e984470a33e6..183c47f10568 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -984,6 +984,30 @@ fn sanitize_backed_candidate_v2( return false }; + // Check session index in the receipt. As we drop allowed relay parents at session change + // we only allow here candidates that have the session index equal to the current session. + let Some(session_index) = candidate.descriptor().session_index() else { + log::debug!( + target: LOG_TARGET, + "Invalid V2 candidate receipt {:?} for paraid {:?}, missing session index.", + candidate.candidate().hash(), + candidate.descriptor().para_id(), + ); + return false + }; + + if session_index != shared::CurrentSessionIndex::::get() { + log::debug!( + target: LOG_TARGET, + "Dropping V2 candidate receipt {:?} for paraid {:?}, invalid session index {}, current session {}", + candidate.candidate().hash(), + candidate.descriptor().para_id(), + session_index, + shared::CurrentSessionIndex::::get() + ); + return false + } + // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure // any v1 descendants of v2 candidates are dropped. if !allow_v2_receipts { diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 2795130913f1..78d5c050cf58 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -62,7 +62,9 @@ mod enter { use frame_support::assert_ok; use frame_system::limits; use polkadot_primitives::{ - vstaging::{InternalVersion, SchedulerParams}, + vstaging::{ + CandidateDescriptorV2, CommittedCandidateReceiptV2, InternalVersion, SchedulerParams, + }, AvailabilityBitfield, CandidateDescriptor, UncheckedSigned, }; use sp_runtime::Perbill; @@ -1549,7 +1551,6 @@ mod enter { num_validators_per_core: 5, code_upgrade: None, fill_claimqueue: true, - // 8 cores ! elastic_paras: [(2, 8)].into_iter().collect(), unavailable_cores: unavailable_cores.clone(), v2_descriptor: true, @@ -1627,7 +1628,6 @@ mod enter { num_validators_per_core: 1, code_upgrade: None, fill_claimqueue: true, - // 8 cores ! elastic_paras: [(2, 3)].into_iter().collect(), unavailable_cores: unavailable_cores.clone(), v2_descriptor: false, @@ -1713,6 +1713,101 @@ mod enter { Pallet::::enter(frame_system::RawOrigin::None.into(), inherent_data).unwrap(); }); } + + // A test to ensure that the paras_inherent filters out candidates + // with invalid sesison index in the descriptor. + #[test] + fn invalid_session_index() { + let config = default_config(); + assert!(config.configuration.config.scheduler_params.lookahead > 0); + new_test_ext(config).execute_with(|| { + // Set the elastic scaling MVP feature. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::ElasticScalingMVP as u8, + true, + ) + .unwrap(); + + // Enable the v2 receipts. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::CandidateReceiptV2 as u8, + true, + ) + .unwrap(); + + let mut backed_and_concluding = BTreeMap::new(); + backed_and_concluding.insert(0, 1); + backed_and_concluding.insert(1, 1); + backed_and_concluding.insert(2, 1); + + let unavailable_cores = vec![]; + + let scenario = make_inherent_data(TestConfig { + dispute_statements: BTreeMap::new(), + dispute_sessions: vec![], // No disputes + backed_and_concluding, + num_validators_per_core: 1, + code_upgrade: None, + fill_claimqueue: true, + elastic_paras: [(2, 3)].into_iter().collect(), + unavailable_cores, + v2_descriptor: true, + }); + + let mut inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (2 validators per core, 5 backed candidates) + assert_eq!(inherent_data.bitfields.len(), 5); + // * 5 v2 candidate descriptors passed, 1 is invalid + assert_eq!(inherent_data.backed_candidates.len(), 5); + + let index = inherent_data.backed_candidates.len() - 1; + + // Put invalid session index in last candidate + let backed_candidate = inherent_data.backed_candidates[index].clone(); + + let candidate = CommittedCandidateReceiptV2 { + descriptor: CandidateDescriptorV2::new( + backed_candidate.descriptor().para_id(), + backed_candidate.descriptor().relay_parent(), + backed_candidate.descriptor().core_index().unwrap(), + 100, + backed_candidate.descriptor().persisted_validation_data_hash(), + backed_candidate.descriptor().pov_hash(), + backed_candidate.descriptor().erasure_root(), + backed_candidate.descriptor().para_head(), + backed_candidate.descriptor().validation_code_hash(), + ), + commitments: backed_candidate.candidate().commitments.clone(), + }; + + inherent_data.backed_candidates[index] = BackedCandidate::new( + candidate, + backed_candidate.validity_votes().to_vec(), + backed_candidate.validator_indices_and_core_index(false).0.into(), + None, + ); + + let mut expected_inherent_data = inherent_data.clone(); + expected_inherent_data.backed_candidates.truncate(index); + + let mut create_inherent_data = InherentData::new(); + create_inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &inherent_data) + .unwrap(); + + // 1 candidate with invalid session is filtered out + assert_eq!( + Pallet::::create_inherent_inner(&create_inherent_data).unwrap(), + expected_inherent_data + ); + + Pallet::::enter(frame_system::RawOrigin::None.into(), inherent_data).unwrap_err(); + }); + } } fn default_header() -> polkadot_primitives::Header { From 984e8e1c97cf2c8238c6a6ea6b481211db543217 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 29 Aug 2024 11:49:06 +0300 Subject: [PATCH 067/103] add unknown version Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 962c9a467fe7..ccb4c132b0ae 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -126,6 +126,8 @@ pub enum CandidateDescriptorVersion { V1, /// The new `CandidateDescriptorV2`. V2, + /// An unknown version. + Unknown, } /// A unique descriptor of the candidate receipt. @@ -431,7 +433,7 @@ impl CandidateDescriptorV2 { match self.version.0 { 0 => CandidateDescriptorVersion::V2, - _ => CandidateDescriptorVersion::V1, + _ => CandidateDescriptorVersion::Unknown, } } @@ -816,8 +818,9 @@ mod tests { } #[test] - fn is_invalid_version_decodes_as_v1() { + fn invalid_version_descriptor() { let mut new_ccr = dummy_committed_candidate_receipt_v2(); + assert_eq!(new_ccr.descriptor.version(), CandidateDescriptorVersion::V2); // Put some unknown version. new_ccr.descriptor.version = InternalVersion(100); @@ -825,7 +828,7 @@ mod tests { let new_ccr: CommittedCandidateReceiptV2 = Decode::decode(&mut new_ccr.encode().as_slice()).unwrap(); - assert_eq!(new_ccr.descriptor.version(), CandidateDescriptorVersion::V1); + assert_eq!(new_ccr.descriptor.version(), CandidateDescriptorVersion::Unknown); } #[test] From fab215daebc0d1800bb387548c5306d4e54796fb Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 29 Aug 2024 11:56:24 +0300 Subject: [PATCH 068/103] add check for unknown version and test Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index ccb4c132b0ae..1838d94e53b8 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -114,7 +114,7 @@ impl> Default for SchedulerParams } /// A type representing the version of the candidate descriptor and internal version number. -#[derive(PartialEq, Eq, Encode, Decode, Clone, TypeInfo, RuntimeDebug)] +#[derive(PartialEq, Eq, Encode, Decode, Clone, TypeInfo, RuntimeDebug, Copy)] #[cfg_attr(feature = "std", derive(Hash))] pub struct InternalVersion(pub u8); @@ -403,6 +403,8 @@ pub enum CandidateReceiptError { NoAssignment, /// No core was selected. NoCoreSelected, + /// Unknown version. + UnknownVersion(InternalVersion), } macro_rules! impl_getter { @@ -503,6 +505,10 @@ impl CommittedCandidateReceiptV2 { return Ok(()) } + if self.descriptor.version() == CandidateDescriptorVersion::Unknown { + return Err(CandidateReceiptError::UnknownVersion(self.descriptor.version)) + } + if assigned_cores.is_empty() { return Err(CandidateReceiptError::NoAssignment) } @@ -829,6 +835,10 @@ mod tests { Decode::decode(&mut new_ccr.encode().as_slice()).unwrap(); assert_eq!(new_ccr.descriptor.version(), CandidateDescriptorVersion::Unknown); + assert_eq!( + new_ccr.check(&vec![].as_slice()), + Err(CandidateReceiptError::UnknownVersion(InternalVersion(100))) + ) } #[test] From 9bbe2cc788d28d993c73e52b60a4a8c417e10db1 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 29 Aug 2024 15:20:41 +0300 Subject: [PATCH 069/103] typo Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index c01c792c6d89..d154ba11af28 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -407,7 +407,7 @@ impl CandidateCommitments { pub enum CandidateReceiptError { /// The specified core index is invalid. InvalidCoreIndex, - /// The core index in commitments doesnt match the one in descriptor + /// The core index in commitments doesn't match the one in descriptor CoreIndexMismatch, /// The core selector or claim queue offset is invalid. InvalidSelectedCore, From 4dda9dfdf7d88360b5dfd361619e96b39a4e8ae7 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 29 Aug 2024 15:23:01 +0300 Subject: [PATCH 070/103] adjust comments Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/paras_inherent/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 183c47f10568..a445813d984a 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -984,8 +984,6 @@ fn sanitize_backed_candidate_v2( return false }; - // Check session index in the receipt. As we drop allowed relay parents at session change - // we only allow here candidates that have the session index equal to the current session. let Some(session_index) = candidate.descriptor().session_index() else { log::debug!( target: LOG_TARGET, @@ -996,6 +994,7 @@ fn sanitize_backed_candidate_v2( return false }; + // Check if session index is equal to current session index. if session_index != shared::CurrentSessionIndex::::get() { log::debug!( target: LOG_TARGET, From f8ef4ce2486aa32600faab709c2da7bfb6a17e19 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 30 Aug 2024 15:56:01 +0300 Subject: [PATCH 071/103] fix merge damage Signed-off-by: Andrei Sandu --- .../primitives/src/vstaging/async_backing.rs | 4 +-- polkadot/primitives/src/vstaging/mod.rs | 26 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/polkadot/primitives/src/vstaging/async_backing.rs b/polkadot/primitives/src/vstaging/async_backing.rs index bdf94e0f00db..8706214b5a01 100644 --- a/polkadot/primitives/src/vstaging/async_backing.rs +++ b/polkadot/primitives/src/vstaging/async_backing.rs @@ -37,7 +37,7 @@ pub struct CandidatePendingAvailability { } impl From> - for crate::v7::async_backing::CandidatePendingAvailability + for crate::v8::async_backing::CandidatePendingAvailability { fn from(value: CandidatePendingAvailability) -> Self { Self { @@ -62,7 +62,7 @@ pub struct BackingState { pub pending_availability: Vec>, } -impl From> for crate::v7::async_backing::BackingState { +impl From> for crate::v8::async_backing::BackingState { fn from(value: BackingState) -> Self { Self { constraints: value.constraints, diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 8bf098c84a82..95d044762710 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -193,25 +193,25 @@ pub enum CandidateEvent { CandidateTimedOut(CandidateReceiptV2, HeadData, CoreIndex), } -impl From> for super::v7::CandidateEvent { +impl From> for super::v8::CandidateEvent { fn from(value: CandidateEvent) -> Self { match value { CandidateEvent::CandidateBacked(receipt, head_data, core_index, group_index) => - super::v7::CandidateEvent::CandidateBacked( + super::v8::CandidateEvent::CandidateBacked( receipt.into(), head_data, core_index, group_index, ), CandidateEvent::CandidateIncluded(receipt, head_data, core_index, group_index) => - super::v7::CandidateEvent::CandidateIncluded( + super::v8::CandidateEvent::CandidateIncluded( receipt.into(), head_data, core_index, group_index, ), CandidateEvent::CandidateTimedOut(receipt, head_data, core_index) => - super::v7::CandidateEvent::CandidateTimedOut(receipt.into(), head_data, core_index), + super::v8::CandidateEvent::CandidateTimedOut(receipt.into(), head_data, core_index), } } } @@ -275,13 +275,13 @@ impl Ord for CommittedCandidateReceiptV2 { } } -impl From> for super::v7::CommittedCandidateReceipt { +impl From> for super::v8::CommittedCandidateReceipt { fn from(value: CommittedCandidateReceiptV2) -> Self { Self { descriptor: value.descriptor.into(), commitments: value.commitments } } } -impl From> for super::v7::CandidateReceipt { +impl From> for super::v8::CandidateReceipt { fn from(value: CandidateReceiptV2) -> Self { Self { descriptor: value.descriptor.into(), commitments_hash: value.commitments_hash } } @@ -609,7 +609,7 @@ pub struct ScrapedOnChainVotes { pub disputes: MultiDisputeStatementSet, } -impl From> for super::v7::ScrapedOnChainVotes { +impl From> for super::v8::ScrapedOnChainVotes { fn from(value: ScrapedOnChainVotes) -> Self { Self { session: value.session, @@ -672,7 +672,7 @@ pub enum CoreState { Free, } -impl From> for super::v7::OccupiedCore { +impl From> for super::v8::OccupiedCore { fn from(value: OccupiedCore) -> Self { Self { next_up_on_available: value.next_up_on_available, @@ -687,13 +687,13 @@ impl From> for super::v7::OccupiedCore { } } -impl From> for super::v7::CoreState { +impl From> for super::v8::CoreState { fn from(value: CoreState) -> Self { match value { - CoreState::Free => super::v7::CoreState::Free, - CoreState::Scheduled(core) => super::v7::CoreState::Scheduled(core), + CoreState::Free => super::v8::CoreState::Free, + CoreState::Scheduled(core) => super::v8::CoreState::Scheduled(core), CoreState::Occupied(occupied_core) => - super::v7::CoreState::Occupied(occupied_core.into()), + super::v8::CoreState::Occupied(occupied_core.into()), } } } @@ -702,7 +702,7 @@ impl From> for super::v7::CoreState { mod tests { use super::*; use crate::{ - v7::{ + v8::{ tests::dummy_committed_candidate_receipt as dummy_old_committed_candidate_receipt, CommittedCandidateReceipt, Hash, HeadData, ValidationCode, }, From 04e31a191201eb928796a14c94be8de2e67cf09e Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 30 Aug 2024 15:58:20 +0300 Subject: [PATCH 072/103] unused Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 95d044762710..57cba85c10d9 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -19,10 +19,10 @@ use crate::{ValidatorIndex, ValidityAttestation}; // Put any primitives used by staging APIs functions here use super::{ - async_backing::Constraints, Balance, BlakeTwo256, BlockNumber, CandidateCommitments, + async_backing::Constraints, BlakeTwo256, BlockNumber, CandidateCommitments, CandidateDescriptor, CandidateHash, CollatorId, CollatorSignature, CoreIndex, GroupIndex, Hash, HashT, HeadData, Header, Id, Id as ParaId, MultiDisputeStatementSet, ScheduledCore, - UncheckedSignedAvailabilityBitfields, ValidationCodeHash, ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, + UncheckedSignedAvailabilityBitfields, ValidationCodeHash, }; use bitvec::prelude::*; use sp_application_crypto::ByteArray; @@ -30,7 +30,6 @@ use sp_application_crypto::ByteArray; use alloc::{vec, vec::Vec}; use codec::{Decode, Encode}; use scale_info::TypeInfo; -use sp_arithmetic::Perbill; use sp_core::RuntimeDebug; use sp_runtime::traits::Header as HeaderT; use sp_staking::SessionIndex; From 5fd127949c93ff1d28cb455d78d54ebe7e54cc1a Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 30 Aug 2024 17:38:46 +0300 Subject: [PATCH 073/103] fix Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/paras_inherent/tests.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 0097c1267383..2675ece471cf 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -58,7 +58,7 @@ mod enter { use core::panic; use frame_support::assert_ok; use frame_system::limits; - use polkadot_primitives::{AvailabilityBitfield, SchedulerParams, UncheckedSigned}; + use polkadot_primitives::{AvailabilityBitfield, SchedulerParams, UncheckedSigned, vstaging::InternalVersion}; use sp_runtime::Perbill; struct TestConfig { @@ -988,6 +988,7 @@ mod enter { fill_claimqueue: true, elastic_paras: BTreeMap::new(), unavailable_cores: vec![], + v2_descriptor: false, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1633,7 +1634,7 @@ mod sanitizers { use bitvec::order::Lsb0; use polkadot_primitives::{ AvailabilityBitfield, GroupIndex, Hash, Id as ParaId, SignedAvailabilityBitfield, - ValidatorIndex, + ValidatorIndex }; use rstest::rstest; use sp_core::crypto::UncheckedFrom; From 19d6f325b9168908f98d474a0dc3c92223c25048 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 2 Sep 2024 15:33:17 +0300 Subject: [PATCH 074/103] fix benchmark build Signed-off-by: Andrei Sandu --- .../parachains/src/inclusion/benchmarking.rs | 27 +++++++++---------- .../parachains/src/paras_inherent/tests.rs | 6 +++-- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/polkadot/runtime/parachains/src/inclusion/benchmarking.rs b/polkadot/runtime/parachains/src/inclusion/benchmarking.rs index 978ef718ea40..cb6329bf88ea 100644 --- a/polkadot/runtime/parachains/src/inclusion/benchmarking.rs +++ b/polkadot/runtime/parachains/src/inclusion/benchmarking.rs @@ -25,10 +25,9 @@ use bitvec::{bitvec, prelude::Lsb0}; use frame_benchmarking::benchmarks; use pallet_message_queue as mq; use polkadot_primitives::{ - CandidateCommitments, CollatorId, CollatorSignature, CommittedCandidateReceipt, HrmpChannelId, - OutboundHrmpMessage, SessionIndex, + vstaging::CommittedCandidateReceiptV2 as CommittedCandidateReceipt, CandidateCommitments, + HrmpChannelId, OutboundHrmpMessage, SessionIndex, }; -use sp_core::sr25519; fn create_candidate_commitments( para_id: ParaId, @@ -124,17 +123,17 @@ benchmarks! { let core_index = CoreIndex::from(0); let backing_group = GroupIndex::from(0); - let descriptor = CandidateDescriptor:: { - para_id: para, - relay_parent: Default::default(), - collator: CollatorId::from(sr25519::Public::from_raw([42u8; 32])), - persisted_validation_data_hash: Default::default(), - pov_hash: Default::default(), - erasure_root: Default::default(), - signature: CollatorSignature::from(sr25519::Signature::from_raw([42u8; 64])), - para_head: Default::default(), - validation_code_hash: ValidationCode(vec![1, 2, 3]).hash(), - }; + let descriptor = CandidateDescriptor::::new( + para, + Default::default(), + CoreIndex(0), + 1, + Default::default(), + Default::default(), + Default::default(), + Default::default(), + ValidationCode(vec![1, 2, 3]).hash(), + ); let receipt = CommittedCandidateReceipt:: { descriptor, diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 2675ece471cf..ac42ac1611df 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -58,7 +58,9 @@ mod enter { use core::panic; use frame_support::assert_ok; use frame_system::limits; - use polkadot_primitives::{AvailabilityBitfield, SchedulerParams, UncheckedSigned, vstaging::InternalVersion}; + use polkadot_primitives::{ + vstaging::InternalVersion, AvailabilityBitfield, SchedulerParams, UncheckedSigned, + }; use sp_runtime::Perbill; struct TestConfig { @@ -1634,7 +1636,7 @@ mod sanitizers { use bitvec::order::Lsb0; use polkadot_primitives::{ AvailabilityBitfield, GroupIndex, Hash, Id as ParaId, SignedAvailabilityBitfield, - ValidatorIndex + ValidatorIndex, }; use rstest::rstest; use sp_core::crypto::UncheckedFrom; From 4ec3fc8f9588d46ac928474300056b5bbe5691d9 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 2 Sep 2024 19:31:41 +0300 Subject: [PATCH 075/103] typos Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index d9d5a07fc92c..93589ecfae0a 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -298,9 +298,9 @@ pub struct ClaimQueueOffset(pub u8); /// Signals that a parachain can send to the relay chain via the UMP queue. #[derive(PartialEq, Eq, Clone, Encode, Decode, TypeInfo, RuntimeDebug)] pub enum UMPSignal { - /// A message sent by a parachain to select the core the candidate is commited to. + /// A message sent by a parachain to select the core the candidate is committed to. /// Relay chain validators, in particular backers, use the `CoreSelector` and - /// `ClaimQueueOffset` to compute the index of the core the candidate has commited to. + /// `ClaimQueueOffset` to compute the index of the core the candidate has committed to. SelectCore(CoreSelector, ClaimQueueOffset), } /// Separator between `XCM` and `UMPSignal`. @@ -451,7 +451,7 @@ impl CandidateDescriptorV2 { } impl CommittedCandidateReceiptV2 { - /// Checks if descriptor core index is equal to the commited core index. + /// Checks if descriptor core index is equal to the committed core index. /// Input `assigned_cores` must contain the sorted cores assigned to the para at /// the committed claim queue offset. pub fn check(&self, assigned_cores: &[&CoreIndex]) -> Result<(), CandidateReceiptError> { From 2ba0a276bc7ae7fa38cb1c7bd0bd40d50df09e82 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 2 Sep 2024 19:31:59 +0300 Subject: [PATCH 076/103] fmt Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 3 ++- .../runtime/parachains/src/paras_inherent/tests.rs | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 93589ecfae0a..ddad8170aecc 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -459,7 +459,8 @@ impl CommittedCandidateReceiptV2 { // Don't check v1 descriptors. CandidateDescriptorVersion::V1 => return Ok(()), CandidateDescriptorVersion::V2 => {}, - CandidateDescriptorVersion::Unknown => return Err(CandidateReceiptError::UnknownVersion(self.descriptor.version)), + CandidateDescriptorVersion::Unknown => + return Err(CandidateReceiptError::UnknownVersion(self.descriptor.version)), } if self.commitments.selected_core().is_none() { diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index b57811d61dbc..d31d4bd2a734 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -21,10 +21,10 @@ use crate::{ configuration::{self, HostConfiguration}, mock::MockGenesisConfig, }; -use polkadot_primitives::vstaging::{ - ClaimQueueOffset, CoreSelector, UMPSignal, UMP_SEPARATOR, +use polkadot_primitives::{ + vstaging::{ClaimQueueOffset, CoreSelector, UMPSignal, UMP_SEPARATOR}, + SchedulerParams, }; -use polkadot_primitives::SchedulerParams; fn default_config() -> MockGenesisConfig { MockGenesisConfig { @@ -63,9 +63,7 @@ mod enter { use frame_support::assert_ok; use frame_system::limits; use polkadot_primitives::{ - vstaging::{ - CandidateDescriptorV2, CommittedCandidateReceiptV2, InternalVersion, - }, + vstaging::{CandidateDescriptorV2, CommittedCandidateReceiptV2, InternalVersion}, AvailabilityBitfield, CandidateDescriptor, UncheckedSigned, }; use sp_runtime::Perbill; From e468d62166e4ad264ab79e3346e908ecb1d7e8d7 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 2 Sep 2024 19:36:49 +0300 Subject: [PATCH 077/103] fix comment Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/ump_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/runtime/parachains/src/ump_tests.rs b/polkadot/runtime/parachains/src/ump_tests.rs index a687515d330e..571fc5ab2271 100644 --- a/polkadot/runtime/parachains/src/ump_tests.rs +++ b/polkadot/runtime/parachains/src/ump_tests.rs @@ -644,7 +644,7 @@ fn cannot_offboard_while_ump_dispatch_queued() { }); } -/// A para-chain cannot send an UMP to the relay chain while it is offboarding. +/// Test UMP signals are filtered out and don't consume `max_upward_message_num_per_candidate`. #[test] fn enqueue_ump_signals() { let para = 100.into(); From 18a0496807fe9f4a2c5ba657d9f157d4af6f6de4 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 4 Sep 2024 13:53:26 +0300 Subject: [PATCH 078/103] mixed v1, v2, v2 without select core tests, plus some benchbuilder improvements Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 148 +++++++++++++----- polkadot/runtime/parachains/src/builder.rs | 23 ++- .../parachains/src/paras_inherent/mod.rs | 47 ++---- .../parachains/src/paras_inherent/tests.rs | 106 ++++++++++++- 4 files changed, 245 insertions(+), 79 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index ddad8170aecc..e8598aa87600 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -24,12 +24,15 @@ use super::{ HashT, HeadData, Header, Id, Id as ParaId, MultiDisputeStatementSet, ScheduledCore, UncheckedSignedAvailabilityBitfields, ValidationCodeHash, }; +use alloc::{ + collections::{BTreeMap, VecDeque}, + vec, + vec::Vec, +}; use bitvec::prelude::*; -use sp_application_crypto::ByteArray; - -use alloc::{vec, vec::Vec}; use codec::{Decode, Encode}; use scale_info::TypeInfo; +use sp_application_crypto::ByteArray; use sp_core::RuntimeDebug; use sp_runtime::traits::Header as HeaderT; use sp_staking::SessionIndex; @@ -334,8 +337,10 @@ impl CandidateCommitments { /// `assigned_cores` must be a sorted vec of all core indices assigned to a parachain. pub fn committed_core_index(&self, assigned_cores: &[&CoreIndex]) -> Option { if assigned_cores.is_empty() { + println!("Assigned cores empty"); return None } + println!("Selected core: {:?}", self.selected_core()); self.selected_core().and_then(|(core_selector, _cq_offset)| { let core_index = @@ -356,7 +361,8 @@ pub enum CandidateReceiptError { InvalidSelectedCore, /// The parachain is not assigned to any core at specified claim queue offset. NoAssignment, - /// No core was selected. + /// No core was selected. The `SelectCore` commitment is mandatory for + /// v2 receipts if parachains has multiple cores assigned. NoCoreSelected, /// Unknown version. UnknownVersion(InternalVersion), @@ -452,9 +458,12 @@ impl CandidateDescriptorV2 { impl CommittedCandidateReceiptV2 { /// Checks if descriptor core index is equal to the committed core index. - /// Input `assigned_cores` must contain the sorted cores assigned to the para at - /// the committed claim queue offset. - pub fn check(&self, assigned_cores: &[&CoreIndex]) -> Result<(), CandidateReceiptError> { + /// Input `claim_queue` must contain a snapshot of the claim queue at the + /// candidate relay parent. + pub fn check( + &self, + claim_queue: &BTreeMap>, + ) -> Result<(), CandidateReceiptError> { match self.descriptor.version() { // Don't check v1 descriptors. CandidateDescriptorVersion::V1 => return Ok(()), @@ -463,18 +472,48 @@ impl CommittedCandidateReceiptV2 { return Err(CandidateReceiptError::UnknownVersion(self.descriptor.version)), } - if self.commitments.selected_core().is_none() { - return Err(CandidateReceiptError::NoCoreSelected) - } - - if assigned_cores.is_empty() { + if claim_queue.is_empty() { + println!("cq empty"); return Err(CandidateReceiptError::NoAssignment) } - let core_index = self - .commitments - .committed_core_index(assigned_cores) - .ok_or(CandidateReceiptError::NoAssignment)?; + let (offset, core_selected) = + if let Some((_core_selector, cq_offset)) = self.commitments.selected_core() { + (cq_offset.0 as usize, true) + } else { + // If no core has been selected then we use offset 0 (top of claim queue) + (0, false) + }; + + // The cores assigned to the parachain at above computed offset. + // + // TODO: this might be inneficient to do for each candidate. + // A BTreeMap> can be computed + // once per relay chain block, making this search here much faster. + let assigned_cores = claim_queue + .iter() + .filter_map(move |(core_index, paras)| { + let para_at_offset = *paras.get(offset)?; + if para_at_offset == self.descriptor.para_id() { + Some(core_index) + } else { + None + } + }) + .collect::>(); + + let core_index = if core_selected { + self.commitments + .committed_core_index(assigned_cores.as_slice()) + .ok_or(CandidateReceiptError::NoAssignment)? + } else { + // `SelectCore` commitment is mandatory for elastic scaling parachains. + if assigned_cores.len() > 1 { + return Err(CandidateReceiptError::NoCoreSelected) + } + + **assigned_cores.get(0).ok_or(CandidateReceiptError::NoAssignment)? + }; let descriptor_core_index = CoreIndex(self.descriptor.core_index as u32); if core_index != descriptor_core_index { @@ -531,6 +570,12 @@ impl BackedCandidate { &self.candidate } + /// Get a mutable reference to the committed candidate receipt of the candidate. + /// Only for testing. + #[cfg(feature = "test")] + pub fn candidate_mut(&mut self) -> &mut CommittedCandidateReceiptV2 { + &mut self.candidate + } /// Get a reference to the descriptor of the candidate. pub fn descriptor(&self) -> &CandidateDescriptorV2 { &self.candidate.descriptor @@ -797,7 +842,7 @@ mod tests { assert_eq!(new_ccr.descriptor.version(), CandidateDescriptorVersion::Unknown); assert_eq!( - new_ccr.check(&vec![].as_slice()), + new_ccr.check(&BTreeMap::new()), Err(CandidateReceiptError::UnknownVersion(InternalVersion(100))) ) } @@ -821,7 +866,13 @@ mod tests { .upward_messages .force_push(UMPSignal::SelectCore(CoreSelector(0), ClaimQueueOffset(1)).encode()); - assert_eq!(new_ccr.check(&vec![&CoreIndex(123)]), Ok(())); + let mut cq = BTreeMap::new(); + cq.insert( + CoreIndex(123), + vec![new_ccr.descriptor.para_id(), new_ccr.descriptor.para_id()].into(), + ); + + assert_eq!(new_ccr.check(&cq), Ok(())); } #[test] @@ -833,11 +884,12 @@ mod tests { new_ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); new_ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); - // The check should fail because no `SelectCore` signal was sent. - assert_eq!( - new_ccr.check(&vec![&CoreIndex(0), &CoreIndex(100)]), - Err(CandidateReceiptError::NoCoreSelected) - ); + let mut cq = BTreeMap::new(); + cq.insert(CoreIndex(0), vec![new_ccr.descriptor.para_id()].into()); + + // The check should not fail because no `SelectCore` signal was sent. + // The message is optional. + assert!(new_ccr.check(&cq).is_ok()); // Garbage message. new_ccr.commitments.upward_messages.force_push(vec![0, 13, 200].encode()); @@ -845,12 +897,18 @@ mod tests { // No `SelectCore` can be decoded. assert_eq!(new_ccr.commitments.selected_core(), None); - // Failure is expected. - assert_eq!( - new_ccr.check(&vec![&CoreIndex(0), &CoreIndex(100)]), - Err(CandidateReceiptError::NoCoreSelected) + let mut cq = BTreeMap::new(); + cq.insert( + CoreIndex(0), + vec![new_ccr.descriptor.para_id(), new_ccr.descriptor.para_id()].into(), + ); + cq.insert( + CoreIndex(100), + vec![new_ccr.descriptor.para_id(), new_ccr.descriptor.para_id()].into(), ); + assert_eq!(new_ccr.check(&cq), Err(CandidateReceiptError::NoCoreSelected)); + new_ccr.commitments.upward_messages.clear(); new_ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); @@ -866,7 +924,7 @@ mod tests { .force_push(UMPSignal::SelectCore(CoreSelector(1), ClaimQueueOffset(1)).encode()); // Duplicate doesn't override first signal. - assert_eq!(new_ccr.check(&vec![&CoreIndex(0), &CoreIndex(100)]), Ok(())); + assert_eq!(new_ccr.check(&cq), Ok(())); } #[test] @@ -903,7 +961,14 @@ mod tests { Decode::decode(&mut encoded_ccr.as_slice()).unwrap(); assert_eq!(v2_ccr.descriptor.core_index(), Some(CoreIndex(123))); - assert_eq!(new_ccr.check(&vec![&CoreIndex(123)]), Ok(())); + + let mut cq = BTreeMap::new(); + cq.insert( + CoreIndex(123), + vec![new_ccr.descriptor.para_id(), new_ccr.descriptor.para_id()].into(), + ); + + assert_eq!(new_ccr.check(&cq), Ok(())); assert_eq!(new_ccr.hash(), v2_ccr.hash()); } @@ -930,7 +995,12 @@ mod tests { assert_eq!(v1_ccr.descriptor.version(), CandidateDescriptorVersion::V1); assert!(v1_ccr.commitments.selected_core().is_some()); - assert!(v1_ccr.check(&vec![&CoreIndex(0), &CoreIndex(1)]).is_ok()); + + let mut cq = BTreeMap::new(); + cq.insert(CoreIndex(0), vec![v1_ccr.descriptor.para_id()].into()); + cq.insert(CoreIndex(1), vec![v1_ccr.descriptor.para_id()].into()); + + assert!(v1_ccr.check(&cq).is_ok()); assert_eq!( v1_ccr.commitments.committed_core_index(&vec![&CoreIndex(10), &CoreIndex(5)]), @@ -941,7 +1011,7 @@ mod tests { } #[test] - fn test_core_select_is_mandatory() { + fn test_core_select_is_optional() { // Testing edge case when collators provide zeroed signature and collator id. let mut old_ccr = dummy_old_committed_candidate_receipt(); old_ccr.descriptor.para_id = ParaId::new(1000); @@ -950,11 +1020,19 @@ mod tests { let new_ccr: CommittedCandidateReceiptV2 = Decode::decode(&mut encoded_ccr.as_slice()).unwrap(); + let mut cq = BTreeMap::new(); + cq.insert(CoreIndex(0), vec![new_ccr.descriptor.para_id()].into()); + // Since collator sig and id are zeroed, it means that the descriptor uses format - // version 2. - // We expect the check to fail in such case because there will be no `SelectCore` - // commitment. - assert_eq!(new_ccr.check(&vec![&CoreIndex(0)]), Err(CandidateReceiptError::NoCoreSelected)); + // version 2. Should still pass checks without core selector. + assert!(new_ccr.check(&cq).is_ok()); + + let mut cq = BTreeMap::new(); + cq.insert(CoreIndex(0), vec![new_ccr.descriptor.para_id()].into()); + cq.insert(CoreIndex(1), vec![new_ccr.descriptor.para_id()].into()); + + // Should fail because 2 cores are assigned, + assert_eq!(new_ccr.check(&cq), Err(CandidateReceiptError::NoCoreSelected)); // Adding collator signature should make it decode as v1. old_ccr.descriptor.signature = dummy_collator_signature(); diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index ca6c088860c9..b12637da2e42 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -144,9 +144,14 @@ pub(crate) struct BenchBuilder { unavailable_cores: Vec, /// Use v2 candidate descriptor. candidate_descriptor_v2: bool, + /// Apply custom changes to generated candidates + candidate_modifier: Option>, _phantom: core::marker::PhantomData, } +pub type CandidateModifier = + fn(CommittedCandidateReceipt) -> CommittedCandidateReceipt; + /// Paras inherent `enter` benchmark scenario. #[cfg(any(feature = "runtime-benchmarks", test))] pub(crate) struct Bench { @@ -176,6 +181,7 @@ impl BenchBuilder { fill_claimqueue: true, unavailable_cores: vec![], candidate_descriptor_v2: false, + candidate_modifier: None, _phantom: core::marker::PhantomData::, } } @@ -290,6 +296,15 @@ impl BenchBuilder { self } + /// Set the candidate modifier. + pub(crate) fn set_candidate_modifier( + mut self, + modifier: Option>, + ) -> Self { + self.candidate_modifier = modifier; + self + } + /// Get the maximum number of validators per core. fn max_validators_per_core(&self) -> u32 { self.max_validators_per_core.unwrap_or(Self::fallback_max_validators_per_core()) @@ -725,6 +740,11 @@ impl BenchBuilder { ); } + // Maybe apply the candidate modifier + if let Some(modifier) = self.candidate_modifier { + candidate = modifier(candidate); + } + let candidate_hash = candidate.hash(); let validity_votes: Vec<_> = group_validators @@ -744,7 +764,8 @@ impl BenchBuilder { }) .collect(); - let core_idx = if self.candidate_descriptor_v2 { + // Don't inject core when it is available in descriptor. + let core_idx = if candidate.descriptor.core_index().is_some() { None } else { configuration::ActiveConfig::::get() diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 0cb2574f82a2..fced2ad8a512 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -997,16 +997,17 @@ fn sanitize_backed_candidate_v2( return true } - let Some((core_selector, cq_offset)) = candidate.candidate().commitments.selected_core() else { + // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure + // any v1 descendants of v2 candidates are dropped. + if !allow_v2_receipts { log::debug!( target: LOG_TARGET, - "Dropping V2 candidate receipt {:?} for paraid {:?}, no `SelectCore` commitment.", + "V2 candidate descriptors not allowed. Dropping candidate {:?} for paraid {:?}.", candidate.candidate().hash(), candidate.descriptor().para_id() ); - return false - }; + } let Some(session_index) = candidate.descriptor().session_index() else { log::debug!( @@ -1031,18 +1032,6 @@ fn sanitize_backed_candidate_v2( return false } - // It is mandatory to filter these before calling `filter_unchained_candidates` to ensure - // any v1 descendants of v2 candidates are dropped. - if !allow_v2_receipts { - log::debug!( - target: LOG_TARGET, - "V2 candidate descriptors not allowed. Dropping candidate {:?} for paraid {:?}.", - candidate.candidate().hash(), - candidate.descriptor().para_id() - ); - return false - } - // Get the claim queue snapshot at the candidate relay parent. let Some((rp_info, _)) = allowed_relay_parents.acquire_info(candidate.descriptor().relay_parent(), None) @@ -1056,31 +1045,14 @@ fn sanitize_backed_candidate_v2( return false }; - // The cores assigned to the parachain at the committed claim queue offset. - let assigned_cores = rp_info - .claim_queue - .iter() - .filter_map(move |(core_index, paras)| { - let para_at_offset = *paras.get(cq_offset.0 as usize)?; - if para_at_offset == candidate.descriptor().para_id() { - Some(core_index) - } else { - None - } - }) - .collect::>(); - let assigned_cores = assigned_cores.as_slice(); - - // Check if core index in descriptor matches the one in the commitments. - if let Err(err) = candidate.candidate().check(assigned_cores) { + // Check validity of `core_index` and `session_index`. + if let Err(err) = candidate.candidate().check(&rp_info.claim_queue) { log::debug!( target: LOG_TARGET, - "Dropping candidate {:?} for paraid {:?}, {:?}, core_selector={:?}, cq_offset={:?}", + "Dropping candidate {:?} for paraid {:?}, {:?}", candidate.candidate().hash(), candidate.descriptor().para_id(), err, - core_selector, - cq_offset, ); return false @@ -1540,6 +1512,7 @@ fn map_candidates_to_cores(allowed_relay_parents, &candidate) { + println!("Core index: {:?}", core_index); if scheduled_cores.remove(&core_index) { temp_backed_candidates.push((candidate, core_index)); } else { @@ -1563,7 +1536,7 @@ fn map_candidates_to_cores MockGenesisConfig { mod enter { use super::{inclusion::tests::TestCandidateBuilder, *}; use crate::{ - builder::{junk_collator, junk_collator_signature, Bench, BenchBuilder}, + builder::{junk_collator, junk_collator_signature, Bench, BenchBuilder, CandidateModifier}, mock::{mock_assigner, new_test_ext, BlockLength, BlockWeights, RuntimeOrigin, Test}, scheduler::{ common::{Assignment, AssignmentProvider}, @@ -78,6 +78,7 @@ mod enter { elastic_paras: BTreeMap, unavailable_cores: Vec, v2_descriptor: bool, + candidate_modifier: Option::Hash>>, } fn make_inherent_data( @@ -91,6 +92,7 @@ mod enter { elastic_paras, unavailable_cores, v2_descriptor, + candidate_modifier, }: TestConfig, ) -> Bench { let extra_cores = elastic_paras @@ -109,7 +111,8 @@ mod enter { .set_dispute_sessions(&dispute_sessions[..]) .set_fill_claimqueue(fill_claimqueue) .set_unavailable_cores(unavailable_cores) - .set_candidate_descriptor_v2(v2_descriptor); + .set_candidate_descriptor_v2(v2_descriptor) + .set_candidate_modifier(candidate_modifier); // Setup some assignments as needed: mock_assigner::Pallet::::set_core_count(builder.max_cores()); @@ -166,6 +169,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor, + candidate_modifier: None, }); // We expect the scenario to have cores 0 & 1 with pending availability. The backed @@ -269,6 +273,7 @@ mod enter { elastic_paras: [(2, 3)].into_iter().collect(), unavailable_cores: vec![], v2_descriptor, + candidate_modifier: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -371,6 +376,7 @@ mod enter { elastic_paras: [(2, 4)].into_iter().collect(), unavailable_cores: unavailable_cores.clone(), v2_descriptor: false, + candidate_modifier: None, }); let mut expected_para_inherent_data = scenario.data.clone(); @@ -628,6 +634,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor: false, + candidate_modifier: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -702,6 +709,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor: false, + candidate_modifier: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -774,6 +782,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor: false, + candidate_modifier: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -862,6 +871,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor: false, + candidate_modifier: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -950,6 +960,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor: false, + candidate_modifier: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1010,6 +1021,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor: false, + candidate_modifier: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1097,6 +1109,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor: false, + candidate_modifier: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1205,6 +1218,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor: false, + candidate_modifier: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1274,6 +1288,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor: false, + candidate_modifier: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1341,6 +1356,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor: false, + candidate_modifier: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1464,6 +1480,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor, + candidate_modifier: None, }); let mut para_inherent_data = scenario.data.clone(); @@ -1554,6 +1571,7 @@ mod enter { elastic_paras: BTreeMap::new(), unavailable_cores: vec![], v2_descriptor: false, + candidate_modifier: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1613,6 +1631,7 @@ mod enter { elastic_paras: [(2, 8)].into_iter().collect(), unavailable_cores: unavailable_cores.clone(), v2_descriptor: true, + candidate_modifier: None, }); let mut unfiltered_para_inherent_data = scenario.data.clone(); @@ -1690,6 +1709,7 @@ mod enter { elastic_paras: [(2, 3)].into_iter().collect(), unavailable_cores: unavailable_cores.clone(), v2_descriptor: false, + candidate_modifier: None, }); let inherent_data = scenario.data.clone(); @@ -1741,10 +1761,10 @@ mod enter { num_validators_per_core: 1, code_upgrade: None, fill_claimqueue: true, - // 3 cores elastic_paras: [(2, 3)].into_iter().collect(), unavailable_cores: unavailable_cores.clone(), - v2_descriptor: false, + v2_descriptor: true, + candidate_modifier: None, }); let mut inherent_data = scenario.data.clone(); @@ -1773,8 +1793,81 @@ mod enter { }); } - // A test to ensure that the paras_inherent filters out candidates - // with invalid sesison index in the descriptor. + // Mixed test with v1, v2 with/without `UMPSignal::SelectCore` + #[test] + fn mixed_v1_and_v2_optional_commitments() { + let config = default_config(); + assert!(config.configuration.config.scheduler_params.lookahead > 0); + new_test_ext(config).execute_with(|| { + // Set the elastic scaling MVP feature. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::ElasticScalingMVP as u8, + true, + ) + .unwrap(); + + // Enable the v2 receipts. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::CandidateReceiptV2 as u8, + true, + ) + .unwrap(); + + let mut backed_and_concluding = BTreeMap::new(); + backed_and_concluding.insert(0, 1); + backed_and_concluding.insert(1, 1); + backed_and_concluding.insert(2, 1); + backed_and_concluding.insert(3, 1); + backed_and_concluding.insert(4, 1); + + let unavailable_cores = vec![]; + + let candidate_modifier = |mut candidate: CommittedCandidateReceiptV2| { + // first candidate has v2 descriptor with no commitments + if candidate.descriptor.para_id() == ParaId::from(0) { + candidate.commitments.upward_messages.clear(); + } + + if candidate.descriptor.para_id() > ParaId::from(2) { + let mut v1: CandidateDescriptor = candidate.descriptor.into(); + + v1.collator = junk_collator(); + v1.signature = junk_collator_signature(); + + candidate.descriptor = v1.into(); + } + candidate + }; + + let scenario = make_inherent_data(TestConfig { + dispute_statements: BTreeMap::new(), + dispute_sessions: vec![], // No disputes + backed_and_concluding, + num_validators_per_core: 1, + code_upgrade: None, + fill_claimqueue: true, + elastic_paras: Default::default(), + unavailable_cores: unavailable_cores.clone(), + v2_descriptor: true, + candidate_modifier: Some(candidate_modifier), + }); + + let inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (2 validators per core, 5 backed candidates) + assert_eq!(inherent_data.bitfields.len(), 5); + // * 5 v2 candidate descriptors. + assert_eq!(inherent_data.backed_candidates.len(), 5); + + Pallet::::enter(frame_system::RawOrigin::None.into(), inherent_data).unwrap(); + }); + } + + // A test to ensure that the `paras_inherent`` filters out candidates with invalid + // session index in the descriptor. #[test] fn invalid_session_index() { let config = default_config(); @@ -1813,6 +1906,7 @@ mod enter { elastic_paras: [(2, 3)].into_iter().collect(), unavailable_cores, v2_descriptor: true, + candidate_modifier: None, }); let mut inherent_data = scenario.data.clone(); From d3202699a4c37e3e2748f885a8be797c3255f082 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 4 Sep 2024 18:24:18 +0300 Subject: [PATCH 079/103] Add allowed relay parents storage migration Signed-off-by: Andrei Sandu --- .../parachains/src/paras_inherent/mod.rs | 1 - polkadot/runtime/parachains/src/shared.rs | 4 + .../parachains/src/shared/migration.rs | 174 ++++++++++++++++++ polkadot/runtime/rococo/src/lib.rs | 1 + polkadot/runtime/westend/src/lib.rs | 1 + 5 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 polkadot/runtime/parachains/src/shared/migration.rs diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index fced2ad8a512..554ba505abf8 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -1512,7 +1512,6 @@ fn map_candidates_to_cores(allowed_relay_parents, &candidate) { - println!("Core index: {:?}", core_index); if scheduled_cores.remove(&core_index) { temp_backed_candidates.push((candidate, core_index)); } else { diff --git a/polkadot/runtime/parachains/src/shared.rs b/polkadot/runtime/parachains/src/shared.rs index 7330fcea72c1..4e877607d92e 100644 --- a/polkadot/runtime/parachains/src/shared.rs +++ b/polkadot/runtime/parachains/src/shared.rs @@ -40,9 +40,13 @@ pub use pallet::*; // which guarantees that at least one full session has passed before any changes are applied. pub(crate) const SESSION_DELAY: SessionIndex = 2; +const LOG_TARGET: &str = "runtime::inclusion"; + #[cfg(test)] mod tests; +mod migration; + /// Information about a relay parent. #[derive(Encode, Decode, Default, TypeInfo, Debug)] pub struct RelayParentInfo { diff --git a/polkadot/runtime/parachains/src/shared/migration.rs b/polkadot/runtime/parachains/src/shared/migration.rs new file mode 100644 index 000000000000..46ee89912e8b --- /dev/null +++ b/polkadot/runtime/parachains/src/shared/migration.rs @@ -0,0 +1,174 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +use super::*; +use codec::{Decode, Encode}; +use frame_support::{ + pallet_prelude::ValueQuery, traits::UncheckedOnRuntimeUpgrade, weights::Weight, +}; +pub use v1::MigrateToV1; + +pub mod v0 { + use super::*; + use alloc::collections::vec_deque::VecDeque; + + #[derive(Encode, Decode, Default, TypeInfo)] + pub struct AllowedRelayParentsTracker { + // The past relay parents, paired with state roots, that are viable to build upon. + // + // They are in ascending chronologic order, so the newest relay parents are at + // the back of the deque. + // + // (relay_parent, state_root) + pub buffer: VecDeque<(Hash, Hash)>, + + // The number of the most recent relay-parent, if any. + // If the buffer is empty, this value has no meaning and may + // be nonsensical. + pub latest_number: BlockNumber, + } + + impl From> + for super::AllowedRelayParentsTracker + { + fn from(value: AllowedRelayParentsTracker) -> Self { + Self { + latest_number: value.latest_number, + buffer: value + .buffer + .into_iter() + .map(|(relay_parent, state_root)| super::RelayParentInfo { + relay_parent, + state_root, + claim_queue: Default::default(), + }) + .collect(), + } + } + } +} + +mod v1 { + use super::*; + use alloc::vec::Vec; + + use codec::Decode; + #[cfg(feature = "try-runtime")] + use frame_support::{ + ensure, + traits::{GetStorageVersion, StorageVersion}, + }; + + use frame_support::storage_alias; + + /// All allowed relay-parents storage at version 0. + #[storage_alias] + pub(crate) type AllowedRelayParents = StorageValue< + Pallet, + super::v0::AllowedRelayParentsTracker<::Hash, BlockNumberFor>, + ValueQuery, + >; + + pub struct VersionUncheckedMigrateToV1(core::marker::PhantomData); + + impl UncheckedOnRuntimeUpgrade for VersionUncheckedMigrateToV1 { + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { + log::trace!(target: crate::shared::LOG_TARGET, "Running pre_upgrade() for inclusion MigrateToV1"); + let bytes = u32::to_ne_bytes(AllowedRelayParents::::get().buffer.len() as u32); + + Ok(bytes.to_vec()) + } + + fn on_runtime_upgrade() -> Weight { + let mut weight: Weight = Weight::zero(); + + // Read old storage. + let old_rp_tracker = AllowedRelayParents::::take(); + + super::AllowedRelayParents::::set(old_rp_tracker.into()); + + weight = weight.saturating_add(T::DbWeight::get().reads_writes(1, 1)); + + weight + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { + log::trace!(target: crate::shared::LOG_TARGET, "Running post_upgrade() for inclusion MigrateToV1"); + ensure!( + Pallet::::on_chain_storage_version() >= StorageVersion::new(1), + "Storage version should be >= 1 after the migration" + ); + + let relay_parent_count = + u32::decode(&mut &state[..]).expect("Was properly encoded") as usize; + + let rp_tracker = AllowedRelayParents::::get(); + + ensure!( + relay_parent_count == rp_tracker.buffer.len(), + "Number of allowed relay parents should be the same as the one before the upgrade." + ); + + Ok(()) + } + } + + /// Migrate shared module storage to v1. + pub type MigrateToV1 = frame_support::migrations::VersionedMigration< + 0, + 1, + VersionUncheckedMigrateToV1, + Pallet, + ::DbWeight, + >; +} + +#[cfg(test)] +mod tests { + use super::{v1::VersionUncheckedMigrateToV1, *}; + use crate::mock::{new_test_ext, MockGenesisConfig, Test}; + use frame_support::traits::UncheckedOnRuntimeUpgrade; + use polkadot_primitives::Hash; + + #[test] + fn migrate_to_v1() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + let rp_tracker = v0::AllowedRelayParentsTracker { + latest_number: 9, + buffer: (0..10u64) + .into_iter() + .map(|idx| (Hash::from_low_u64_ne(idx), Hash::from_low_u64_ne(2 * idx))) + .collect::>(), + }; + + v1::AllowedRelayParents::::put(rp_tracker); + + as UncheckedOnRuntimeUpgrade>::on_runtime_upgrade(); + + let rp_tracker = AllowedRelayParents::::get(); + + for idx in 0..10u64 { + let relay_parent = Hash::from_low_u64_ne(idx); + let state_root = Hash::from_low_u64_ne(2 * idx); + let (info, block_num) = rp_tracker.acquire_info(relay_parent, None).unwrap(); + + assert!(info.claim_queue.is_empty()); + assert_eq!(info.relay_parent, relay_parent); + assert_eq!(info.state_root, state_root); + assert_eq!(block_num as u64, idx); + } + }); + } +} diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 6b046e190830..bae376bc49a0 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1679,6 +1679,7 @@ pub mod migrations { // permanent pallet_xcm::migration::MigrateToLatestXcmVersion, parachains_inclusion::migration::MigrateToV1, + parachains_shared::migration::MigrateToV1, ); } diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index b02c2d8c671e..e2c334e4edb4 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -1786,6 +1786,7 @@ pub mod migrations { Runtime, MaxAgentsToMigrate, >, + parachains_shared::migration::MigrateToV1, ); } From 8490488088e735fa5940c6bf2dbf8b4a20eeabf0 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 5 Sep 2024 15:34:08 +0300 Subject: [PATCH 080/103] fix migration Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 3 -- polkadot/runtime/parachains/src/shared.rs | 7 +++-- .../parachains/src/shared/migration.rs | 28 ++++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index e8598aa87600..cd6f1b85cfe0 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -337,10 +337,8 @@ impl CandidateCommitments { /// `assigned_cores` must be a sorted vec of all core indices assigned to a parachain. pub fn committed_core_index(&self, assigned_cores: &[&CoreIndex]) -> Option { if assigned_cores.is_empty() { - println!("Assigned cores empty"); return None } - println!("Selected core: {:?}", self.selected_core()); self.selected_core().and_then(|(core_selector, _cq_offset)| { let core_index = @@ -473,7 +471,6 @@ impl CommittedCandidateReceiptV2 { } if claim_queue.is_empty() { - println!("cq empty"); return Err(CandidateReceiptError::NoAssignment) } diff --git a/polkadot/runtime/parachains/src/shared.rs b/polkadot/runtime/parachains/src/shared.rs index 4e877607d92e..f9d534660b7c 100644 --- a/polkadot/runtime/parachains/src/shared.rs +++ b/polkadot/runtime/parachains/src/shared.rs @@ -40,12 +40,10 @@ pub use pallet::*; // which guarantees that at least one full session has passed before any changes are applied. pub(crate) const SESSION_DELAY: SessionIndex = 2; -const LOG_TARGET: &str = "runtime::inclusion"; - #[cfg(test)] mod tests; -mod migration; +pub mod migration; /// Information about a relay parent. #[derive(Encode, Decode, Default, TypeInfo, Debug)] @@ -142,8 +140,11 @@ impl pub mod pallet { use super::*; + const STORAGE_VERSION: StorageVersion = StorageVersion::new(1); + #[pallet::pallet] #[pallet::without_storage_info] + #[pallet::storage_version(STORAGE_VERSION)] pub struct Pallet(_); #[pallet::config] diff --git a/polkadot/runtime/parachains/src/shared/migration.rs b/polkadot/runtime/parachains/src/shared/migration.rs index 46ee89912e8b..77c646ed0aa3 100644 --- a/polkadot/runtime/parachains/src/shared/migration.rs +++ b/polkadot/runtime/parachains/src/shared/migration.rs @@ -16,7 +16,9 @@ use codec::{Decode, Encode}; use frame_support::{ pallet_prelude::ValueQuery, traits::UncheckedOnRuntimeUpgrade, weights::Weight, }; -pub use v1::MigrateToV1; + +#[cfg(feature = "try-runtime")] +const LOG_TARGET: &str = "runtime::shared"; pub mod v0 { use super::*; @@ -60,8 +62,8 @@ pub mod v0 { mod v1 { use super::*; - use alloc::vec::Vec; + #[cfg(feature = "try-runtime")] use codec::Decode; #[cfg(feature = "try-runtime")] use frame_support::{ @@ -84,7 +86,7 @@ mod v1 { impl UncheckedOnRuntimeUpgrade for VersionUncheckedMigrateToV1 { #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { - log::trace!(target: crate::shared::LOG_TARGET, "Running pre_upgrade() for inclusion MigrateToV1"); + log::trace!(LOG_TARGET, "Running pre_upgrade() for inclusion MigrateToV1"); let bytes = u32::to_ne_bytes(AllowedRelayParents::::get().buffer.len() as u32); Ok(bytes.to_vec()) @@ -105,7 +107,7 @@ mod v1 { #[cfg(feature = "try-runtime")] fn post_upgrade(state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { - log::trace!(target: crate::shared::LOG_TARGET, "Running post_upgrade() for inclusion MigrateToV1"); + log::trace!(target: LOG_TARGET, "Running post_upgrade() for inclusion MigrateToV1"); ensure!( Pallet::::on_chain_storage_version() >= StorageVersion::new(1), "Storage version should be >= 1 after the migration" @@ -124,17 +126,17 @@ mod v1 { Ok(()) } } - - /// Migrate shared module storage to v1. - pub type MigrateToV1 = frame_support::migrations::VersionedMigration< - 0, - 1, - VersionUncheckedMigrateToV1, - Pallet, - ::DbWeight, - >; } +/// Migrate shared module storage to v1. +pub type MigrateToV1 = frame_support::migrations::VersionedMigration< + 0, + 1, + v1::VersionUncheckedMigrateToV1, + Pallet, + ::DbWeight, +>; + #[cfg(test)] mod tests { use super::{v1::VersionUncheckedMigrateToV1, *}; From db6748630f4d2a91006f8ff3ac5dc6352db99291 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 5 Sep 2024 15:46:14 +0300 Subject: [PATCH 081/103] fix Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/shared/migration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/runtime/parachains/src/shared/migration.rs b/polkadot/runtime/parachains/src/shared/migration.rs index 77c646ed0aa3..0126f817e0da 100644 --- a/polkadot/runtime/parachains/src/shared/migration.rs +++ b/polkadot/runtime/parachains/src/shared/migration.rs @@ -86,7 +86,7 @@ mod v1 { impl UncheckedOnRuntimeUpgrade for VersionUncheckedMigrateToV1 { #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { - log::trace!(LOG_TARGET, "Running pre_upgrade() for inclusion MigrateToV1"); + log::trace!(target: LOG_TARGET, "Running pre_upgrade() for inclusion MigrateToV1"); let bytes = u32::to_ne_bytes(AllowedRelayParents::::get().buffer.len() as u32); Ok(bytes.to_vec()) From 03cf8c1d405ecee7e19eaa04bc95c13156a50d2f Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 5 Sep 2024 16:11:43 +0300 Subject: [PATCH 082/103] clippy Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/paras_inherent/tests.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 1a0078e62d29..b411e71d532c 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -15,16 +15,12 @@ // along with Polkadot. If not, see . use super::*; -use rstest::rstest; use crate::{ configuration::{self, HostConfiguration}, mock::MockGenesisConfig, }; -use polkadot_primitives::{ - vstaging::{ClaimQueueOffset, CoreSelector, UMPSignal, UMP_SEPARATOR}, - SchedulerParams, -}; +use polkadot_primitives::SchedulerParams; fn default_config() -> MockGenesisConfig { MockGenesisConfig { @@ -48,6 +44,9 @@ fn default_config() -> MockGenesisConfig { #[cfg(not(feature = "runtime-benchmarks"))] mod enter { use super::{inclusion::tests::TestCandidateBuilder, *}; + use polkadot_primitives::vstaging::{ClaimQueueOffset, CoreSelector, UMPSignal, UMP_SEPARATOR}; + use rstest::rstest; + use crate::{ builder::{junk_collator, junk_collator_signature, Bench, BenchBuilder, CandidateModifier}, mock::{mock_assigner, new_test_ext, BlockLength, BlockWeights, RuntimeOrigin, Test}, From 43f6de755895d5c683c9b18abc9275abb2bcbe96 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 5 Sep 2024 16:17:33 +0300 Subject: [PATCH 083/103] feedback Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 20 +++++++++---------- .../parachains/src/paras_inherent/mod.rs | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index cd6f1b85cfe0..23bb19a2f85d 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -458,7 +458,7 @@ impl CommittedCandidateReceiptV2 { /// Checks if descriptor core index is equal to the committed core index. /// Input `claim_queue` must contain a snapshot of the claim queue at the /// candidate relay parent. - pub fn check( + pub fn check_core_index( &self, claim_queue: &BTreeMap>, ) -> Result<(), CandidateReceiptError> { @@ -839,7 +839,7 @@ mod tests { assert_eq!(new_ccr.descriptor.version(), CandidateDescriptorVersion::Unknown); assert_eq!( - new_ccr.check(&BTreeMap::new()), + new_ccr.check_core_index(&BTreeMap::new()), Err(CandidateReceiptError::UnknownVersion(InternalVersion(100))) ) } @@ -869,7 +869,7 @@ mod tests { vec![new_ccr.descriptor.para_id(), new_ccr.descriptor.para_id()].into(), ); - assert_eq!(new_ccr.check(&cq), Ok(())); + assert_eq!(new_ccr.check_core_index(&cq), Ok(())); } #[test] @@ -886,7 +886,7 @@ mod tests { // The check should not fail because no `SelectCore` signal was sent. // The message is optional. - assert!(new_ccr.check(&cq).is_ok()); + assert!(new_ccr.check_core_index(&cq).is_ok()); // Garbage message. new_ccr.commitments.upward_messages.force_push(vec![0, 13, 200].encode()); @@ -904,7 +904,7 @@ mod tests { vec![new_ccr.descriptor.para_id(), new_ccr.descriptor.para_id()].into(), ); - assert_eq!(new_ccr.check(&cq), Err(CandidateReceiptError::NoCoreSelected)); + assert_eq!(new_ccr.check_core_index(&cq), Err(CandidateReceiptError::NoCoreSelected)); new_ccr.commitments.upward_messages.clear(); new_ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); @@ -921,7 +921,7 @@ mod tests { .force_push(UMPSignal::SelectCore(CoreSelector(1), ClaimQueueOffset(1)).encode()); // Duplicate doesn't override first signal. - assert_eq!(new_ccr.check(&cq), Ok(())); + assert_eq!(new_ccr.check_core_index(&cq), Ok(())); } #[test] @@ -965,7 +965,7 @@ mod tests { vec![new_ccr.descriptor.para_id(), new_ccr.descriptor.para_id()].into(), ); - assert_eq!(new_ccr.check(&cq), Ok(())); + assert_eq!(new_ccr.check_core_index(&cq), Ok(())); assert_eq!(new_ccr.hash(), v2_ccr.hash()); } @@ -997,7 +997,7 @@ mod tests { cq.insert(CoreIndex(0), vec![v1_ccr.descriptor.para_id()].into()); cq.insert(CoreIndex(1), vec![v1_ccr.descriptor.para_id()].into()); - assert!(v1_ccr.check(&cq).is_ok()); + assert!(v1_ccr.check_core_index(&cq).is_ok()); assert_eq!( v1_ccr.commitments.committed_core_index(&vec![&CoreIndex(10), &CoreIndex(5)]), @@ -1022,14 +1022,14 @@ mod tests { // Since collator sig and id are zeroed, it means that the descriptor uses format // version 2. Should still pass checks without core selector. - assert!(new_ccr.check(&cq).is_ok()); + assert!(new_ccr.check_core_index(&cq).is_ok()); let mut cq = BTreeMap::new(); cq.insert(CoreIndex(0), vec![new_ccr.descriptor.para_id()].into()); cq.insert(CoreIndex(1), vec![new_ccr.descriptor.para_id()].into()); // Should fail because 2 cores are assigned, - assert_eq!(new_ccr.check(&cq), Err(CandidateReceiptError::NoCoreSelected)); + assert_eq!(new_ccr.check_core_index(&cq), Err(CandidateReceiptError::NoCoreSelected)); // Adding collator signature should make it decode as v1. old_ccr.descriptor.signature = dummy_collator_signature(); diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 554ba505abf8..8ffbc635fae2 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -1045,8 +1045,8 @@ fn sanitize_backed_candidate_v2( return false }; - // Check validity of `core_index` and `session_index`. - if let Err(err) = candidate.candidate().check(&rp_info.claim_queue) { + // Check validity of `core_index`. + if let Err(err) = candidate.candidate().check_core_index(&rp_info.claim_queue) { log::debug!( target: LOG_TARGET, "Dropping candidate {:?} for paraid {:?}, {:?}", From 70e48d2e83eb4177661d38f6f7054c64fb2337a0 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 5 Sep 2024 19:09:16 +0300 Subject: [PATCH 084/103] sir, make it faster Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 26 ++++------ polkadot/runtime/parachains/src/shared.rs | 28 +++++++++-- .../runtime/parachains/src/shared/tests.rs | 49 +++++++++++++++++++ 3 files changed, 82 insertions(+), 21 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 23bb19a2f85d..83ddd64587a6 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -25,7 +25,7 @@ use super::{ UncheckedSignedAvailabilityBitfields, ValidationCodeHash, }; use alloc::{ - collections::{BTreeMap, VecDeque}, + collections::{BTreeMap, BTreeSet, VecDeque}, vec, vec::Vec, }; @@ -460,7 +460,7 @@ impl CommittedCandidateReceiptV2 { /// candidate relay parent. pub fn check_core_index( &self, - claim_queue: &BTreeMap>, + cores_per_para: &BTreeMap>>, ) -> Result<(), CandidateReceiptError> { match self.descriptor.version() { // Don't check v1 descriptors. @@ -470,7 +470,7 @@ impl CommittedCandidateReceiptV2 { return Err(CandidateReceiptError::UnknownVersion(self.descriptor.version)), } - if claim_queue.is_empty() { + if cores_per_para.is_empty() { return Err(CandidateReceiptError::NoAssignment) } @@ -483,20 +483,12 @@ impl CommittedCandidateReceiptV2 { }; // The cores assigned to the parachain at above computed offset. - // - // TODO: this might be inneficient to do for each candidate. - // A BTreeMap> can be computed - // once per relay chain block, making this search here much faster. - let assigned_cores = claim_queue - .iter() - .filter_map(move |(core_index, paras)| { - let para_at_offset = *paras.get(offset)?; - if para_at_offset == self.descriptor.para_id() { - Some(core_index) - } else { - None - } - }) + let assigned_cores = cores_per_para + .get(&self.descriptor.para_id()) + .ok_or(CandidateReceiptError::NoAssignment)? + .get(offset) + .ok_or(CandidateReceiptError::NoAssignment)? + .into_iter() .collect::>(); let core_index = if core_selected { diff --git a/polkadot/runtime/parachains/src/shared.rs b/polkadot/runtime/parachains/src/shared.rs index f9d534660b7c..511a6aff4258 100644 --- a/polkadot/runtime/parachains/src/shared.rs +++ b/polkadot/runtime/parachains/src/shared.rs @@ -20,7 +20,7 @@ //! dependent on any of the other pallets. use alloc::{ - collections::{btree_map::BTreeMap, vec_deque::VecDeque}, + collections::{btree_map::BTreeMap, btree_set::BTreeSet, vec_deque::VecDeque}, vec::Vec, }; use frame_support::{pallet_prelude::*, traits::DisabledValidators}; @@ -52,8 +52,9 @@ pub struct RelayParentInfo { pub relay_parent: Hash, // The state root at this block pub state_root: Hash, - // Claim queue snapshot - pub claim_queue: BTreeMap>, + // Claim queue snapshot, optimized for accessing the assignments by `ParaId`. + // For each para we store the cores assigned per depth. + pub claim_queue: BTreeMap>>, } /// Keeps tracks of information about all viable relay parents. @@ -85,10 +86,29 @@ impl number: BlockNumber, max_ancestry_len: u32, ) { + let mut per_para_claim_queue = BTreeMap::new(); + + // Re-map the claim queue by `ParaId`. + for (core, paras) in claim_queue { + // Iterate paras assigned to this core at each depth. + for (depth, para) in paras.into_iter().enumerate() { + let depths: &mut VecDeque> = + per_para_claim_queue.entry(para).or_default(); + let initialize_count = depth.saturating_sub(depths.len()) + 1; + depths.extend((0..initialize_count).into_iter().map(|_| BTreeSet::new())); + + depths[depth].insert(core); + } + } + // + 1 for the most recent block, which is always allowed. let buffer_size_limit = max_ancestry_len as usize + 1; - self.buffer.push_back(RelayParentInfo { relay_parent, state_root, claim_queue }); + self.buffer.push_back(RelayParentInfo { + relay_parent, + state_root, + claim_queue: per_para_claim_queue, + }); self.latest_number = number; while self.buffer.len() > buffer_size_limit { diff --git a/polkadot/runtime/parachains/src/shared/tests.rs b/polkadot/runtime/parachains/src/shared/tests.rs index fd4b6555a48c..32e6afa4f21b 100644 --- a/polkadot/runtime/parachains/src/shared/tests.rs +++ b/polkadot/runtime/parachains/src/shared/tests.rs @@ -52,6 +52,55 @@ fn tracker_earliest_block_number() { assert_eq!(tracker.hypothetical_earliest_block_number(now + 1, max_ancestry_len), 1); } +#[test] +fn tracker_claim_queue_remap() { + let mut tracker = AllowedRelayParentsTracker::::default(); + + let mut claim_queue = BTreeMap::new(); + claim_queue.insert(CoreIndex(0), vec![Id::from(0), Id::from(1), Id::from(2)].into()); + claim_queue.insert(CoreIndex(1), vec![Id::from(0), Id::from(0), Id::from(100)].into()); + claim_queue.insert(CoreIndex(2), vec![Id::from(1), Id::from(2), Id::from(100)].into()); + + tracker.update(Hash::zero(), Hash::zero(), claim_queue, 1u32, 3u32); + + let (info, _block_num) = tracker.acquire_info(Hash::zero(), None).unwrap(); + assert_eq!( + info.claim_queue.get(&Id::from(0)).unwrap()[0], + vec![CoreIndex(0), CoreIndex(1)].into_iter().collect::>() + ); + assert_eq!( + info.claim_queue.get(&Id::from(1)).unwrap()[0], + vec![CoreIndex(2)].into_iter().collect::>() + ); + assert_eq!(info.claim_queue.get(&Id::from(2)).unwrap()[0], BTreeSet::new()); + assert_eq!(info.claim_queue.get(&Id::from(100)).unwrap()[0], BTreeSet::new()); + + assert_eq!( + info.claim_queue.get(&Id::from(0)).unwrap()[1], + vec![CoreIndex(0)].into_iter().collect::>() + ); + assert_eq!( + info.claim_queue.get(&Id::from(1)).unwrap()[1], + vec![CoreIndex(1)].into_iter().collect::>() + ); + assert_eq!( + info.claim_queue.get(&Id::from(2)).unwrap()[1], + vec![CoreIndex(2)].into_iter().collect::>() + ); + assert_eq!(info.claim_queue.get(&Id::from(100)).unwrap()[1], BTreeSet::new()); + + assert_eq!(info.claim_queue.get(&Id::from(0)).unwrap()[1], BTreeSet::new()); + assert_eq!(info.claim_queue.get(&Id::from(1)).unwrap()[1], BTreeSet::new()); + assert_eq!( + info.claim_queue.get(&Id::from(2)).unwrap()[1], + vec![CoreIndex(0)].into_iter().collect::>() + ); + assert_eq!( + info.claim_queue.get(&Id::from(100)).unwrap()[1], + vec![CoreIndex(1), CoreIndex(1)].into_iter().collect::>() + ); +} + #[test] fn tracker_acquire_info() { let mut tracker = AllowedRelayParentsTracker::::default(); From 1e26c7315b7124db20df1a350e61c7df6f419b55 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 5 Sep 2024 19:09:54 +0300 Subject: [PATCH 085/103] fix Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/shared/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/runtime/parachains/src/shared/tests.rs b/polkadot/runtime/parachains/src/shared/tests.rs index 32e6afa4f21b..936f21694ac9 100644 --- a/polkadot/runtime/parachains/src/shared/tests.rs +++ b/polkadot/runtime/parachains/src/shared/tests.rs @@ -77,11 +77,11 @@ fn tracker_claim_queue_remap() { assert_eq!( info.claim_queue.get(&Id::from(0)).unwrap()[1], - vec![CoreIndex(0)].into_iter().collect::>() + vec![CoreIndex(1)].into_iter().collect::>() ); assert_eq!( info.claim_queue.get(&Id::from(1)).unwrap()[1], - vec![CoreIndex(1)].into_iter().collect::>() + vec![CoreIndex(0)].into_iter().collect::>() ); assert_eq!( info.claim_queue.get(&Id::from(2)).unwrap()[1], From f4e3fb5873e790a55680b4f02bcfcfad22f440f4 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 5 Sep 2024 19:11:18 +0300 Subject: [PATCH 086/103] one last fix Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/shared/tests.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/polkadot/runtime/parachains/src/shared/tests.rs b/polkadot/runtime/parachains/src/shared/tests.rs index 936f21694ac9..a78610903eaf 100644 --- a/polkadot/runtime/parachains/src/shared/tests.rs +++ b/polkadot/runtime/parachains/src/shared/tests.rs @@ -89,15 +89,15 @@ fn tracker_claim_queue_remap() { ); assert_eq!(info.claim_queue.get(&Id::from(100)).unwrap()[1], BTreeSet::new()); - assert_eq!(info.claim_queue.get(&Id::from(0)).unwrap()[1], BTreeSet::new()); - assert_eq!(info.claim_queue.get(&Id::from(1)).unwrap()[1], BTreeSet::new()); + assert_eq!(info.claim_queue.get(&Id::from(0)).unwrap()[2], BTreeSet::new()); + assert_eq!(info.claim_queue.get(&Id::from(1)).unwrap()[2], BTreeSet::new()); assert_eq!( - info.claim_queue.get(&Id::from(2)).unwrap()[1], + info.claim_queue.get(&Id::from(2)).unwrap()[2], vec![CoreIndex(0)].into_iter().collect::>() ); assert_eq!( - info.claim_queue.get(&Id::from(100)).unwrap()[1], - vec![CoreIndex(1), CoreIndex(1)].into_iter().collect::>() + info.claim_queue.get(&Id::from(100)).unwrap()[2], + vec![CoreIndex(1), CoreIndex(2)].into_iter().collect::>() ); } From 2e87ad3979f22167fd75a7a5852d7b22c4da5678 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 5 Sep 2024 23:34:50 +0300 Subject: [PATCH 087/103] fixes Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 45 +++++++++++++++---- polkadot/runtime/parachains/src/shared.rs | 25 +++-------- .../runtime/parachains/src/shared/tests.rs | 4 +- 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 83ddd64587a6..a21f2fa00392 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -750,6 +750,29 @@ impl From> for super::v8::CoreState { } } +/// Returns a mapping between the para id and the core indices assigned at different +/// depths in the claim queue. +pub fn remap_claim_queue( + claim_queue: BTreeMap>, +) -> BTreeMap>> { + let mut per_para_claim_queue = BTreeMap::new(); + + for (core, paras) in claim_queue { + // Iterate paras assigned to this core at each depth. + for (depth, para) in paras.into_iter().enumerate() { + let depths: &mut VecDeque> = + per_para_claim_queue.entry(para).or_insert_with(|| Default::default()); + + let initialize_count = (depth + 1).saturating_sub(depths.len()); + depths.extend((0..initialize_count).into_iter().map(|_| BTreeSet::new())); + depths[depth].insert(core); + } + } + + println!("Remaped CQ: {:?}", per_para_claim_queue); + per_para_claim_queue +} + #[cfg(test)] mod tests { use super::*; @@ -861,7 +884,7 @@ mod tests { vec![new_ccr.descriptor.para_id(), new_ccr.descriptor.para_id()].into(), ); - assert_eq!(new_ccr.check_core_index(&cq), Ok(())); + assert_eq!(new_ccr.check_core_index(&remap_claim_queue(cq)), Ok(())); } #[test] @@ -878,7 +901,7 @@ mod tests { // The check should not fail because no `SelectCore` signal was sent. // The message is optional. - assert!(new_ccr.check_core_index(&cq).is_ok()); + assert!(new_ccr.check_core_index(&remap_claim_queue(cq)).is_ok()); // Garbage message. new_ccr.commitments.upward_messages.force_push(vec![0, 13, 200].encode()); @@ -896,7 +919,10 @@ mod tests { vec![new_ccr.descriptor.para_id(), new_ccr.descriptor.para_id()].into(), ); - assert_eq!(new_ccr.check_core_index(&cq), Err(CandidateReceiptError::NoCoreSelected)); + assert_eq!( + new_ccr.check_core_index(&remap_claim_queue(cq.clone())), + Err(CandidateReceiptError::NoCoreSelected) + ); new_ccr.commitments.upward_messages.clear(); new_ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); @@ -913,7 +939,7 @@ mod tests { .force_push(UMPSignal::SelectCore(CoreSelector(1), ClaimQueueOffset(1)).encode()); // Duplicate doesn't override first signal. - assert_eq!(new_ccr.check_core_index(&cq), Ok(())); + assert_eq!(new_ccr.check_core_index(&remap_claim_queue(cq)), Ok(())); } #[test] @@ -957,7 +983,7 @@ mod tests { vec![new_ccr.descriptor.para_id(), new_ccr.descriptor.para_id()].into(), ); - assert_eq!(new_ccr.check_core_index(&cq), Ok(())); + assert_eq!(new_ccr.check_core_index(&remap_claim_queue(cq)), Ok(())); assert_eq!(new_ccr.hash(), v2_ccr.hash()); } @@ -989,7 +1015,7 @@ mod tests { cq.insert(CoreIndex(0), vec![v1_ccr.descriptor.para_id()].into()); cq.insert(CoreIndex(1), vec![v1_ccr.descriptor.para_id()].into()); - assert!(v1_ccr.check_core_index(&cq).is_ok()); + assert!(v1_ccr.check_core_index(&remap_claim_queue(cq)).is_ok()); assert_eq!( v1_ccr.commitments.committed_core_index(&vec![&CoreIndex(10), &CoreIndex(5)]), @@ -1014,14 +1040,17 @@ mod tests { // Since collator sig and id are zeroed, it means that the descriptor uses format // version 2. Should still pass checks without core selector. - assert!(new_ccr.check_core_index(&cq).is_ok()); + assert!(new_ccr.check_core_index(&remap_claim_queue(cq)).is_ok()); let mut cq = BTreeMap::new(); cq.insert(CoreIndex(0), vec![new_ccr.descriptor.para_id()].into()); cq.insert(CoreIndex(1), vec![new_ccr.descriptor.para_id()].into()); // Should fail because 2 cores are assigned, - assert_eq!(new_ccr.check_core_index(&cq), Err(CandidateReceiptError::NoCoreSelected)); + assert_eq!( + new_ccr.check_core_index(&remap_claim_queue(cq)), + Err(CandidateReceiptError::NoCoreSelected) + ); // Adding collator signature should make it decode as v1. old_ccr.descriptor.signature = dummy_collator_signature(); diff --git a/polkadot/runtime/parachains/src/shared.rs b/polkadot/runtime/parachains/src/shared.rs index 511a6aff4258..4e982313791c 100644 --- a/polkadot/runtime/parachains/src/shared.rs +++ b/polkadot/runtime/parachains/src/shared.rs @@ -25,7 +25,9 @@ use alloc::{ }; use frame_support::{pallet_prelude::*, traits::DisabledValidators}; use frame_system::pallet_prelude::BlockNumberFor; -use polkadot_primitives::{CoreIndex, Id, SessionIndex, ValidatorId, ValidatorIndex}; +use polkadot_primitives::{ + vstaging::remap_claim_queue, CoreIndex, Id, SessionIndex, ValidatorId, ValidatorIndex, +}; use sp_runtime::traits::AtLeast32BitUnsigned; use rand::{seq::SliceRandom, SeedableRng}; @@ -86,29 +88,12 @@ impl number: BlockNumber, max_ancestry_len: u32, ) { - let mut per_para_claim_queue = BTreeMap::new(); - - // Re-map the claim queue by `ParaId`. - for (core, paras) in claim_queue { - // Iterate paras assigned to this core at each depth. - for (depth, para) in paras.into_iter().enumerate() { - let depths: &mut VecDeque> = - per_para_claim_queue.entry(para).or_default(); - let initialize_count = depth.saturating_sub(depths.len()) + 1; - depths.extend((0..initialize_count).into_iter().map(|_| BTreeSet::new())); - - depths[depth].insert(core); - } - } + let claim_queue = remap_claim_queue(claim_queue); // + 1 for the most recent block, which is always allowed. let buffer_size_limit = max_ancestry_len as usize + 1; - self.buffer.push_back(RelayParentInfo { - relay_parent, - state_root, - claim_queue: per_para_claim_queue, - }); + self.buffer.push_back(RelayParentInfo { relay_parent, state_root, claim_queue }); self.latest_number = number; while self.buffer.len() > buffer_size_limit { diff --git a/polkadot/runtime/parachains/src/shared/tests.rs b/polkadot/runtime/parachains/src/shared/tests.rs index a78610903eaf..fb04db5d79bb 100644 --- a/polkadot/runtime/parachains/src/shared/tests.rs +++ b/polkadot/runtime/parachains/src/shared/tests.rs @@ -89,8 +89,8 @@ fn tracker_claim_queue_remap() { ); assert_eq!(info.claim_queue.get(&Id::from(100)).unwrap()[1], BTreeSet::new()); - assert_eq!(info.claim_queue.get(&Id::from(0)).unwrap()[2], BTreeSet::new()); - assert_eq!(info.claim_queue.get(&Id::from(1)).unwrap()[2], BTreeSet::new()); + assert_eq!(info.claim_queue.get(&Id::from(0)).unwrap().get(2), None); + assert_eq!(info.claim_queue.get(&Id::from(1)).unwrap().get(2), None); assert_eq!( info.claim_queue.get(&Id::from(2)).unwrap()[2], vec![CoreIndex(0)].into_iter().collect::>() From 54432becf69429854ab46e21268df011f7498370 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 5 Sep 2024 23:36:05 +0300 Subject: [PATCH 088/103] remove println Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index a21f2fa00392..1a08aa691687 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -769,7 +769,6 @@ pub fn remap_claim_queue( } } - println!("Remaped CQ: {:?}", per_para_claim_queue); per_para_claim_queue } From cfbecb0e16c8b92878562558f195e13e21fe8603 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 6 Sep 2024 18:09:47 +0300 Subject: [PATCH 089/103] add prdoc Signed-off-by: Andrei Sandu --- prdoc/pr_5423.prdoc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 prdoc/pr_5423.prdoc diff --git a/prdoc/pr_5423.prdoc b/prdoc/pr_5423.prdoc new file mode 100644 index 000000000000..ee75b3d7849a --- /dev/null +++ b/prdoc/pr_5423.prdoc @@ -0,0 +1,18 @@ +title: Runtime support for candidate receipt v2 (RFC103) + +doc: + - audience: [Runtime Dev, Node Dev] + description: | + Implementation of [RFC103](https://github.com/polkadot-fellows/RFCs/pull/103) in the relay chain runtime. + The runtime will accept and validate the new receipts only if the `FeatureIndex::CandidateReceiptV2` + feature bit is enabled. + +crates: + - name: polkadot-primitives + bump: major + - name: polkadot-runtime-parachains + bump: patch + - name: rococo-runtime + bump: patch + - name: westend-runtime + bump: patch \ No newline at end of file From 3a518f25066a18b750ae8a3b5eaaf80fa4a13ee4 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 6 Sep 2024 18:16:21 +0300 Subject: [PATCH 090/103] fix comment Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 1a08aa691687..6bdbfc632705 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -456,8 +456,8 @@ impl CandidateDescriptorV2 { impl CommittedCandidateReceiptV2 { /// Checks if descriptor core index is equal to the committed core index. - /// Input `claim_queue` must contain a snapshot of the claim queue at the - /// candidate relay parent. + /// Input `cores_per_para` is a claim queue snapshot stored as a mapping + /// between `ParaId` and the cores assigned per depth. pub fn check_core_index( &self, cores_per_para: &BTreeMap>>, From 54106e254fb753307376d26e6645a9eab2300565 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 6 Sep 2024 19:01:45 +0300 Subject: [PATCH 091/103] refactor map_candidates_to_cores Signed-off-by: Andrei Sandu --- .../parachains/src/paras_inherent/mod.rs | 175 +++++++++--------- 1 file changed, 84 insertions(+), 91 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 8ffbc635fae2..3db9cee6db31 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -1467,104 +1467,96 @@ fn map_candidates_to_cores= 1 && core_index_enabled { - // We must preserve the dependency order given in the input. - let mut temp_backed_candidates = Vec::with_capacity(scheduled_cores.len()); - - for candidate in backed_candidates { - if scheduled_cores.len() == 0 { - // We've got candidates for all of this para's assigned cores. Move on to - // the next para. - log::debug!( - target: LOG_TARGET, - "Found enough candidates for paraid: {:?}.", - candidate.descriptor().para_id() - ); - break; - } - - if let Some(core_index) = get_core_index::(allowed_relay_parents, &candidate) - { - if scheduled_cores.remove(&core_index) { - temp_backed_candidates.push((candidate, core_index)); - } else { - // if we got a candidate for a core index which is not scheduled, stop - // the work for this para. the already processed candidate chain in - // temp_backed_candidates is still fine though. - log::debug!( - target: LOG_TARGET, - "Found a backed candidate {:?} with core index {}, which is not scheduled for paraid {:?}.", - candidate.candidate().hash(), - core_index.0, - candidate.descriptor().para_id() - ); - - break; - } - } else { - // if we got a candidate which does not contain its core index, stop the - // work for this para. the already processed candidate chain in - // temp_backed_candidates is still fine though. - - log::debug!( - target: LOG_TARGET, - "Found a backed candidate {:?} without core index informationm, but paraid {:?} has multiple scheduled cores.", - candidate.candidate().hash(), - candidate.descriptor().para_id() - ); - - break; - } - } + if let Some(core_index) = + get_core_index::(core_index_enabled, allowed_relay_parents, &candidate) + { + if scheduled_cores.remove(&core_index) { + temp_backed_candidates.push((candidate, core_index)); + } else { + // if we got a candidate for a core index which is not scheduled, stop + // the work for this para. the already processed candidate chain in + // temp_backed_candidates is still fine though. + log::debug!( + target: LOG_TARGET, + "Found a backed candidate {:?} with core index {}, which is not scheduled for paraid {:?}.", + candidate.candidate().hash(), + core_index.0, + candidate.descriptor().para_id() + ); - if !temp_backed_candidates.is_empty() { - backed_candidates_with_core - .entry(para_id) - .or_insert_with(|| vec![]) - .extend(temp_backed_candidates); + break; } } else { - log::warn!( + // No core index is fine, if para has just 1 core assigned. + if scheduled_cores.len() == 1 { + backed_candidates_with_core.insert( + para_id, + vec![( + // We need the first one here, as we assume candidates of a + // para are in dependency order. + candidate, + scheduled_cores.pop_first().expect("Length is 1"), + )], + ); + break; + } + + // if we got a candidate which does not contain its core index, stop the + // work for this para. the already processed candidate chain in + // temp_backed_candidates is still fine though. + + log::debug!( target: LOG_TARGET, - "Found a paraid {:?} which has multiple scheduled cores but ElasticScalingMVP feature is not enabled: {:?}", - para_id, - scheduled_cores + "Found a backed candidate {:?} without core index informationm, but paraid {:?} has multiple scheduled cores.", + candidate.candidate().hash(), + candidate.descriptor().para_id() ); + + break; } - } else { - log::debug!( - target: LOG_TARGET, - "Paraid: {:?} has no entry in scheduled cores but {} candidates were supplied.", - para_id, - backed_candidates.len() - ); + } + + if !temp_backed_candidates.is_empty() { + backed_candidates_with_core + .entry(para_id) + .or_insert_with(|| vec![]) + .extend(temp_backed_candidates); } } @@ -1573,24 +1565,25 @@ fn map_candidates_to_cores( + core_index_enabled: bool, allowed_relay_parents: &AllowedRelayParentsTracker>, candidate: &BackedCandidate, ) -> Option { - candidate - .candidate() - .descriptor - .core_index() - .or_else(|| get_injected_core_index::(allowed_relay_parents, &candidate)) + candidate.candidate().descriptor.core_index().or_else(|| { + get_injected_core_index::(core_index_enabled, allowed_relay_parents, &candidate) + }) } fn get_injected_core_index( + core_index_enabled: bool, allowed_relay_parents: &AllowedRelayParentsTracker>, candidate: &BackedCandidate, ) -> Option { // After stripping the 8 bit extensions, the `validator_indices` field length is expected // to be equal to backing group size. If these don't match, the `CoreIndex` is badly encoded, // or not supported. - let (validator_indices, maybe_core_idx) = candidate.validator_indices_and_core_index(true); + let (validator_indices, maybe_core_idx) = + candidate.validator_indices_and_core_index(core_index_enabled); let Some(core_idx) = maybe_core_idx else { return None }; From b44a6043d339624e03c01afe024184b6fafdefdb Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 9 Sep 2024 14:14:42 +0300 Subject: [PATCH 092/103] doc updates Signed-off-by: Andrei Sandu --- .../src/node/utility/candidate-validation.md | 2 +- polkadot/roadmap/implementers-guide/src/runtime/inclusion.md | 2 +- polkadot/runtime/parachains/src/paras_inherent/mod.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/candidate-validation.md b/polkadot/roadmap/implementers-guide/src/node/utility/candidate-validation.md index 1a3ff1c6aff0..aad77de0aded 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/candidate-validation.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/candidate-validation.md @@ -85,7 +85,7 @@ state. Once we have all parameters, we can spin up a background task to perform the validation in a way that doesn't hold up the entire event loop. Before invoking the validation function itself, this should first do some basic checks: - * The collator signature is valid + * The collator signature is valid (only if `CandidateDescriptor` has version 1) * The PoV provided matches the `pov_hash` field of the descriptor For more details please see [PVF Host and Workers](pvf-host-and-workers.md). diff --git a/polkadot/roadmap/implementers-guide/src/runtime/inclusion.md b/polkadot/roadmap/implementers-guide/src/runtime/inclusion.md index 5031433cf5a1..48909db07ba5 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/inclusion.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/inclusion.md @@ -109,7 +109,7 @@ All failed checks should lead to an unrecoverable error making the block invalid 1. Ensure that any code upgrade scheduled by the candidate does not happen within `config.validation_upgrade_cooldown` of `Paras::last_code_upgrade(para_id, true)`, if any, comparing against the value of `Paras::FutureCodeUpgrades` for the given para ID. - 1. Check the collator's signature on the candidate data. + 1. Check the collator's signature on the candidate data (only if `CandidateDescriptor` is version 1) 1. check the backing of the candidate using the signatures and the bitfields, comparing against the validators assigned to the groups, fetched with the `group_validators` lookup, while group indices are computed by `Scheduler` according to group rotation info. diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 3db9cee6db31..f571dba314a5 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -1448,8 +1448,8 @@ fn filter_unchained_candidates Date: Fri, 13 Sep 2024 15:42:34 +0300 Subject: [PATCH 093/103] feedback Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 32 +++++++-------- polkadot/runtime/parachains/src/builder.rs | 2 +- .../parachains/src/paras_inherent/mod.rs | 13 ++----- .../parachains/src/paras_inherent/tests.rs | 12 +++--- polkadot/runtime/parachains/src/shared.rs | 6 +-- .../parachains/src/shared/migration.rs | 39 +++++++++++-------- .../runtime/parachains/src/shared/tests.rs | 24 ++++++------ 7 files changed, 63 insertions(+), 65 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 6bdbfc632705..ebd93ce77091 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -460,7 +460,7 @@ impl CommittedCandidateReceiptV2 { /// between `ParaId` and the cores assigned per depth. pub fn check_core_index( &self, - cores_per_para: &BTreeMap>>, + cores_per_para: &BTreeMap>>, ) -> Result<(), CandidateReceiptError> { match self.descriptor.version() { // Don't check v1 descriptors. @@ -476,7 +476,7 @@ impl CommittedCandidateReceiptV2 { let (offset, core_selected) = if let Some((_core_selector, cq_offset)) = self.commitments.selected_core() { - (cq_offset.0 as usize, true) + (cq_offset.0, true) } else { // If no core has been selected then we use offset 0 (top of claim queue) (0, false) @@ -486,7 +486,7 @@ impl CommittedCandidateReceiptV2 { let assigned_cores = cores_per_para .get(&self.descriptor.para_id()) .ok_or(CandidateReceiptError::NoAssignment)? - .get(offset) + .get(&offset) .ok_or(CandidateReceiptError::NoAssignment)? .into_iter() .collect::>(); @@ -752,20 +752,18 @@ impl From> for super::v8::CoreState { /// Returns a mapping between the para id and the core indices assigned at different /// depths in the claim queue. -pub fn remap_claim_queue( +pub fn transpose_claim_queue( claim_queue: BTreeMap>, -) -> BTreeMap>> { +) -> BTreeMap>> { let mut per_para_claim_queue = BTreeMap::new(); for (core, paras) in claim_queue { // Iterate paras assigned to this core at each depth. for (depth, para) in paras.into_iter().enumerate() { - let depths: &mut VecDeque> = + let depths: &mut BTreeMap> = per_para_claim_queue.entry(para).or_insert_with(|| Default::default()); - let initialize_count = (depth + 1).saturating_sub(depths.len()); - depths.extend((0..initialize_count).into_iter().map(|_| BTreeSet::new())); - depths[depth].insert(core); + depths.entry(depth as u8).or_default().insert(core); } } @@ -883,7 +881,7 @@ mod tests { vec![new_ccr.descriptor.para_id(), new_ccr.descriptor.para_id()].into(), ); - assert_eq!(new_ccr.check_core_index(&remap_claim_queue(cq)), Ok(())); + assert_eq!(new_ccr.check_core_index(&transpose_claim_queue(cq)), Ok(())); } #[test] @@ -900,7 +898,7 @@ mod tests { // The check should not fail because no `SelectCore` signal was sent. // The message is optional. - assert!(new_ccr.check_core_index(&remap_claim_queue(cq)).is_ok()); + assert!(new_ccr.check_core_index(&transpose_claim_queue(cq)).is_ok()); // Garbage message. new_ccr.commitments.upward_messages.force_push(vec![0, 13, 200].encode()); @@ -919,7 +917,7 @@ mod tests { ); assert_eq!( - new_ccr.check_core_index(&remap_claim_queue(cq.clone())), + new_ccr.check_core_index(&transpose_claim_queue(cq.clone())), Err(CandidateReceiptError::NoCoreSelected) ); @@ -938,7 +936,7 @@ mod tests { .force_push(UMPSignal::SelectCore(CoreSelector(1), ClaimQueueOffset(1)).encode()); // Duplicate doesn't override first signal. - assert_eq!(new_ccr.check_core_index(&remap_claim_queue(cq)), Ok(())); + assert_eq!(new_ccr.check_core_index(&transpose_claim_queue(cq)), Ok(())); } #[test] @@ -982,7 +980,7 @@ mod tests { vec![new_ccr.descriptor.para_id(), new_ccr.descriptor.para_id()].into(), ); - assert_eq!(new_ccr.check_core_index(&remap_claim_queue(cq)), Ok(())); + assert_eq!(new_ccr.check_core_index(&transpose_claim_queue(cq)), Ok(())); assert_eq!(new_ccr.hash(), v2_ccr.hash()); } @@ -1014,7 +1012,7 @@ mod tests { cq.insert(CoreIndex(0), vec![v1_ccr.descriptor.para_id()].into()); cq.insert(CoreIndex(1), vec![v1_ccr.descriptor.para_id()].into()); - assert!(v1_ccr.check_core_index(&remap_claim_queue(cq)).is_ok()); + assert!(v1_ccr.check_core_index(&transpose_claim_queue(cq)).is_ok()); assert_eq!( v1_ccr.commitments.committed_core_index(&vec![&CoreIndex(10), &CoreIndex(5)]), @@ -1039,7 +1037,7 @@ mod tests { // Since collator sig and id are zeroed, it means that the descriptor uses format // version 2. Should still pass checks without core selector. - assert!(new_ccr.check_core_index(&remap_claim_queue(cq)).is_ok()); + assert!(new_ccr.check_core_index(&transpose_claim_queue(cq)).is_ok()); let mut cq = BTreeMap::new(); cq.insert(CoreIndex(0), vec![new_ccr.descriptor.para_id()].into()); @@ -1047,7 +1045,7 @@ mod tests { // Should fail because 2 cores are assigned, assert_eq!( - new_ccr.check_core_index(&remap_claim_queue(cq)), + new_ccr.check_core_index(&transpose_claim_queue(cq)), Err(CandidateReceiptError::NoCoreSelected) ); diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index b12637da2e42..1654590d109e 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -689,7 +689,7 @@ impl BenchBuilder { para_id, relay_parent, core_idx, - 2, + self.target_session, persisted_validation_data_hash, pov_hash, Default::default(), diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index f571dba314a5..a581d31d8a85 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -1525,15 +1525,8 @@ fn map_candidates_to_cores::set_node_feature( RuntimeOrigin::root(), FeatureIndex::ElasticScalingMVP as u8, - true, + injected_core, ) .unwrap(); @@ -1707,7 +1709,7 @@ mod enter { fill_claimqueue: true, elastic_paras: [(2, 3)].into_iter().collect(), unavailable_cores: unavailable_cores.clone(), - v2_descriptor: false, + v2_descriptor: true, candidate_modifier: None, }); diff --git a/polkadot/runtime/parachains/src/shared.rs b/polkadot/runtime/parachains/src/shared.rs index 4e982313791c..f582bf0d90b5 100644 --- a/polkadot/runtime/parachains/src/shared.rs +++ b/polkadot/runtime/parachains/src/shared.rs @@ -26,7 +26,7 @@ use alloc::{ use frame_support::{pallet_prelude::*, traits::DisabledValidators}; use frame_system::pallet_prelude::BlockNumberFor; use polkadot_primitives::{ - vstaging::remap_claim_queue, CoreIndex, Id, SessionIndex, ValidatorId, ValidatorIndex, + vstaging::transpose_claim_queue, CoreIndex, Id, SessionIndex, ValidatorId, ValidatorIndex, }; use sp_runtime::traits::AtLeast32BitUnsigned; @@ -56,7 +56,7 @@ pub struct RelayParentInfo { pub state_root: Hash, // Claim queue snapshot, optimized for accessing the assignments by `ParaId`. // For each para we store the cores assigned per depth. - pub claim_queue: BTreeMap>>, + pub claim_queue: BTreeMap>>, } /// Keeps tracks of information about all viable relay parents. @@ -88,7 +88,7 @@ impl number: BlockNumber, max_ancestry_len: u32, ) { - let claim_queue = remap_claim_queue(claim_queue); + let claim_queue = transpose_claim_queue(claim_queue); // + 1 for the most recent block, which is always allowed. let buffer_size_limit = max_ancestry_len as usize + 1; diff --git a/polkadot/runtime/parachains/src/shared/migration.rs b/polkadot/runtime/parachains/src/shared/migration.rs index 0126f817e0da..f2af28c189cc 100644 --- a/polkadot/runtime/parachains/src/shared/migration.rs +++ b/polkadot/runtime/parachains/src/shared/migration.rs @@ -24,6 +24,16 @@ pub mod v0 { use super::*; use alloc::collections::vec_deque::VecDeque; + use frame_support::storage_alias; + + /// All allowed relay-parents storage at version 0. + #[storage_alias] + pub(crate) type AllowedRelayParents = StorageValue< + Pallet, + super::v0::AllowedRelayParentsTracker<::Hash, BlockNumberFor>, + ValueQuery, + >; + #[derive(Encode, Decode, Default, TypeInfo)] pub struct AllowedRelayParentsTracker { // The past relay parents, paired with state roots, that are viable to build upon. @@ -71,23 +81,13 @@ mod v1 { traits::{GetStorageVersion, StorageVersion}, }; - use frame_support::storage_alias; - - /// All allowed relay-parents storage at version 0. - #[storage_alias] - pub(crate) type AllowedRelayParents = StorageValue< - Pallet, - super::v0::AllowedRelayParentsTracker<::Hash, BlockNumberFor>, - ValueQuery, - >; - pub struct VersionUncheckedMigrateToV1(core::marker::PhantomData); impl UncheckedOnRuntimeUpgrade for VersionUncheckedMigrateToV1 { #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { - log::trace!(target: LOG_TARGET, "Running pre_upgrade() for inclusion MigrateToV1"); - let bytes = u32::to_ne_bytes(AllowedRelayParents::::get().buffer.len() as u32); + log::trace!(target: LOG_TARGET, "Running pre_upgrade() for shared MigrateToV1"); + let bytes = u32::to_ne_bytes(v0::AllowedRelayParents::::get().buffer.len() as u32); Ok(bytes.to_vec()) } @@ -96,7 +96,7 @@ mod v1 { let mut weight: Weight = Weight::zero(); // Read old storage. - let old_rp_tracker = AllowedRelayParents::::take(); + let old_rp_tracker = v0::AllowedRelayParents::::take(); super::AllowedRelayParents::::set(old_rp_tracker.into()); @@ -107,14 +107,17 @@ mod v1 { #[cfg(feature = "try-runtime")] fn post_upgrade(state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { - log::trace!(target: LOG_TARGET, "Running post_upgrade() for inclusion MigrateToV1"); + log::trace!(target: LOG_TARGET, "Running post_upgrade() for shared MigrateToV1"); ensure!( Pallet::::on_chain_storage_version() >= StorageVersion::new(1), "Storage version should be >= 1 after the migration" ); - let relay_parent_count = - u32::decode(&mut &state[..]).expect("Was properly encoded") as usize; + let relay_parent_count = u32::from_ne_bytes( + state + .try_into() + .expect("u32::from_ne_bytes(to_ne_bytes(u32)) always works; qed"), + ); let rp_tracker = AllowedRelayParents::::get(); @@ -155,12 +158,14 @@ mod tests { .collect::>(), }; - v1::AllowedRelayParents::::put(rp_tracker); + v0::AllowedRelayParents::::put(rp_tracker); as UncheckedOnRuntimeUpgrade>::on_runtime_upgrade(); let rp_tracker = AllowedRelayParents::::get(); + assert_eq!(rp_tracker.buffer.len(), 10); + for idx in 0..10u64 { let relay_parent = Hash::from_low_u64_ne(idx); let state_root = Hash::from_low_u64_ne(2 * idx); diff --git a/polkadot/runtime/parachains/src/shared/tests.rs b/polkadot/runtime/parachains/src/shared/tests.rs index fb04db5d79bb..6da84e254f05 100644 --- a/polkadot/runtime/parachains/src/shared/tests.rs +++ b/polkadot/runtime/parachains/src/shared/tests.rs @@ -65,38 +65,38 @@ fn tracker_claim_queue_remap() { let (info, _block_num) = tracker.acquire_info(Hash::zero(), None).unwrap(); assert_eq!( - info.claim_queue.get(&Id::from(0)).unwrap()[0], + info.claim_queue.get(&Id::from(0)).unwrap()[&0], vec![CoreIndex(0), CoreIndex(1)].into_iter().collect::>() ); assert_eq!( - info.claim_queue.get(&Id::from(1)).unwrap()[0], + info.claim_queue.get(&Id::from(1)).unwrap()[&0], vec![CoreIndex(2)].into_iter().collect::>() ); - assert_eq!(info.claim_queue.get(&Id::from(2)).unwrap()[0], BTreeSet::new()); - assert_eq!(info.claim_queue.get(&Id::from(100)).unwrap()[0], BTreeSet::new()); + assert_eq!(info.claim_queue.get(&Id::from(2)).unwrap().get(&0), None); + assert_eq!(info.claim_queue.get(&Id::from(100)).unwrap().get(&0), None); assert_eq!( - info.claim_queue.get(&Id::from(0)).unwrap()[1], + info.claim_queue.get(&Id::from(0)).unwrap()[&1], vec![CoreIndex(1)].into_iter().collect::>() ); assert_eq!( - info.claim_queue.get(&Id::from(1)).unwrap()[1], + info.claim_queue.get(&Id::from(1)).unwrap()[&1], vec![CoreIndex(0)].into_iter().collect::>() ); assert_eq!( - info.claim_queue.get(&Id::from(2)).unwrap()[1], + info.claim_queue.get(&Id::from(2)).unwrap()[&1], vec![CoreIndex(2)].into_iter().collect::>() ); - assert_eq!(info.claim_queue.get(&Id::from(100)).unwrap()[1], BTreeSet::new()); + assert_eq!(info.claim_queue.get(&Id::from(100)).unwrap().get(&1), None); - assert_eq!(info.claim_queue.get(&Id::from(0)).unwrap().get(2), None); - assert_eq!(info.claim_queue.get(&Id::from(1)).unwrap().get(2), None); + assert_eq!(info.claim_queue.get(&Id::from(0)).unwrap().get(&2), None); + assert_eq!(info.claim_queue.get(&Id::from(1)).unwrap().get(&2), None); assert_eq!( - info.claim_queue.get(&Id::from(2)).unwrap()[2], + info.claim_queue.get(&Id::from(2)).unwrap()[&2], vec![CoreIndex(0)].into_iter().collect::>() ); assert_eq!( - info.claim_queue.get(&Id::from(100)).unwrap()[2], + info.claim_queue.get(&Id::from(100)).unwrap()[&2], vec![CoreIndex(1), CoreIndex(2)].into_iter().collect::>() ); } From 218f53036819b53a9032ca3d79c55945dcdea617 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Fri, 13 Sep 2024 15:52:24 +0300 Subject: [PATCH 094/103] refactor Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/paras_inherent/tests.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index d2cfea19ab6e..1078d6848c0c 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -231,8 +231,7 @@ mod enter { #[case((true, false))] #[case((true, true))] #[case((false, true))] - fn include_backed_candidates_elastic_scaling(#[case] params: (bool, bool)) { - let (v2_descriptor, injected_core) = params; + fn include_backed_candidates_elastic_scaling(#[case] v2_descriptor: bool, injected_core: bool) { // ParaId 0 has one pending candidate on core 0. // ParaId 1 has one pending candidate on core 1. // ParaId 2 has three pending candidates on cores 2, 3 and 4. From 216937ae90c15ffdcaaabbf0619f45bcb5985719 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Sat, 14 Sep 2024 00:03:05 +0300 Subject: [PATCH 095/103] fix try-runtime Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/shared/migration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/runtime/parachains/src/shared/migration.rs b/polkadot/runtime/parachains/src/shared/migration.rs index f2af28c189cc..a8b9612bb1c9 100644 --- a/polkadot/runtime/parachains/src/shared/migration.rs +++ b/polkadot/runtime/parachains/src/shared/migration.rs @@ -122,7 +122,7 @@ mod v1 { let rp_tracker = AllowedRelayParents::::get(); ensure!( - relay_parent_count == rp_tracker.buffer.len(), + relay_parent_count as usize == rp_tracker.buffer.len(), "Number of allowed relay parents should be the same as the one before the upgrade." ); From c0aee8c93581a31b24e2ec9c63b1a76e7a791374 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 16 Sep 2024 15:30:50 +0300 Subject: [PATCH 096/103] check ump signal count and test Signed-off-by: Andrei Sandu --- .../runtime/parachains/src/inclusion/mod.rs | 16 +++- .../parachains/src/paras_inherent/tests.rs | 79 ++++++++++++++++++- 2 files changed, 90 insertions(+), 5 deletions(-) diff --git a/polkadot/runtime/parachains/src/inclusion/mod.rs b/polkadot/runtime/parachains/src/inclusion/mod.rs index 248fda8a3c56..683ebb61e404 100644 --- a/polkadot/runtime/parachains/src/inclusion/mod.rs +++ b/polkadot/runtime/parachains/src/inclusion/mod.rs @@ -440,6 +440,9 @@ pub(crate) enum UmpAcceptanceCheckErr { TotalSizeExceeded { total_size: u64, limit: u64 }, /// A para-chain cannot send UMP messages while it is offboarding. IsOffboarding, + /// The allowed number of `UMPSignal` messages in the queue was exceeded. + /// Currenly only one such message is allowed. + TooManyUMPSignals { count: u32 }, } impl fmt::Debug for UmpAcceptanceCheckErr { @@ -468,6 +471,9 @@ impl fmt::Debug for UmpAcceptanceCheckErr { UmpAcceptanceCheckErr::IsOffboarding => { write!(fmt, "upward message rejected because the para is off-boarding") }, + UmpAcceptanceCheckErr::TooManyUMPSignals { count } => { + write!(fmt, "the umpq queue has too many `UMPSignal` mesages ({} > 1 )", count) + }, } } } @@ -939,7 +945,15 @@ impl Pallet { let upward_messages = if let Some(separator_index) = upward_messages.iter().rposition(|message| message.is_empty()) { - upward_messages.split_at(separator_index).0 + let (upward_messages, ump_signals) = upward_messages.split_at(separator_index); + + if ump_signals.len() > 2 { + return Err(UmpAcceptanceCheckErr::TooManyUMPSignals { + count: ump_signals.len() as u32, + }) + } + + upward_messages } else { upward_messages }; diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 1078d6848c0c..6010350dc332 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -228,10 +228,13 @@ mod enter { } #[rstest] - #[case((true, false))] - #[case((true, true))] - #[case((false, true))] - fn include_backed_candidates_elastic_scaling(#[case] v2_descriptor: bool, injected_core: bool) { + #[case(true, false)] + #[case(true, true)] + #[case(false, true)] + fn include_backed_candidates_elastic_scaling( + #[case] v2_descriptor: bool, + #[case] injected_core: bool, + ) { // ParaId 0 has one pending candidate on core 0. // ParaId 1 has one pending candidate on core 1. // ParaId 2 has three pending candidates on cores 2, 3 and 4. @@ -1671,6 +1674,74 @@ mod enter { }); } + #[test] + fn too_many_ump_signals() { + let config = default_config(); + assert!(config.configuration.config.scheduler_params.lookahead > 0); + new_test_ext(config).execute_with(|| { + // Set the elastic scaling MVP feature. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::CandidateReceiptV2 as u8, + true, + ) + .unwrap(); + + let mut backed_and_concluding = BTreeMap::new(); + backed_and_concluding.insert(0, 1); + backed_and_concluding.insert(1, 1); + backed_and_concluding.insert(2, 1); + + let unavailable_cores = vec![]; + + let scenario = make_inherent_data(TestConfig { + dispute_statements: BTreeMap::new(), + dispute_sessions: vec![], // No disputes + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, + fill_claimqueue: true, + elastic_paras: [(2, 8)].into_iter().collect(), + unavailable_cores: unavailable_cores.clone(), + v2_descriptor: true, + candidate_modifier: Some(|mut candidate: CommittedCandidateReceiptV2| { + if candidate.descriptor.para_id() == 2.into() { + // Add an extra message so `verify_backed_candidates` fails. + candidate.commitments.upward_messages.force_push( + UMPSignal::SelectCore(CoreSelector(123 as u8), ClaimQueueOffset(2)) + .encode(), + ); + } + candidate + }), + }); + + let unfiltered_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 10 backed candidates) + assert_eq!(unfiltered_para_inherent_data.bitfields.len(), 50); + // * 10 v2 candidate descriptors. + assert_eq!(unfiltered_para_inherent_data.backed_candidates.len(), 10); + + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &unfiltered_para_inherent_data) + .unwrap(); + + let dispatch_error = Pallet::::enter( + frame_system::RawOrigin::None.into(), + unfiltered_para_inherent_data, + ) + .unwrap_err() + .error; + + // We expect `enter` to fail because the inherent data contains backed candidates with + // v2 descriptors. + assert_eq!(dispatch_error, Error::::CandidatesFilteredDuringExecution.into()); + }); + } + #[test] fn v2_descriptors_are_accepted() { let config = default_config(); From 1ef79523e24f1772b9ba644cf6fbe307b4e61b98 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 18 Sep 2024 22:14:38 +0300 Subject: [PATCH 097/103] remove unused Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/shared/migration.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/polkadot/runtime/parachains/src/shared/migration.rs b/polkadot/runtime/parachains/src/shared/migration.rs index a8b9612bb1c9..43d6249b6846 100644 --- a/polkadot/runtime/parachains/src/shared/migration.rs +++ b/polkadot/runtime/parachains/src/shared/migration.rs @@ -73,8 +73,6 @@ pub mod v0 { mod v1 { use super::*; - #[cfg(feature = "try-runtime")] - use codec::Decode; #[cfg(feature = "try-runtime")] use frame_support::{ ensure, From 5790b8e784aad2b075a7ec3a8067e8447c47a4a9 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 19 Sep 2024 12:42:16 +0300 Subject: [PATCH 098/103] fix prdoc Signed-off-by: Andrei Sandu --- prdoc/pr_5423.prdoc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/prdoc/pr_5423.prdoc b/prdoc/pr_5423.prdoc index ee75b3d7849a..dbd685d73dc3 100644 --- a/prdoc/pr_5423.prdoc +++ b/prdoc/pr_5423.prdoc @@ -15,4 +15,6 @@ crates: - name: rococo-runtime bump: patch - name: westend-runtime - bump: patch \ No newline at end of file + bump: patch + - name: polkadot + bump: patch From ba9d3ffbfa7a032c3e9ecd734f5a8a78be6bc404 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Thu, 19 Sep 2024 15:45:21 +0300 Subject: [PATCH 099/103] more tests cases Signed-off-by: Andrei Sandu --- .../runtime/parachains/src/inclusion/tests.rs | 50 ++++++- .../parachains/src/paras_inherent/tests.rs | 133 ++++++++++++------ 2 files changed, 135 insertions(+), 48 deletions(-) diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index f12f007e1677..87d21e209a49 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -26,8 +26,13 @@ use crate::{ shared::AllowedRelayParentsTracker, }; use polkadot_primitives::{ - effective_minimum_backing_votes, AvailabilityBitfield, CandidateDescriptor, - SignedAvailabilityBitfields, UncheckedSignedAvailabilityBitfields, + effective_minimum_backing_votes, + vstaging::{ + CandidateDescriptorV2, CandidateDescriptorVersion, ClaimQueueOffset, CoreSelector, + UMPSignal, UMP_SEPARATOR, + }, + AvailabilityBitfield, CandidateDescriptor, SignedAvailabilityBitfields, + UncheckedSignedAvailabilityBitfields, }; use assert_matches::assert_matches; @@ -262,6 +267,10 @@ pub(crate) struct TestCandidateBuilder { pub(crate) new_validation_code: Option, pub(crate) validation_code: ValidationCode, pub(crate) hrmp_watermark: BlockNumber, + /// Creates a v2 descriptor if set. + pub(crate) core_index: Option, + /// The core selector to use. + pub(crate) core_selector: Option, } impl std::default::Default for TestCandidateBuilder { @@ -277,14 +286,28 @@ impl std::default::Default for TestCandidateBuilder { new_validation_code: None, validation_code: dummy_validation_code(), hrmp_watermark: 0u32.into(), + core_index: None, + core_selector: None, } } } impl TestCandidateBuilder { pub(crate) fn build(self) -> CommittedCandidateReceipt { - CommittedCandidateReceipt { - descriptor: CandidateDescriptor { + let descriptor = if let Some(core_index) = self.core_index { + CandidateDescriptorV2::new( + self.para_id, + self.relay_parent, + core_index, + 0, + self.persisted_validation_data_hash, + self.pov_hash, + Default::default(), + self.para_head_hash.unwrap_or_else(|| self.head_data.hash()), + self.validation_code.hash(), + ) + } else { + CandidateDescriptor { para_id: self.para_id, pov_hash: self.pov_hash, relay_parent: self.relay_parent, @@ -301,14 +324,31 @@ impl TestCandidateBuilder { ) .expect("32 bytes; qed"), } - .into(), + .into() + }; + let mut ccr = CommittedCandidateReceipt { + descriptor, commitments: CandidateCommitments { head_data: self.head_data, new_validation_code: self.new_validation_code, hrmp_watermark: self.hrmp_watermark, ..Default::default() }, + }; + + if ccr.descriptor.version() == CandidateDescriptorVersion::V2 { + ccr.commitments.upward_messages.force_push(UMP_SEPARATOR); + + ccr.commitments.upward_messages.force_push( + UMPSignal::SelectCore( + CoreSelector(self.core_selector.unwrap_or_default()), + ClaimQueueOffset(0), + ) + .encode(), + ); } + + ccr } } diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index 6010350dc332..e991890df954 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -2520,19 +2520,12 @@ mod sanitizers { // Para 6 is not scheduled. One candidate supplied. // Para 7 is scheduled on core 7 and 8, but the candidate contains the wrong core index. // Para 8 is scheduled on core 9, but the candidate contains the wrong core index. - fn get_test_data_multiple_cores_per_para(core_index_enabled: bool) -> TestData { + fn get_test_data_multiple_cores_per_para( + core_index_enabled: bool, + v2_descriptor: bool, + ) -> TestData { const RELAY_PARENT_NUM: u32 = 3; - // Add the relay parent to `shared` pallet. Otherwise some code (e.g. filtering backing - // votes) won't behave correctly - shared::Pallet::::add_allowed_relay_parent( - default_header().hash(), - Default::default(), - Default::default(), - RELAY_PARENT_NUM, - 1, - ); - let header = default_header(); let relay_parent = header.hash(); let session_index = SessionIndex::from(0_u32); @@ -2651,6 +2644,21 @@ mod sanitizers { ), ])); + // Add the relay parent to `shared` pallet. Otherwise some code (e.g. filtering backing + // votes) won't behave correctly + shared::Pallet::::add_allowed_relay_parent( + relay_parent, + Default::default(), + scheduler::ClaimQueue::::get() + .into_iter() + .map(|(core_index, paras)| { + (core_index, paras.into_iter().map(|e| e.para_id()).collect()) + }) + .collect(), + RELAY_PARENT_NUM, + 1, + ); + // Set the on-chain included head data and current code hash. for id in 1..=8u32 { paras::Pallet::::set_current_head(ParaId::from(id), HeadData(vec![id as u8])); @@ -2680,6 +2688,14 @@ mod sanitizers { let mut backed_candidates = vec![]; let mut expected_backed_candidates_with_core = BTreeMap::new(); + let maybe_core_index = |core_index: CoreIndex| -> Option { + if !v2_descriptor { + None + } else { + Some(core_index) + } + }; + // Para 1 { let candidate = TestCandidateBuilder { @@ -2696,6 +2712,7 @@ mod sanitizers { hrmp_watermark: RELAY_PARENT_NUM, head_data: HeadData(vec![1, 1]), validation_code: ValidationCode(vec![1]), + core_index: maybe_core_index(CoreIndex(0)), ..Default::default() } .build(); @@ -2711,7 +2728,7 @@ mod sanitizers { core_index_enabled.then_some(CoreIndex(0 as u32)), ); backed_candidates.push(backed.clone()); - if core_index_enabled { + if core_index_enabled || v2_descriptor { expected_backed_candidates_with_core .entry(ParaId::from(1)) .or_insert(vec![]) @@ -2732,6 +2749,8 @@ mod sanitizers { .hash(), hrmp_watermark: RELAY_PARENT_NUM, validation_code: ValidationCode(vec![1]), + core_index: maybe_core_index(CoreIndex(1)), + core_selector: Some(1), ..Default::default() } .build(); @@ -2746,7 +2765,7 @@ mod sanitizers { core_index_enabled.then_some(CoreIndex(1 as u32)), ); backed_candidates.push(backed.clone()); - if core_index_enabled { + if core_index_enabled || v2_descriptor { expected_backed_candidates_with_core .entry(ParaId::from(1)) .or_insert(vec![]) @@ -2769,6 +2788,7 @@ mod sanitizers { .hash(), hrmp_watermark: RELAY_PARENT_NUM, validation_code: ValidationCode(vec![2]), + core_index: maybe_core_index(CoreIndex(2)), ..Default::default() } .build(); @@ -2783,7 +2803,7 @@ mod sanitizers { core_index_enabled.then_some(CoreIndex(2 as u32)), ); backed_candidates.push(backed.clone()); - if core_index_enabled { + if core_index_enabled || v2_descriptor { expected_backed_candidates_with_core .entry(ParaId::from(2)) .or_insert(vec![]) @@ -2806,6 +2826,7 @@ mod sanitizers { .hash(), hrmp_watermark: RELAY_PARENT_NUM, validation_code: ValidationCode(vec![3]), + core_index: maybe_core_index(CoreIndex(4)), ..Default::default() } .build(); @@ -2841,6 +2862,7 @@ mod sanitizers { .hash(), hrmp_watermark: RELAY_PARENT_NUM, validation_code: ValidationCode(vec![4]), + core_index: maybe_core_index(CoreIndex(5)), ..Default::default() } .build(); @@ -2875,6 +2897,7 @@ mod sanitizers { .hash(), hrmp_watermark: RELAY_PARENT_NUM, validation_code: ValidationCode(vec![4]), + core_index: maybe_core_index(CoreIndex(5)), ..Default::default() } .build(); @@ -2908,6 +2931,7 @@ mod sanitizers { .hash(), hrmp_watermark: RELAY_PARENT_NUM, validation_code: ValidationCode(vec![6]), + core_index: maybe_core_index(CoreIndex(6)), ..Default::default() } .build(); @@ -2939,6 +2963,7 @@ mod sanitizers { .hash(), hrmp_watermark: RELAY_PARENT_NUM, validation_code: ValidationCode(vec![7]), + core_index: maybe_core_index(CoreIndex(6)), ..Default::default() } .build(); @@ -2970,6 +2995,7 @@ mod sanitizers { .hash(), hrmp_watermark: RELAY_PARENT_NUM, validation_code: ValidationCode(vec![8]), + core_index: maybe_core_index(CoreIndex(7)), ..Default::default() } .build(); @@ -2984,7 +3010,7 @@ mod sanitizers { core_index_enabled.then_some(CoreIndex(7 as u32)), ); backed_candidates.push(backed.clone()); - if !core_index_enabled { + if !core_index_enabled && !v2_descriptor { expected_backed_candidates_with_core .entry(ParaId::from(8)) .or_insert(vec![]) @@ -3045,14 +3071,6 @@ mod sanitizers { let header = default_header(); let relay_parent = header.hash(); - shared::Pallet::::add_allowed_relay_parent( - relay_parent, - Default::default(), - Default::default(), - RELAY_PARENT_NUM, - 1, - ); - let session_index = SessionIndex::from(0_u32); let keystore = LocalKeystore::in_memory(); @@ -3164,6 +3182,19 @@ mod sanitizers { ), ])); + shared::Pallet::::add_allowed_relay_parent( + relay_parent, + Default::default(), + scheduler::ClaimQueue::::get() + .into_iter() + .map(|(core_index, paras)| { + (core_index, paras.into_iter().map(|e| e.para_id()).collect()) + }) + .collect(), + RELAY_PARENT_NUM, + 1, + ); + // Set the on-chain included head data and current code hash. for id in 1..=4u32 { paras::Pallet::::set_current_head(ParaId::from(id), HeadData(vec![id as u8])); @@ -3958,15 +3989,20 @@ mod sanitizers { } #[rstest] - #[case(false)] - #[case(true)] - fn test_with_multiple_cores_per_para(#[case] core_index_enabled: bool) { + #[case(false, false)] + #[case(true, false)] + #[case(false, true)] + #[case(true, true)] + fn test_with_multiple_cores_per_para( + #[case] core_index_enabled: bool, + #[case] v2_descriptor: bool, + ) { new_test_ext(default_config()).execute_with(|| { let TestData { backed_candidates, expected_backed_candidates_with_core, scheduled_paras: scheduled, - } = get_test_data_multiple_cores_per_para(core_index_enabled); + } = get_test_data_multiple_cores_per_para(core_index_enabled, v2_descriptor); assert_eq!( sanitize_backed_candidates::( @@ -3975,7 +4011,7 @@ mod sanitizers { BTreeSet::new(), scheduled, core_index_enabled, - false, + v2_descriptor, ), expected_backed_candidates_with_core, ); @@ -4162,17 +4198,22 @@ mod sanitizers { // nothing is scheduled, so no paraids match, thus all backed candidates are skipped #[rstest] - #[case(false, false)] - #[case(true, true)] - #[case(false, true)] - #[case(true, false)] + #[case(false, false, true)] + #[case(true, true, true)] + #[case(false, true, true)] + #[case(true, false, true)] + #[case(false, false, false)] + #[case(true, true, false)] + #[case(false, true, false)] + #[case(true, false, false)] fn nothing_scheduled( #[case] core_index_enabled: bool, #[case] multiple_cores_per_para: bool, + #[case] v2_descriptor: bool, ) { new_test_ext(default_config()).execute_with(|| { let TestData { backed_candidates, .. } = if multiple_cores_per_para { - get_test_data_multiple_cores_per_para(core_index_enabled) + get_test_data_multiple_cores_per_para(core_index_enabled, v2_descriptor) } else { get_test_data_one_core_per_para(core_index_enabled) }; @@ -4229,8 +4270,14 @@ mod sanitizers { } // candidates that have concluded as invalid are filtered out, as well as their descendants. - #[test] - fn concluded_invalid_are_filtered_out_multiple_cores_per_para() { + #[rstest] + #[case(false, true)] + #[case(true, false)] + #[case(true, true)] + fn concluded_invalid_are_filtered_out_multiple_cores_per_para( + #[case] core_index_enabled: bool, + #[case] v2_descriptor: bool, + ) { // Mark the first candidate of paraid 1 as invalid. Its descendant should also // be dropped. Also mark the candidate of paraid 3 as invalid. new_test_ext(default_config()).execute_with(|| { @@ -4239,7 +4286,7 @@ mod sanitizers { scheduled_paras: scheduled, mut expected_backed_candidates_with_core, .. - } = get_test_data_multiple_cores_per_para(true); + } = get_test_data_multiple_cores_per_para(core_index_enabled, v2_descriptor); let mut invalid_set = std::collections::BTreeSet::new(); @@ -4258,8 +4305,8 @@ mod sanitizers { &shared::AllowedRelayParents::::get(), invalid_set, scheduled, - true, - false, + core_index_enabled, + v2_descriptor, ); // We'll be left with candidates from paraid 2 and 4. @@ -4278,7 +4325,7 @@ mod sanitizers { scheduled_paras: scheduled, mut expected_backed_candidates_with_core, .. - } = get_test_data_multiple_cores_per_para(true); + } = get_test_data_multiple_cores_per_para(core_index_enabled, v2_descriptor); let mut invalid_set = std::collections::BTreeSet::new(); @@ -4295,8 +4342,8 @@ mod sanitizers { &shared::AllowedRelayParents::::get(), invalid_set, scheduled, - true, - false, + core_index_enabled, + v2_descriptor, ); // Only the second candidate of paraid 1 should be removed. @@ -4507,7 +4554,7 @@ mod sanitizers { // Disable Bob, only the second candidate of paraid 1 should be removed. new_test_ext(default_config()).execute_with(|| { let TestData { mut expected_backed_candidates_with_core, .. } = - get_test_data_multiple_cores_per_para(true); + get_test_data_multiple_cores_per_para(true, false); set_disabled_validators(vec![1]); @@ -4529,7 +4576,7 @@ mod sanitizers { for disabled in [vec![0], vec![0, 1]] { new_test_ext(default_config()).execute_with(|| { let TestData { mut expected_backed_candidates_with_core, .. } = - get_test_data_multiple_cores_per_para(true); + get_test_data_multiple_cores_per_para(true, false); set_disabled_validators(disabled); From 9c4e2aee40eaf8b2d4afb9b780ea166bcb7c7dc7 Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Mon, 23 Sep 2024 13:01:29 +0300 Subject: [PATCH 100/103] stricter UMP signal checks and tests Signed-off-by: Andrei Sandu --- .../runtime/parachains/src/inclusion/mod.rs | 11 ++- .../parachains/src/paras_inherent/tests.rs | 69 ++++++++++++++++++- polkadot/runtime/parachains/src/ump_tests.rs | 2 +- 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/polkadot/runtime/parachains/src/inclusion/mod.rs b/polkadot/runtime/parachains/src/inclusion/mod.rs index 683ebb61e404..79029952e7c1 100644 --- a/polkadot/runtime/parachains/src/inclusion/mod.rs +++ b/polkadot/runtime/parachains/src/inclusion/mod.rs @@ -443,6 +443,8 @@ pub(crate) enum UmpAcceptanceCheckErr { /// The allowed number of `UMPSignal` messages in the queue was exceeded. /// Currenly only one such message is allowed. TooManyUMPSignals { count: u32 }, + /// The UMP queue contains an invalid `UMPSignal` + InvalidUMPSignal, } impl fmt::Debug for UmpAcceptanceCheckErr { @@ -472,7 +474,10 @@ impl fmt::Debug for UmpAcceptanceCheckErr { write!(fmt, "upward message rejected because the para is off-boarding") }, UmpAcceptanceCheckErr::TooManyUMPSignals { count } => { - write!(fmt, "the umpq queue has too many `UMPSignal` mesages ({} > 1 )", count) + write!(fmt, "the ump queue has too many `UMPSignal` mesages ({} > 1 )", count) + }, + UmpAcceptanceCheckErr::InvalidUMPSignal => { + write!(fmt, "the ump queue contains an invalid UMP signal") }, } } @@ -953,6 +958,10 @@ impl Pallet { }) } + if ump_signals.len() == 1 { + return Err(UmpAcceptanceCheckErr::InvalidUMPSignal) + } + upward_messages } else { upward_messages diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index e991890df954..f5c3d5077764 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -1742,6 +1742,73 @@ mod enter { }); } + #[test] + fn invalid_ump_signals() { + let config = default_config(); + assert!(config.configuration.config.scheduler_params.lookahead > 0); + new_test_ext(config).execute_with(|| { + // Set the elastic scaling MVP feature. + configuration::Pallet::::set_node_feature( + RuntimeOrigin::root(), + FeatureIndex::CandidateReceiptV2 as u8, + true, + ) + .unwrap(); + + let mut backed_and_concluding = BTreeMap::new(); + backed_and_concluding.insert(0, 1); + backed_and_concluding.insert(1, 1); + backed_and_concluding.insert(2, 1); + + let unavailable_cores = vec![]; + + let scenario = make_inherent_data(TestConfig { + dispute_statements: BTreeMap::new(), + dispute_sessions: vec![], // No disputes + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, + fill_claimqueue: true, + elastic_paras: [(2, 8)].into_iter().collect(), + unavailable_cores: unavailable_cores.clone(), + v2_descriptor: true, + candidate_modifier: Some(|mut candidate: CommittedCandidateReceiptV2| { + if candidate.descriptor.para_id() == 1.into() { + // Drop the core selector to make it invalid + candidate + .commitments + .upward_messages + .truncate(candidate.commitments.upward_messages.len() - 1); + } + candidate + }), + }); + + let unfiltered_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 10 backed candidates) + assert_eq!(unfiltered_para_inherent_data.bitfields.len(), 50); + // * 10 v2 candidate descriptors. + assert_eq!(unfiltered_para_inherent_data.backed_candidates.len(), 10); + + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &unfiltered_para_inherent_data) + .unwrap(); + + let dispatch_error = Pallet::::enter( + frame_system::RawOrigin::None.into(), + unfiltered_para_inherent_data, + ) + .unwrap_err() + .error; + + // We expect `enter` to fail because the inherent data contains backed candidates with + // v2 descriptors. + assert_eq!(dispatch_error, Error::::CandidatesFilteredDuringExecution.into()); + }); + } #[test] fn v2_descriptors_are_accepted() { let config = default_config(); @@ -1937,7 +2004,7 @@ mod enter { }); } - // A test to ensure that the `paras_inherent`` filters out candidates with invalid + // A test to ensure that the `paras_inherent` filters out candidates with invalid // session index in the descriptor. #[test] fn invalid_session_index() { diff --git a/polkadot/runtime/parachains/src/ump_tests.rs b/polkadot/runtime/parachains/src/ump_tests.rs index 571fc5ab2271..d6cef850fb6a 100644 --- a/polkadot/runtime/parachains/src/ump_tests.rs +++ b/polkadot/runtime/parachains/src/ump_tests.rs @@ -164,7 +164,7 @@ mod check_upward_messages { let max_per_candidate = configuration::ActiveConfig::::get().max_upward_message_num_per_candidate; - for msg_size in 0..=max_size { + for msg_size in 1..=max_size { check(P_0, vec![vec![0; msg_size as usize]], None); } for msg_size in max_size + 1..max_size + 10 { From 43bbb9dd0d0f966ab07adf51d126433f1f74a52b Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Tue, 24 Sep 2024 11:35:57 +0300 Subject: [PATCH 101/103] type alias Signed-off-by: Andrei Sandu --- polkadot/primitives/src/vstaging/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index ebd93ce77091..bc687f7e2fbe 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -460,7 +460,7 @@ impl CommittedCandidateReceiptV2 { /// between `ParaId` and the cores assigned per depth. pub fn check_core_index( &self, - cores_per_para: &BTreeMap>>, + cores_per_para: &TransposedClaimQueue, ) -> Result<(), CandidateReceiptError> { match self.descriptor.version() { // Don't check v1 descriptors. @@ -750,11 +750,14 @@ impl From> for super::v8::CoreState { } } +/// The claim queue mapped by parachain id. +pub type TransposedClaimQueue = BTreeMap>>; + /// Returns a mapping between the para id and the core indices assigned at different /// depths in the claim queue. pub fn transpose_claim_queue( claim_queue: BTreeMap>, -) -> BTreeMap>> { +) -> TransposedClaimQueue { let mut per_para_claim_queue = BTreeMap::new(); for (core, paras) in claim_queue { From 7a1f38252c373893626e2c27096842247abdd7ec Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 2 Oct 2024 13:19:56 +0300 Subject: [PATCH 102/103] missing feedback Signed-off-by: Andrei Sandu --- .../runtime/parachains/src/inclusion/mod.rs | 12 ++++++------ .../parachains/src/runtime_api_impl/v10.rs | 19 ++++++++++++++++++- .../parachains/src/shared/migration.rs | 17 +++++++++++++++++ 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/polkadot/runtime/parachains/src/inclusion/mod.rs b/polkadot/runtime/parachains/src/inclusion/mod.rs index 79029952e7c1..ba0f7392f6b7 100644 --- a/polkadot/runtime/parachains/src/inclusion/mod.rs +++ b/polkadot/runtime/parachains/src/inclusion/mod.rs @@ -444,7 +444,7 @@ pub(crate) enum UmpAcceptanceCheckErr { /// Currenly only one such message is allowed. TooManyUMPSignals { count: u32 }, /// The UMP queue contains an invalid `UMPSignal` - InvalidUMPSignal, + NoUmpSignal, } impl fmt::Debug for UmpAcceptanceCheckErr { @@ -474,10 +474,10 @@ impl fmt::Debug for UmpAcceptanceCheckErr { write!(fmt, "upward message rejected because the para is off-boarding") }, UmpAcceptanceCheckErr::TooManyUMPSignals { count } => { - write!(fmt, "the ump queue has too many `UMPSignal` mesages ({} > 1 )", count) + write!(fmt, "the ump queue has too many `UMPSignal` messages ({} > 1 )", count) }, - UmpAcceptanceCheckErr::InvalidUMPSignal => { - write!(fmt, "the ump queue contains an invalid UMP signal") + UmpAcceptanceCheckErr::NoUmpSignal => { + write!(fmt, "Required UMP signal not found") }, } } @@ -948,7 +948,7 @@ impl Pallet { ) -> Result<(), UmpAcceptanceCheckErr> { // Filter any pending UMP signals and the separator. let upward_messages = if let Some(separator_index) = - upward_messages.iter().rposition(|message| message.is_empty()) + upward_messages.iter().position(|message| message.is_empty()) { let (upward_messages, ump_signals) = upward_messages.split_at(separator_index); @@ -959,7 +959,7 @@ impl Pallet { } if ump_signals.len() == 1 { - return Err(UmpAcceptanceCheckErr::InvalidUMPSignal) + return Err(UmpAcceptanceCheckErr::NoUmpSignal) } upward_messages diff --git a/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs b/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs index ead825b38f07..65b4b0d86594 100644 --- a/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs +++ b/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs @@ -451,7 +451,24 @@ pub fn backing_state( // // Thus, minimum relay parent is ensured to have asynchronous backing enabled. let now = frame_system::Pallet::::block_number(); - let min_relay_parent_number = shared::AllowedRelayParents::::get() + + // Use the right storage depending on version to ensure #64 doesn't cause issues with this + // migration. + let min_relay_parent_number = if shared::Pallet::::on_chain_storage_version() == + StorageVersion::new(0) + { + shared::migration::v0::AllowedRelayParents::::get().hypothetical_earliest_block_number( + now, + config.async_backing_params.allowed_ancestry_len, + ) + } else { + shared::AllowedRelayParents::::get().hypothetical_earliest_block_number( + now, + config.async_backing_params.allowed_ancestry_len, + ) + }; + + shared::AllowedRelayParents::::get() .hypothetical_earliest_block_number(now, config.async_backing_params.allowed_ancestry_len); let required_parent = paras::Heads::::get(para_id)?; diff --git a/polkadot/runtime/parachains/src/shared/migration.rs b/polkadot/runtime/parachains/src/shared/migration.rs index 43d6249b6846..ae0412c6e26c 100644 --- a/polkadot/runtime/parachains/src/shared/migration.rs +++ b/polkadot/runtime/parachains/src/shared/migration.rs @@ -50,6 +50,23 @@ pub mod v0 { pub latest_number: BlockNumber, } + // Required to workaround #64. + impl + AllowedRelayParentsTracker + { + /// Returns block number of the earliest block the buffer would contain if + /// `now` is pushed into it. + pub(crate) fn hypothetical_earliest_block_number( + &self, + now: BlockNumber, + max_ancestry_len: u32, + ) -> BlockNumber { + let allowed_ancestry_len = max_ancestry_len.min(self.buffer.len() as u32); + + now - allowed_ancestry_len.into() + } + } + impl From> for super::AllowedRelayParentsTracker { From 01ce087f1e2e5735b9db5418fb2d54dda20fef3c Mon Sep 17 00:00:00 2001 From: Andrei Sandu Date: Wed, 2 Oct 2024 15:26:36 +0300 Subject: [PATCH 103/103] :facepalm: Signed-off-by: Andrei Sandu --- polkadot/runtime/parachains/src/runtime_api_impl/v10.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs b/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs index 65b4b0d86594..0310b66ea54b 100644 --- a/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs +++ b/polkadot/runtime/parachains/src/runtime_api_impl/v10.rs @@ -468,9 +468,6 @@ pub fn backing_state( ) }; - shared::AllowedRelayParents::::get() - .hypothetical_earliest_block_number(now, config.async_backing_params.allowed_ancestry_len); - let required_parent = paras::Heads::::get(para_id)?; let validation_code_hash = paras::CurrentCodeHash::::get(para_id)?;