From 5018d881237f77a8a39318865db95c1e58164bd9 Mon Sep 17 00:00:00 2001 From: Yiannis Marangos Date: Fri, 25 Oct 2024 13:27:05 +0300 Subject: [PATCH] feat(types)!: Add versioned consts (#412) --- node/src/daser.rs | 7 +- node/src/p2p/shwap.rs | 3 +- node/tests/node.rs | 3 +- rpc/README.md | 5 +- rpc/src/client.rs | 10 +- rpc/src/share.rs | 20 ++- rpc/tests/blob.rs | 31 ++--- rpc/tests/share.rs | 20 +-- rpc/tests/state.rs | 6 +- types/README.md | 4 +- types/src/blob.rs | 159 +++++++++++++---------- types/src/blob/commitment.rs | 13 +- types/src/byzantine.rs | 18 +-- types/src/consts.rs | 111 +++++++++++++++- types/src/data_availability_header.rs | 41 +++--- types/src/eds.rs | 177 ++++++++++++++++---------- types/src/extended_header.rs | 11 +- types/src/fraud_proof.rs | 3 +- types/src/lib.rs | 3 +- types/src/row.rs | 15 ++- types/src/row_namespace_data.rs | 18 ++- types/src/sample.rs | 5 +- types/src/share.rs | 3 +- types/src/test_utils.rs | 16 +-- types/src/validate.rs | 7 + 25 files changed, 470 insertions(+), 239 deletions(-) diff --git a/node/src/daser.rs b/node/src/daser.rs index 42d2d359..96c2595f 100644 --- a/node/src/daser.rs +++ b/node/src/daser.rs @@ -480,6 +480,7 @@ mod tests { use crate::test_utils::{async_test, MockP2pHandle}; use bytes::BytesMut; use celestia_proto::bitswap::Block; + use celestia_types::consts::appconsts::AppVersion; use celestia_types::sample::{Sample, SampleId}; use celestia_types::test_utils::{generate_dummy_eds, ExtendedHeaderGenerator}; use celestia_types::{AxisType, DataAvailabilityHeader, ExtendedDataSquare}; @@ -570,7 +571,7 @@ mod tests { let mut headers = Vec::new(); for _ in 0..20 { - let eds = generate_dummy_eds(2); + let eds = generate_dummy_eds(2, AppVersion::V2); let dah = DataAvailabilityHeader::from_eds(&eds); let header = gen.next_with_dah(dah); @@ -644,7 +645,7 @@ mod tests { handle.expect_no_cmd().await; // Push block 21 in the store - let eds = generate_dummy_eds(2); + let eds = generate_dummy_eds(2, AppVersion::V2); let dah = DataAvailabilityHeader::from_eds(&eds); let header = gen.next_with_dah(dah); store.insert(header).await.unwrap(); @@ -663,7 +664,7 @@ mod tests { square_width: usize, simulate_invalid_sampling: bool, ) { - let eds = generate_dummy_eds(square_width); + let eds = generate_dummy_eds(square_width, AppVersion::V2); let dah = DataAvailabilityHeader::from_eds(&eds); let header = gen.next_with_dah(dah); let height = header.height().value(); diff --git a/node/src/p2p/shwap.rs b/node/src/p2p/shwap.rs index f74b7900..5d69538b 100644 --- a/node/src/p2p/shwap.rs +++ b/node/src/p2p/shwap.rs @@ -112,6 +112,7 @@ mod tests { use crate::store::InMemoryStore; use crate::test_utils::async_test; use bytes::BytesMut; + use celestia_types::consts::appconsts::AppVersion; use celestia_types::test_utils::{generate_dummy_eds, ExtendedHeaderGenerator}; use celestia_types::{AxisType, DataAvailabilityHeader}; @@ -119,7 +120,7 @@ mod tests { async fn hash() { let store = Arc::new(InMemoryStore::new()); - let eds = generate_dummy_eds(4); + let eds = generate_dummy_eds(4, AppVersion::V2); let dah = DataAvailabilityHeader::from_eds(&eds); let mut gen = ExtendedHeaderGenerator::new(); diff --git a/node/tests/node.rs b/node/tests/node.rs index 644fd94f..b8323316 100644 --- a/node/tests/node.rs +++ b/node/tests/node.rs @@ -3,6 +3,7 @@ use std::time::Duration; use celestia_tendermint_proto::Protobuf; +use celestia_types::consts::appconsts::AppVersion; use celestia_types::consts::HASH_SIZE; use celestia_types::fraud_proof::BadEncodingFraudProof; use celestia_types::hash::Hash; @@ -182,7 +183,7 @@ async fn stops_services_when_network_is_compromised() { store.insert(gen.next_many_verified(64)).await.unwrap(); // create a corrupted block and insert it - let mut eds = generate_dummy_eds(8); + let mut eds = generate_dummy_eds(8, AppVersion::V2); let (header, befp) = corrupt_eds(&mut gen, &mut eds); store.insert(header).await.unwrap(); diff --git a/rpc/README.md b/rpc/README.md index 43628f00..579dff43 100644 --- a/rpc/README.md +++ b/rpc/README.md @@ -6,8 +6,7 @@ This crate builds on top of the [`jsonrpsee`](https://docs.rs/jsonrpsee) clients ```rust,no_run use celestia_rpc::{BlobClient, Client}; -use celestia_types::{Blob, nmt::Namespace}; -use celestia_types::TxConfig; +use celestia_types::{AppVersion, Blob, TxConfig, nmt::Namespace}; async fn submit_blob() { // create a client to the celestia node @@ -18,7 +17,7 @@ async fn submit_blob() { // create a blob that you want to submit let my_namespace = Namespace::new_v0(&[1, 2, 3, 4, 5]).expect("Invalid namespace"); - let blob = Blob::new(my_namespace, b"some data to store on blockchain".to_vec()) + let blob = Blob::new(my_namespace, b"some data to store on blockchain".to_vec(), AppVersion::V2) .expect("Failed to create a blob"); // submit it diff --git a/rpc/src/client.rs b/rpc/src/client.rs index 6132270f..5a1eb37b 100644 --- a/rpc/src/client.rs +++ b/rpc/src/client.rs @@ -16,8 +16,7 @@ mod native { use std::result::Result; use async_trait::async_trait; - use celestia_types::consts::appconsts::SHARE_SIZE; - use celestia_types::consts::data_availability_header::MAX_EXTENDED_SQUARE_WIDTH; + use celestia_types::consts::appconsts::{self, SHARE_SIZE}; use http::{header, HeaderValue}; use jsonrpsee::core::client::{BatchResponse, ClientT, Subscription, SubscriptionClientT}; use jsonrpsee::core::params::BatchRequestBuilder; @@ -29,8 +28,11 @@ mod native { use crate::Error; - const MAX_EDS_SIZE_BYTES: usize = - MAX_EXTENDED_SQUARE_WIDTH * MAX_EXTENDED_SQUARE_WIDTH * SHARE_SIZE; + // NOTE: Always the largest `appconsts::*::SQUARE_SIZE_UPPER_BOUND` needs to be used. + const MAX_EDS_SIZE_BYTES: usize = appconsts::v3::SQUARE_SIZE_UPPER_BOUND + * appconsts::v3::SQUARE_SIZE_UPPER_BOUND + * 4 + * SHARE_SIZE; // The biggest response we might get is for requesting an EDS. // Also, we allow 1 MB extra for any metadata they come with it. diff --git a/rpc/src/share.rs b/rpc/src/share.rs index 7965c72b..296abb93 100644 --- a/rpc/src/share.rs +++ b/rpc/src/share.rs @@ -3,6 +3,7 @@ use std::future::Future; use std::marker::{Send, Sync}; +use celestia_types::consts::appconsts::AppVersion; use celestia_types::nmt::Namespace; use celestia_types::row_namespace_data::NamespacedShares; use celestia_types::{ExtendedDataSquare, ExtendedHeader, RawShare, Share, ShareProof}; @@ -23,11 +24,15 @@ pub struct GetRangeResponse { mod rpc { use super::*; + use celestia_types::eds::RawExtendedDataSquare; #[rpc(client)] pub trait Share { #[method(name = "share.GetEDS")] - async fn share_get_eds(&self, root: &ExtendedHeader) -> Result; + async fn share_get_eds( + &self, + root: &ExtendedHeader, + ) -> Result; #[method(name = "share.GetRange")] async fn share_get_range( @@ -69,7 +74,18 @@ pub trait ShareClient: ClientT { 'b: 'fut, Self: Sized + Sync + 'fut, { - rpc::ShareClient::share_get_eds(self, root) + async move { + let app_version = root.header.version.app; + let app_version = AppVersion::from_u64(app_version).ok_or_else(|| { + let e = format!("Invalid or unsupported AppVersion: {app_version}"); + Error::Custom(e) + })?; + + let raw_eds = rpc::ShareClient::share_get_eds(self, root).await?; + + ExtendedDataSquare::from_raw(raw_eds, app_version) + .map_err(|e| Error::Custom(e.to_string())) + } } /// GetRange gets a list of shares and their corresponding proof. diff --git a/rpc/tests/blob.rs b/rpc/tests/blob.rs index a7d826ec..57577270 100644 --- a/rpc/tests/blob.rs +++ b/rpc/tests/blob.rs @@ -5,6 +5,7 @@ use std::time::Duration; use celestia_rpc::blob::BlobsAtHeight; use celestia_rpc::prelude::*; +use celestia_types::consts::appconsts::AppVersion; use celestia_types::{Blob, Commitment}; use jsonrpsee::core::client::Subscription; @@ -18,7 +19,7 @@ async fn blob_submit_and_get() { let client = new_test_client(AuthLevel::Write).await.unwrap(); let namespace = random_ns(); let data = random_bytes(5); - let blob = Blob::new(namespace, data).unwrap(); + let blob = Blob::new(namespace, data, AppVersion::V2).unwrap(); let submitted_height = blob_submit(&client, &[blob.clone()]).await.unwrap(); @@ -34,7 +35,7 @@ async fn blob_submit_and_get() { .await .unwrap(); - received_blob.validate().unwrap(); + received_blob.validate(AppVersion::V2).unwrap(); assert_blob_equal_to_sent(&received_blob, &blob); let proofs = client @@ -57,8 +58,8 @@ async fn blob_submit_and_get_all() { let namespaces = &[random_ns(), random_ns()]; let blobs = &[ - Blob::new(namespaces[0], random_bytes(5)).unwrap(), - Blob::new(namespaces[1], random_bytes(15)).unwrap(), + Blob::new(namespaces[0], random_bytes(5), AppVersion::V2).unwrap(), + Blob::new(namespaces[1], random_bytes(15), AppVersion::V2).unwrap(), ]; let submitted_height = blob_submit(&client, &blobs[..]).await.unwrap(); @@ -74,7 +75,7 @@ async fn blob_submit_and_get_all() { for (idx, (blob, received_blob)) in blobs.iter().zip(received_blobs.iter()).enumerate() { let namespace = namespaces[idx]; - received_blob.validate().unwrap(); + received_blob.validate(AppVersion::V2).unwrap(); assert_blob_equal_to_sent(received_blob, blob); let proofs = client @@ -91,7 +92,7 @@ async fn blob_submit_and_get_large() { let client = new_test_client(AuthLevel::Write).await.unwrap(); let namespace = random_ns(); let data = random_bytes(1024 * 1024); - let blob = Blob::new(namespace, data).unwrap(); + let blob = Blob::new(namespace, data, AppVersion::V2).unwrap(); let submitted_height = blob_submit(&client, &[blob.clone()]).await.unwrap(); @@ -104,7 +105,7 @@ async fn blob_submit_and_get_large() { .await .unwrap(); - blob.validate().unwrap(); + blob.validate(AppVersion::V2).unwrap(); assert_blob_equal_to_sent(&received_blob, &blob); let proofs = client @@ -129,7 +130,7 @@ async fn blob_subscribe() { assert!(received_blobs.blobs.is_none()); // submit and receive blob - let blob = Blob::new(namespace, random_bytes(10)).unwrap(); + let blob = Blob::new(namespace, random_bytes(10), AppVersion::V2).unwrap(); let current_height = blob_submit(&client, &[blob.clone()]).await.unwrap(); let received = blobs_at_height(current_height, &mut incoming_blobs).await; @@ -137,16 +138,16 @@ async fn blob_subscribe() { assert_blob_equal_to_sent(&received[0], &blob); // submit blob to another ns - let blob_another_ns = Blob::new(random_ns(), random_bytes(10)).unwrap(); + let blob_another_ns = Blob::new(random_ns(), random_bytes(10), AppVersion::V2).unwrap(); let current_height = blob_submit(&client, &[blob_another_ns]).await.unwrap(); let received = blobs_at_height(current_height, &mut incoming_blobs).await; assert!(received.is_empty()); // submit and receive few blobs - let blob1 = Blob::new(namespace, random_bytes(10)).unwrap(); - let blob2 = Blob::new(random_ns(), random_bytes(10)).unwrap(); // different ns - let blob3 = Blob::new(namespace, random_bytes(10)).unwrap(); + let blob1 = Blob::new(namespace, random_bytes(10), AppVersion::V2).unwrap(); + let blob2 = Blob::new(random_ns(), random_bytes(10), AppVersion::V2).unwrap(); // different ns + let blob3 = Blob::new(namespace, random_bytes(10), AppVersion::V2).unwrap(); let current_height = blob_submit(&client, &[blob1.clone(), blob2, blob3.clone()]) .await .unwrap(); @@ -162,7 +163,7 @@ async fn blob_submit_too_large() { let client = new_test_client(AuthLevel::Write).await.unwrap(); let namespace = random_ns(); let data = random_bytes(5 * 1024 * 1024); - let blob = Blob::new(namespace, data).unwrap(); + let blob = Blob::new(namespace, data, AppVersion::V2).unwrap(); blob_submit(&client, &[blob]).await.unwrap_err(); } @@ -172,7 +173,7 @@ async fn blob_get_get_proof_wrong_ns() { let client = new_test_client(AuthLevel::Write).await.unwrap(); let namespace = random_ns(); let data = random_bytes(5); - let blob = Blob::new(namespace, data).unwrap(); + let blob = Blob::new(namespace, data, AppVersion::V2).unwrap(); let submitted_height = blob_submit(&client, &[blob.clone()]).await.unwrap(); @@ -192,7 +193,7 @@ async fn blob_get_get_proof_wrong_commitment() { let client = new_test_client(AuthLevel::Write).await.unwrap(); let namespace = random_ns(); let data = random_bytes(5); - let blob = Blob::new(namespace, data).unwrap(); + let blob = Blob::new(namespace, data, AppVersion::V2).unwrap(); let commitment = Commitment(random_bytes_array()); let submitted_height = blob_submit(&client, &[blob.clone()]).await.unwrap(); diff --git a/rpc/tests/share.rs b/rpc/tests/share.rs index c110d074..57aa9dfa 100644 --- a/rpc/tests/share.rs +++ b/rpc/tests/share.rs @@ -1,6 +1,7 @@ #![cfg(not(target_arch = "wasm32"))] use celestia_rpc::prelude::*; +use celestia_types::consts::appconsts::AppVersion; use celestia_types::nmt::{Namespace, NamespacedSha2Hasher}; use celestia_types::{Blob, Share}; @@ -32,7 +33,7 @@ async fn get_shares_by_namespace() { let blobs: Vec<_> = (0..4) .map(|_| { let data = random_bytes(1024); - Blob::new(namespace, data.clone()).unwrap() + Blob::new(namespace, data.clone(), AppVersion::V2).unwrap() }) .collect(); @@ -45,8 +46,11 @@ async fn get_shares_by_namespace() { .await .unwrap(); - let reconstructed = - Blob::reconstruct_all(ns_shares.rows.iter().flat_map(|row| row.shares.iter())).unwrap(); + let reconstructed = Blob::reconstruct_all( + ns_shares.rows.iter().flat_map(|row| row.shares.iter()), + AppVersion::V2, + ) + .unwrap(); assert_eq!(reconstructed, blobs); } @@ -70,7 +74,7 @@ async fn get_shares_range() { let client = new_test_client(AuthLevel::Write).await.unwrap(); let namespace = random_ns(); let data = random_bytes(1024); - let blob = Blob::new(namespace, data.clone()).unwrap(); + let blob = Blob::new(namespace, data.clone(), AppVersion::V2).unwrap(); let commitment = blob.commitment; let submitted_height = blob_submit(&client, &[blob]).await.unwrap(); @@ -125,7 +129,7 @@ async fn get_shares_by_namespace_wrong_ns() { let client = new_test_client(AuthLevel::Write).await.unwrap(); let namespace = random_ns(); let data = random_bytes(1024); - let blob = Blob::new(namespace, data.clone()).unwrap(); + let blob = Blob::new(namespace, data.clone(), AppVersion::V2).unwrap(); let submitted_height = blob_submit(&client, &[blob]).await.unwrap(); @@ -166,7 +170,7 @@ async fn get_shares_by_namespace_wrong_ns_out_of_range() { let client = new_test_client(AuthLevel::Write).await.unwrap(); let namespace = random_ns(); let data = random_bytes(1024); - let blob = Blob::new(namespace, data.clone()).unwrap(); + let blob = Blob::new(namespace, data.clone(), AppVersion::V2).unwrap(); let submitted_height = blob_submit(&client, &[blob]).await.unwrap(); @@ -192,7 +196,7 @@ async fn get_shares_by_namespace_wrong_roots() { let client = new_test_client(AuthLevel::Write).await.unwrap(); let namespace = random_ns(); let data = random_bytes(1024); - let blob = Blob::new(namespace, data.clone()).unwrap(); + let blob = Blob::new(namespace, data.clone(), AppVersion::V2).unwrap(); blob_submit(&client, &[blob]).await.unwrap(); @@ -211,7 +215,7 @@ async fn get_eds() { let client = new_test_client(AuthLevel::Write).await.unwrap(); let namespace = random_ns(); let data = vec![1, 2, 3, 4]; - let blob = Blob::new(namespace, data.clone()).unwrap(); + let blob = Blob::new(namespace, data.clone(), AppVersion::V2).unwrap(); let submitted_height = blob_submit(&client, &[blob]).await.unwrap(); diff --git a/rpc/tests/state.rs b/rpc/tests/state.rs index cee3bada..9e535914 100644 --- a/rpc/tests/state.rs +++ b/rpc/tests/state.rs @@ -2,7 +2,7 @@ use crate::utils::{random_bytes, random_ns}; use celestia_rpc::prelude::*; -use celestia_types::{Blob, TxConfig}; +use celestia_types::{AppVersion, Blob, TxConfig}; pub mod utils; @@ -37,7 +37,7 @@ async fn submit_pay_for_blob() { let client = new_test_client(AuthLevel::Write).await.unwrap(); let namespace = random_ns(); let data = random_bytes(5); - let blob = Blob::new(namespace, data).unwrap(); + let blob = Blob::new(namespace, data, AppVersion::V2).unwrap(); let tx_response = client .state_submit_pay_for_blob(&[blob.clone().into()], TxConfig::default()) @@ -49,6 +49,6 @@ async fn submit_pay_for_blob() { .await .unwrap(); - received_blob.validate().unwrap(); + received_blob.validate(AppVersion::V2).unwrap(); assert_eq!(received_blob.data, blob.data); } diff --git a/types/README.md b/types/README.md index 055f31b2..937e9c36 100644 --- a/types/README.md +++ b/types/README.md @@ -7,10 +7,10 @@ and [`celestia-proto`](https://github.com/eigerco/lumina/proto) and support the protobuf and serde to the format understood by nodes in celestia network. ```rust -use celestia_types::{Blob, nmt::Namespace}; +use celestia_types::{AppVersion, Blob, nmt::Namespace}; let my_namespace = Namespace::new_v0(&[1, 2, 3, 4, 5]).expect("Invalid namespace"); -let blob = Blob::new(my_namespace, b"some data to store on blockchain".to_vec()) +let blob = Blob::new(my_namespace, b"some data to store on blockchain".to_vec(), AppVersion::V2) .expect("Failed to create a blob"); assert_eq!( diff --git a/types/src/blob.rs b/types/src/blob.rs index 478ed8b6..a5954127 100644 --- a/types/src/blob.rs +++ b/types/src/blob.rs @@ -2,12 +2,12 @@ use std::iter; -use celestia_tendermint_proto::Protobuf; use serde::{Deserialize, Serialize}; mod commitment; use crate::consts::appconsts; +use crate::consts::appconsts::{subtree_root_threshold, AppVersion}; use crate::nmt::Namespace; use crate::{bail_validation, Error, Result, Share}; @@ -47,10 +47,10 @@ impl Blob { /// # Example /// /// ``` - /// use celestia_types::{Blob, nmt::Namespace}; + /// use celestia_types::{AppVersion, Blob, nmt::Namespace}; /// /// let my_namespace = Namespace::new_v0(&[1, 2, 3, 4, 5]).expect("Invalid namespace"); - /// let blob = Blob::new(my_namespace, b"some data to store on blockchain".to_vec()) + /// let blob = Blob::new(my_namespace, b"some data to store on blockchain".to_vec(), AppVersion::V2) /// .expect("Failed to create a blob"); /// /// assert_eq!( @@ -64,14 +64,35 @@ impl Blob { /// }"#}, /// ); /// ``` - pub fn new(namespace: Namespace, data: Vec) -> Result { - let commitment = - Commitment::from_blob(namespace, appconsts::SHARE_VERSION_ZERO, &data[..])?; + pub fn new(namespace: Namespace, data: Vec, app_version: AppVersion) -> Result { + let subtree_root_threshold = subtree_root_threshold(app_version); + let commitment = Commitment::from_blob(namespace, &data[..], 0, subtree_root_threshold)?; Ok(Blob { namespace, data, - share_version: appconsts::SHARE_VERSION_ZERO, + share_version: 0, + commitment, + index: None, + }) + } + + /// Creates a `Blob` from [`RawBlob`] and an [`AppVersion`]. + pub fn from_raw(raw: RawBlob, app_version: AppVersion) -> Result { + let subtree_root_threshold = subtree_root_threshold(app_version); + + let namespace = Namespace::new(raw.namespace_version as u8, &raw.namespace_id)?; + let commitment = Commitment::from_blob( + namespace, + &raw.data[..], + raw.share_version as u8, + subtree_root_threshold, + )?; + + Ok(Blob { + namespace, + data: raw.data, + share_version: raw.share_version as u8, commitment, index: None, }) @@ -87,22 +108,29 @@ impl Blob { /// /// ``` /// use celestia_types::Blob; + /// # use celestia_types::consts::appconsts::AppVersion; /// # use celestia_types::nmt::Namespace; /// # /// # let namespace = Namespace::new_v0(&[1, 2, 3, 4, 5]).expect("Invalid namespace"); /// - /// let mut blob = Blob::new(namespace, b"foo".to_vec()).unwrap(); + /// let mut blob = Blob::new(namespace, b"foo".to_vec(), AppVersion::V2).unwrap(); /// - /// assert!(blob.validate().is_ok()); + /// assert!(blob.validate(AppVersion::V2).is_ok()); /// - /// let other_blob = Blob::new(namespace, b"bar".to_vec()).unwrap(); + /// let other_blob = Blob::new(namespace, b"bar".to_vec(), AppVersion::V2).unwrap(); /// blob.commitment = other_blob.commitment; /// - /// assert!(blob.validate().is_err()); + /// assert!(blob.validate(AppVersion::V2).is_err()); /// ``` - pub fn validate(&self) -> Result<()> { - let computed_commitment = - Commitment::from_blob(self.namespace, self.share_version, &self.data)?; + pub fn validate(&self, app_version: AppVersion) -> Result<()> { + let subtree_root_threshold = subtree_root_threshold(app_version); + + let computed_commitment = Commitment::from_blob( + self.namespace, + &self.data, + self.share_version, + subtree_root_threshold, + )?; if self.commitment != computed_commitment { bail_validation!("blob commitment != localy computed commitment") @@ -124,10 +152,11 @@ impl Blob { /// /// ``` /// use celestia_types::Blob; + /// # use celestia_types::consts::appconsts::AppVersion; /// # use celestia_types::nmt::Namespace; /// # let namespace = Namespace::new_v0(&[1, 2, 3, 4, 5]).expect("Invalid namespace"); /// - /// let blob = Blob::new(namespace, b"foo".to_vec()).unwrap(); + /// let blob = Blob::new(namespace, b"foo".to_vec(), AppVersion::V2).unwrap(); /// let shares = blob.to_shares().unwrap(); /// /// assert_eq!(shares.len(), 1); @@ -152,18 +181,18 @@ impl Blob { /// # Example /// /// ``` - /// use celestia_types::Blob; + /// use celestia_types::{AppVersion, Blob}; /// # use celestia_types::nmt::Namespace; /// # let namespace = Namespace::new_v0(&[1, 2, 3, 4, 5]).expect("Invalid namespace"); /// - /// let blob = Blob::new(namespace, b"foo".to_vec()).unwrap(); + /// let blob = Blob::new(namespace, b"foo".to_vec(), AppVersion::V2).unwrap(); /// let shares = blob.to_shares().unwrap(); /// - /// let reconstructed = Blob::reconstruct(&shares).unwrap(); + /// let reconstructed = Blob::reconstruct(&shares, AppVersion::V2).unwrap(); /// /// assert_eq!(blob, reconstructed); /// ``` - pub fn reconstruct<'a, I>(shares: I) -> Result + pub fn reconstruct<'a, I>(shares: I, app_version: AppVersion) -> Result where I: IntoIterator, { @@ -208,7 +237,7 @@ impl Blob { // remove padding data.truncate(blob_len as usize); - Self::new(namespace, data) + Self::new(namespace, data, app_version) } /// Reconstructs all the blobs from shares. @@ -226,24 +255,24 @@ impl Blob { /// # Example /// /// ``` - /// use celestia_types::Blob; + /// use celestia_types::{AppVersion, Blob}; /// # use celestia_types::nmt::Namespace; /// # let namespace1 = Namespace::new_v0(&[1, 2, 3, 4, 5]).expect("Invalid namespace"); /// # let namespace2 = Namespace::new_v0(&[2, 3, 4, 5, 6]).expect("Invalid namespace"); /// /// let blobs = vec![ - /// Blob::new(namespace1, b"foo".to_vec()).unwrap(), - /// Blob::new(namespace2, b"bar".to_vec()).unwrap(), + /// Blob::new(namespace1, b"foo".to_vec(), AppVersion::V2).unwrap(), + /// Blob::new(namespace2, b"bar".to_vec(), AppVersion::V2).unwrap(), /// ]; /// let shares: Vec<_> = blobs.iter().flat_map(|blob| blob.to_shares().unwrap()).collect(); /// - /// let reconstructed = Blob::reconstruct_all(&shares).unwrap(); + /// let reconstructed = Blob::reconstruct_all(&shares, AppVersion::V2).unwrap(); /// /// assert_eq!(blobs, reconstructed); /// ``` /// /// [`ExtendedDataSquare`]: crate::ExtendedDataSquare - pub fn reconstruct_all<'a, I>(shares: I) -> Result> + pub fn reconstruct_all<'a, I>(shares: I, app_version: AppVersion) -> Result> where I: IntoIterator, { @@ -260,33 +289,13 @@ impl Blob { }; iter::once(start).chain(&mut shares) }; - blobs.push(Blob::reconstruct(&mut blob)?); + blobs.push(Blob::reconstruct(&mut blob, app_version)?); } Ok(blobs) } } -impl Protobuf for Blob {} - -impl TryFrom for Blob { - type Error = Error; - - fn try_from(value: RawBlob) -> Result { - let namespace = Namespace::new(value.namespace_version as u8, &value.namespace_id)?; - let commitment = - Commitment::from_blob(namespace, value.share_version as u8, &value.data[..])?; - - Ok(Blob { - commitment, - namespace, - data: value.data, - share_version: value.share_version as u8, - index: None, - }) - } -} - impl From for RawBlob { fn from(value: Blob) -> RawBlob { RawBlob { @@ -359,14 +368,14 @@ mod tests { fn create_from_raw() { let expected = sample_blob(); let raw = RawBlob::from(expected.clone()); - let created = Blob::try_from(raw).unwrap(); + let created = Blob::from_raw(raw, AppVersion::V2).unwrap(); assert_eq!(created, expected); } #[test] fn validate_blob() { - sample_blob().validate().unwrap(); + sample_blob().validate(AppVersion::V2).unwrap(); } #[test] @@ -374,7 +383,7 @@ mod tests { let mut blob = sample_blob(); blob.commitment.0.fill(7); - blob.validate().unwrap_err(); + blob.validate(AppVersion::V2).unwrap_err(); } #[test] @@ -396,17 +405,17 @@ mod tests { let len = rand::random::() % 1024 * 1024; let data = random_bytes(len); let ns = Namespace::const_v0(rand::random()); - let blob = Blob::new(ns, data).unwrap(); + let blob = Blob::new(ns, data, AppVersion::V2).unwrap(); let shares = blob.to_shares().unwrap(); - assert_eq!(blob, Blob::reconstruct(&shares).unwrap()); + assert_eq!(blob, Blob::reconstruct(&shares, AppVersion::V2).unwrap()); } } #[test] fn reconstruct_empty() { assert!(matches!( - Blob::reconstruct(&Vec::::new()), + Blob::reconstruct(&Vec::::new(), AppVersion::V2), Err(Error::MissingShares) )); } @@ -416,13 +425,16 @@ mod tests { let len = rand::random::() % 1024 * 1024; let data = random_bytes(len); let ns = Namespace::const_v0(rand::random()); - let mut shares = Blob::new(ns, data).unwrap().to_shares().unwrap(); + let mut shares = Blob::new(ns, data, AppVersion::V2) + .unwrap() + .to_shares() + .unwrap(); // modify info byte to remove sequence start bit shares[0].as_mut()[NS_SIZE] &= 0b11111110; assert!(matches!( - Blob::reconstruct(&shares), + Blob::reconstruct(&shares, AppVersion::V2), Err(Error::ExpectedShareWithSequenceStart) )); } @@ -439,10 +451,13 @@ mod tests { }) { let len = (rand::random::() % 1023 + 1) * 2; let data = random_bytes(len); - let shares = Blob::new(ns.unwrap(), data).unwrap().to_shares().unwrap(); + let shares = Blob::new(ns.unwrap(), data, AppVersion::V2) + .unwrap() + .to_shares() + .unwrap(); assert!(matches!( - Blob::reconstruct(&shares), + Blob::reconstruct(&shares, AppVersion::V2), Err(Error::UnexpectedReservedNamespace) )); } @@ -453,11 +468,14 @@ mod tests { let len = rand::random::() % 1024 * 1024 + 2048; let data = random_bytes(len); let ns = Namespace::const_v0(rand::random()); - let shares = Blob::new(ns, data).unwrap().to_shares().unwrap(); + let shares = Blob::new(ns, data, AppVersion::V2) + .unwrap() + .to_shares() + .unwrap(); assert!(matches!( // minimum for len is 4 so 3 will break stuff - Blob::reconstruct(&shares[..2]), + Blob::reconstruct(&shares[..2], AppVersion::V2), Err(Error::MissingShares) )); } @@ -467,13 +485,16 @@ mod tests { let len = rand::random::() % 1024 * 1024 + 512; let data = random_bytes(len); let ns = Namespace::const_v0(rand::random()); - let mut shares = Blob::new(ns, data).unwrap().to_shares().unwrap(); + let mut shares = Blob::new(ns, data, AppVersion::V2) + .unwrap() + .to_shares() + .unwrap(); // change share version in second share shares[1].as_mut()[NS_SIZE] = 0b11111110; assert!(matches!( - Blob::reconstruct(&shares), + Blob::reconstruct(&shares, AppVersion::V2), Err(Error::BlobSharesMetadataMismatch(..)) )); } @@ -484,13 +505,16 @@ mod tests { let data = random_bytes(len); let ns = Namespace::const_v0(rand::random()); let ns2 = Namespace::const_v0(rand::random()); - let mut shares = Blob::new(ns, data).unwrap().to_shares().unwrap(); + let mut shares = Blob::new(ns, data, AppVersion::V2) + .unwrap() + .to_shares() + .unwrap(); // change namespace in second share shares[1].as_mut()[..NS_SIZE].copy_from_slice(ns2.as_bytes()); assert!(matches!( - Blob::reconstruct(&shares), + Blob::reconstruct(&shares, AppVersion::V2), Err(Error::BlobSharesMetadataMismatch(..)) )); } @@ -500,13 +524,16 @@ mod tests { let len = rand::random::() % 1024 * 1024 + 512; let data = random_bytes(len); let ns = Namespace::const_v0(rand::random()); - let mut shares = Blob::new(ns, data).unwrap().to_shares().unwrap(); + let mut shares = Blob::new(ns, data, AppVersion::V2) + .unwrap() + .to_shares() + .unwrap(); // modify info byte to add sequence start bit shares[1].as_mut()[NS_SIZE] |= 0b00000001; assert!(matches!( - Blob::reconstruct(&shares), + Blob::reconstruct(&shares, AppVersion::V2), Err(Error::UnexpectedSequenceStart) )); } @@ -518,7 +545,7 @@ mod tests { let len = rand::random::() % 1024 * 1024 + 512; let data = random_bytes(len); let ns = Namespace::const_v0(rand::random()); - Blob::new(ns, data).unwrap() + Blob::new(ns, data, AppVersion::V2).unwrap() }) .collect(); @@ -526,7 +553,7 @@ mod tests { .iter() .flat_map(|blob| blob.to_shares().unwrap()) .collect(); - let reconstructed = Blob::reconstruct_all(&shares).unwrap(); + let reconstructed = Blob::reconstruct_all(&shares, AppVersion::V2).unwrap(); assert_eq!(blobs, reconstructed); } diff --git a/types/src/blob/commitment.rs b/types/src/blob/commitment.rs index 545443c9..a8242f51 100644 --- a/types/src/blob/commitment.rs +++ b/types/src/blob/commitment.rs @@ -57,20 +57,25 @@ impl Commitment { /// Generate the share commitment from the given blob data. pub fn from_blob( namespace: Namespace, - share_version: u8, blob_data: &[u8], + share_version: u8, + subtree_root_threshold: u64, ) -> Result { let shares = split_blob_to_shares(namespace, share_version, blob_data)?; - Self::from_shares(namespace, &shares) + Self::from_shares(namespace, &shares, subtree_root_threshold) } /// Generate the commitment from the given shares. - pub fn from_shares(namespace: Namespace, mut shares: &[Share]) -> Result { + pub fn from_shares( + namespace: Namespace, + mut shares: &[Share], + subtree_root_threshold: u64, + ) -> Result { // the commitment is the root of a merkle mountain range with max tree size // determined by the number of roots required to create a share commitment // over that blob. The size of the tree is only increased if the number of // subtree roots surpasses a constant threshold. - let subtree_width = subtree_width(shares.len() as u64, appconsts::SUBTREE_ROOT_THRESHOLD); + let subtree_width = subtree_width(shares.len() as u64, subtree_root_threshold); let tree_sizes = merkle_mountain_range_sizes(shares.len() as u64, subtree_width); let mut leaf_sets: Vec<&[_]> = Vec::with_capacity(tree_sizes.len()); diff --git a/types/src/byzantine.rs b/types/src/byzantine.rs index b6603496..2616b861 100644 --- a/types/src/byzantine.rs +++ b/types/src/byzantine.rs @@ -417,6 +417,7 @@ pub(crate) mod test_utils { mod tests { use self::test_utils::befp_from_header_and_eds; use super::*; + use crate::consts::appconsts::AppVersion; use crate::test_utils::{corrupt_eds, generate_dummy_eds, ExtendedHeaderGenerator}; use crate::DataAvailabilityHeader; @@ -426,7 +427,7 @@ mod tests { #[test] fn validate_honest_befp_with_incorrectly_encoded_full_row() { let mut gen = ExtendedHeaderGenerator::new(); - let mut eds = generate_dummy_eds(8); + let mut eds = generate_dummy_eds(8, AppVersion::V2); let (eh, proof) = corrupt_eds(&mut gen, &mut eds); proof.validate(&eh).unwrap(); @@ -435,7 +436,7 @@ mod tests { #[test] fn validate_honest_befp_with_shares_to_rebuild() { let mut gen = ExtendedHeaderGenerator::new(); - let mut eds = generate_dummy_eds(8); + let mut eds = generate_dummy_eds(8, AppVersion::V2); let (eh, mut proof) = corrupt_eds(&mut gen, &mut eds); // remove some shares from the proof so they need to be reconstructed @@ -449,7 +450,7 @@ mod tests { #[test] fn validate_fake_befp() { let mut gen = ExtendedHeaderGenerator::new(); - let mut eds = generate_dummy_eds(8); + let mut eds = generate_dummy_eds(8, AppVersion::V2); let real_dah = DataAvailabilityHeader::from_eds(&eds); let prev_eh = gen.next(); @@ -463,7 +464,7 @@ mod tests { #[test] fn validate_befp_over_correct_data() { let mut gen = ExtendedHeaderGenerator::new(); - let eds = generate_dummy_eds(8); + let eds = generate_dummy_eds(8, AppVersion::V2); let dah = DataAvailabilityHeader::from_eds(&eds); let eh = gen.next_with_dah(dah); @@ -477,7 +478,7 @@ mod tests { #[test] fn validate_befp_wrong_height() { let mut gen = ExtendedHeaderGenerator::new(); - let mut eds = generate_dummy_eds(8); + let mut eds = generate_dummy_eds(8, AppVersion::V2); let (mut eh, proof) = corrupt_eds(&mut gen, &mut eds); eh.header.height = 999u32.into(); @@ -488,7 +489,7 @@ mod tests { #[test] fn validate_befp_wrong_roots_square() { let mut gen = ExtendedHeaderGenerator::new(); - let mut eds = generate_dummy_eds(8); + let mut eds = generate_dummy_eds(8, AppVersion::V2); let (mut eh, proof) = corrupt_eds(&mut gen, &mut eds); eh.dah = DataAvailabilityHeader::new_unchecked(Vec::new(), eh.dah.column_roots().to_vec()); @@ -499,8 +500,7 @@ mod tests { #[test] fn validate_befp_wrong_index() { let mut gen = ExtendedHeaderGenerator::new(); - let mut eds = generate_dummy_eds(8); - + let mut eds = generate_dummy_eds(8, AppVersion::V2); let (eh, mut proof) = corrupt_eds(&mut gen, &mut eds); proof.index = 999; @@ -511,7 +511,7 @@ mod tests { #[test] fn validate_befp_wrong_shares() { let mut gen = ExtendedHeaderGenerator::new(); - let mut eds = generate_dummy_eds(8); + let mut eds = generate_dummy_eds(8, AppVersion::V2); let (eh, mut proof) = corrupt_eds(&mut gen, &mut eds); proof.shares = vec![]; diff --git a/types/src/consts.rs b/types/src/consts.rs index fb02c167..840dbac6 100644 --- a/types/src/consts.rs +++ b/types/src/consts.rs @@ -24,14 +24,117 @@ pub mod version { /// [`celestia-app`]: https://github.com/celestiaorg/celestia-app pub mod appconsts { pub use global_consts::*; - pub use v1::*; // celestia-app/pkg/appconsts/v1/app_consts - mod v1 { + /// Consts of App v1. + pub mod v1 { + /// App version. + pub const VERSION: u64 = 1; + /// Maximum width of the original data square. + pub const SQUARE_SIZE_UPPER_BOUND: usize = 128; /// Maximum width of a single subtree root when generating blob's commitment. pub const SUBTREE_ROOT_THRESHOLD: u64 = 64; + } + + // celestia-app/pkg/appconsts/v2/app_consts + /// Consts of App v2. + pub mod v2 { + /// App version. + pub const VERSION: u64 = 2; /// Maximum width of the original data square. pub const SQUARE_SIZE_UPPER_BOUND: usize = 128; + /// Maximum width of a single subtree root when generating blob's commitment. + pub const SUBTREE_ROOT_THRESHOLD: u64 = 64; + } + + // celestia-app/pkg/appconsts/v3/app_consts + /// Consts of App v3. + pub mod v3 { + /// App version. + pub const VERSION: u64 = 3; + /// Maximum width of the original data square. + pub const SQUARE_SIZE_UPPER_BOUND: usize = 128; + /// Maximum width of a single subtree root when generating blob's commitment. + pub const SUBTREE_ROOT_THRESHOLD: u64 = 64; + /// Cost of each byte in a transaction (in units of gas). + pub const TX_SIZE_COST_PER_BYTE: u64 = 10; + /// Cost of each byte in blob (in units of gas). + pub const GAS_PER_BLOB_BYTE: u64 = 8; + } + + // celestia-app/pkg/appconsts/versioned_consts.go + /// Latest App version. + pub const LATEST_VERSION: u64 = v3::VERSION; + + /// Enum with all valid App versions. + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] + #[repr(u64)] + pub enum AppVersion { + /// App v1 + V1 = v1::VERSION, + /// App v2 + V2 = v2::VERSION, + /// App v3 + V3 = v3::VERSION, + } + + impl AppVersion { + /// Latest App version variant. + pub fn latest() -> AppVersion { + AppVersion::from_u64(LATEST_VERSION).expect("Unknown app version") + } + + /// Creates `AppVersion` from a numeric value. + pub fn from_u64(version: u64) -> Option { + if version == v1::VERSION { + Some(AppVersion::V1) + } else if version == v2::VERSION { + Some(AppVersion::V2) + } else if version == v3::VERSION { + Some(AppVersion::V3) + } else { + None + } + } + + /// Returns the numeric value of App version. + pub fn as_u64(&self) -> u64 { + *self as u64 + } + } + + /// Maximum width of the original data square. + pub const fn square_size_upper_bound(app_version: AppVersion) -> usize { + match app_version { + AppVersion::V1 => v1::SQUARE_SIZE_UPPER_BOUND, + AppVersion::V2 => v2::SQUARE_SIZE_UPPER_BOUND, + AppVersion::V3 => v3::SQUARE_SIZE_UPPER_BOUND, + } + } + + /// Maximum width of a single subtree root when generating blob's commitment. + pub const fn subtree_root_threshold(app_version: AppVersion) -> u64 { + match app_version { + AppVersion::V1 => v1::SUBTREE_ROOT_THRESHOLD, + AppVersion::V2 => v2::SUBTREE_ROOT_THRESHOLD, + AppVersion::V3 => v3::SUBTREE_ROOT_THRESHOLD, + } + } + + /// Cost of each byte in a transaction (in units of gas). + pub const fn tx_size_cost_per_byte(app_version: AppVersion) -> Option { + match app_version { + AppVersion::V1 | AppVersion::V2 => None, + AppVersion::V3 => Some(v3::TX_SIZE_COST_PER_BYTE), + } + } + + /// Cost of each byte in blob (in units of gas). + pub const fn gas_per_blob_byte(app_version: AppVersion) -> Option { + match app_version { + AppVersion::V1 | AppVersion::V2 => None, + AppVersion::V3 => Some(v3::GAS_PER_BLOB_BYTE), + } } // celestia-app/pkg/appconsts/global_consts @@ -97,7 +200,9 @@ pub mod data_availability_header { /// A maximum width of the [`ExtendedDataSquare`]. /// /// [`ExtendedDataSquare`]: crate::eds::ExtendedDataSquare - pub const MAX_EXTENDED_SQUARE_WIDTH: usize = super::appconsts::SQUARE_SIZE_UPPER_BOUND * 2; + pub const fn max_extended_square_width(app_version: super::appconsts::AppVersion) -> usize { + super::appconsts::square_size_upper_bound(app_version) * 2 + } /// A minimum width of the [`ExtendedDataSquare`]. /// /// [`ExtendedDataSquare`]: crate::eds::ExtendedDataSquare diff --git a/types/src/data_availability_header.rs b/types/src/data_availability_header.rs index 53373a59..92b6b386 100644 --- a/types/src/data_availability_header.rs +++ b/types/src/data_availability_header.rs @@ -7,15 +7,16 @@ use celestia_tendermint_proto::Protobuf; use serde::{Deserialize, Serialize}; use sha2::Sha256; +use crate::consts::appconsts::AppVersion; use crate::consts::data_availability_header::{ - MAX_EXTENDED_SQUARE_WIDTH, MIN_EXTENDED_SQUARE_WIDTH, + max_extended_square_width, MIN_EXTENDED_SQUARE_WIDTH, }; use crate::eds::AxisType; use crate::hash::Hash; use crate::nmt::{NamespacedHash, NamespacedHashExt}; use crate::{ bail_validation, bail_verification, validation_error, Error, ExtendedDataSquare, MerkleProof, - Result, ValidateBasic, ValidationError, + Result, ValidateBasicWithAppVersion, ValidationError, }; /// Header with commitments of the data availability. @@ -66,9 +67,13 @@ pub struct DataAvailabilityHeader { impl DataAvailabilityHeader { /// Create new [`DataAvailabilityHeader`]. - pub fn new(row_roots: Vec, column_roots: Vec) -> Result { + pub fn new( + row_roots: Vec, + column_roots: Vec, + app_version: AppVersion, + ) -> Result { let dah = DataAvailabilityHeader::new_unchecked(row_roots, column_roots); - dah.validate_basic()?; + dah.validate_basic(app_version)?; Ok(dah) } @@ -250,8 +255,10 @@ impl From for RawDataAvailabilityHeader { } } -impl ValidateBasic for DataAvailabilityHeader { - fn validate_basic(&self) -> Result<(), ValidationError> { +impl ValidateBasicWithAppVersion for DataAvailabilityHeader { + fn validate_basic(&self, app_version: AppVersion) -> Result<(), ValidationError> { + let max_extended_square_width = max_extended_square_width(app_version); + if self.column_roots.len() != self.row_roots.len() { bail_validation!( "column_roots len ({}) != row_roots len ({})", @@ -268,11 +275,11 @@ impl ValidateBasic for DataAvailabilityHeader { ) } - if self.row_roots.len() > MAX_EXTENDED_SQUARE_WIDTH { + if self.row_roots.len() > max_extended_square_width { bail_validation!( "row_roots len ({}) > maximum ({})", self.row_roots.len(), - MAX_EXTENDED_SQUARE_WIDTH, + max_extended_square_width, ) } @@ -425,7 +432,7 @@ mod tests { fn validate_correct() { let dah = sample_dah(); - dah.validate_basic().unwrap(); + dah.validate_basic(AppVersion::V2).unwrap(); } #[test] @@ -433,7 +440,7 @@ mod tests { let mut dah = sample_dah(); dah.row_roots.pop(); - dah.validate_basic().unwrap_err(); + dah.validate_basic(AppVersion::V2).unwrap_err(); } #[test] @@ -452,12 +459,12 @@ mod tests { .take(MIN_EXTENDED_SQUARE_WIDTH) .collect(); - dah.validate_basic().unwrap(); + dah.validate_basic(AppVersion::V2).unwrap(); dah.row_roots.pop(); dah.column_roots.pop(); - dah.validate_basic().unwrap_err(); + dah.validate_basic(AppVersion::V2).unwrap_err(); } #[test] @@ -467,21 +474,21 @@ mod tests { .row_roots .into_iter() .cycle() - .take(MAX_EXTENDED_SQUARE_WIDTH) + .take(max_extended_square_width(AppVersion::V2)) .collect(); dah.column_roots = dah .column_roots .into_iter() .cycle() - .take(MAX_EXTENDED_SQUARE_WIDTH) + .take(max_extended_square_width(AppVersion::V2)) .collect(); - dah.validate_basic().unwrap(); + dah.validate_basic(AppVersion::V2).unwrap(); dah.row_roots.push(dah.row_roots[0].clone()); dah.column_roots.push(dah.column_roots[0].clone()); - dah.validate_basic().unwrap_err(); + dah.validate_basic(AppVersion::V2).unwrap_err(); } #[test] @@ -607,6 +614,6 @@ mod tests { }) .unzip(); - DataAvailabilityHeader::new(row_roots, col_roots).unwrap() + DataAvailabilityHeader::new(row_roots, col_roots, AppVersion::V2).unwrap() } } diff --git a/types/src/eds.rs b/types/src/eds.rs index e89bc960..581d3e49 100644 --- a/types/src/eds.rs +++ b/types/src/eds.rs @@ -1,11 +1,13 @@ +//! Types related to EDS. + use std::cmp::Ordering; use std::fmt::Display; use serde::{Deserialize, Serialize}; -use crate::consts::appconsts::SHARE_SIZE; +use crate::consts::appconsts::{AppVersion, SHARE_SIZE}; use crate::consts::data_availability_header::{ - MAX_EXTENDED_SQUARE_WIDTH, MIN_EXTENDED_SQUARE_WIDTH, + max_extended_square_width, MIN_EXTENDED_SQUARE_WIDTH, }; use crate::nmt::{Namespace, NamespacedSha2Hasher, Nmt, NmtExt, NS_SIZE}; use crate::row_namespace_data::{RowNamespaceData, RowNamespaceDataId}; @@ -47,7 +49,7 @@ impl Display for AxisType { /// The data matrix in Celestia blocks extended with parity data. /// /// It is created by a fixed size chunks of data, called [`Share`]s. - +/// /// Each share is a cell of the [`ExtendedDataSquare`]. /// /// # Structure @@ -128,8 +130,8 @@ impl Display for AxisType { /// [`Nmt`]: crate::nmt::Nmt /// [`Share`]: crate::share::Share /// [`DataAvailabilityHeader`]: crate::DataAvailabilityHeader -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(try_from = "RawExtendedDataSquare", into = "RawExtendedDataSquare")] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(into = "RawExtendedDataSquare")] pub struct ExtendedDataSquare { /// The raw data of the EDS. data_square: Vec, @@ -151,12 +153,14 @@ impl ExtendedDataSquare { /// - shares are of sizes different than [`SHARE_SIZE`] /// - amount of shares doesn't allow for forming a square /// - width of the square is smaller than [`MIN_EXTENDED_SQUARE_WIDTH`] - /// - width of the square is bigger than [`MAX_EXTENDED_SQUARE_WIDTH`] + /// - width of the square is bigger than [`max_extended_square_width`] /// - width of the square isn't a power of 2 /// - namespaces of shares aren't in non-decreasing order row and column wise - pub fn new(shares: Vec>, codec: String) -> Result { + pub fn new(shares: Vec>, codec: String, app_version: AppVersion) -> Result { const MIN_SHARES: usize = MIN_EXTENDED_SQUARE_WIDTH * MIN_EXTENDED_SQUARE_WIDTH; - const MAX_SHARES: usize = MAX_EXTENDED_SQUARE_WIDTH * MAX_EXTENDED_SQUARE_WIDTH; + + let max_extended_square_width = max_extended_square_width(app_version); + let max_shares = max_extended_square_width * max_extended_square_width; if shares.len() < MIN_SHARES { bail_validation!( @@ -165,11 +169,11 @@ impl ExtendedDataSquare { MIN_SHARES ); } - if shares.len() > MAX_SHARES { + if shares.len() > max_shares { bail_validation!( - "shares len ({}) > MAX_SHARES ({})", + "shares len ({}) > max shares ({})", shares.len(), - MAX_SHARES + max_shares ); } @@ -235,6 +239,11 @@ impl ExtendedDataSquare { Ok(eds) } + /// Creates an `ExtendedDataSquare` from [`RawExtendedDataSquare`] and an [`AppVersion`]. + pub fn from_raw(raw_eds: RawExtendedDataSquare, app_version: AppVersion) -> Result { + ExtendedDataSquare::new(raw_eds.data_square, raw_eds.codec, app_version) + } + /// Crate a new EDS that represents an empty block pub fn empty() -> ExtendedDataSquare { // ODS in this case it is just one tail padded share. @@ -245,7 +254,9 @@ impl ExtendedDataSquare { ] .concat()]; - ExtendedDataSquare::from_ods(ods).expect("invalid EDS") + // App version doesn't matter in this case because an empty ODS is constructed + // with the minimum size allowed shares, which is the same in any version. + ExtendedDataSquare::from_ods(ods, AppVersion::V1).expect("invalid EDS") } /// Create a new EDS out of the provided original data square shares. @@ -261,7 +272,10 @@ impl ExtendedDataSquare { /// will be checked after the parity data is generated. /// /// Additionally, this function will propagate any error from encoding parity data. - pub fn from_ods(mut ods_shares: Vec>) -> Result { + pub fn from_ods( + mut ods_shares: Vec>, + app_version: AppVersion, + ) -> Result { let ods_width = f64::sqrt(ods_shares.len() as f64) as usize; // this couldn't be detected later in `new()` if ods_width * ods_width != ods_shares.len() { @@ -294,7 +308,7 @@ impl ExtendedDataSquare { leopard_codec::encode(row, ods_width)?; } - ExtendedDataSquare::new(eds_shares, "Leopard".to_string()) + ExtendedDataSquare::new(eds_shares, "Leopard".to_string(), app_version) } /// The raw data of the EDS. @@ -431,21 +445,16 @@ impl ExtendedDataSquare { } } +/// Raw representation of [`ExtendedDataSquare`]. #[derive(Serialize, Deserialize)] -struct RawExtendedDataSquare { +pub struct RawExtendedDataSquare { + /// The raw data of the EDS. #[serde(with = "celestia_tendermint_proto::serializers::bytes::vec_base64string")] pub data_square: Vec>, + /// The codec used to encode parity shares. pub codec: String, } -impl TryFrom for ExtendedDataSquare { - type Error = Error; - - fn try_from(eds: RawExtendedDataSquare) -> Result { - ExtendedDataSquare::new(eds.data_square, eds.codec) - } -} - impl From for RawExtendedDataSquare { fn from(eds: ExtendedDataSquare) -> RawExtendedDataSquare { RawExtendedDataSquare { @@ -495,7 +504,8 @@ mod tests { #[test] fn get_namespaced_data() { let eds_json = include_str!("../test_data/shwap_samples/eds.json"); - let eds: ExtendedDataSquare = serde_json::from_str(eds_json).unwrap(); + let raw_eds: RawExtendedDataSquare = serde_json::from_str(eds_json).unwrap(); + let eds = ExtendedDataSquare::from_raw(raw_eds, AppVersion::V2).unwrap(); let dah_json = include_str!("../test_data/shwap_samples/dah.json"); let dah: DataAvailabilityHeader = serde_json::from_str(dah_json).unwrap(); @@ -524,7 +534,8 @@ mod tests { #[test] fn nmt_roots() { let eds_json = include_str!("../test_data/shwap_samples/eds.json"); - let eds: ExtendedDataSquare = serde_json::from_str(eds_json).unwrap(); + let raw_eds: RawExtendedDataSquare = serde_json::from_str(eds_json).unwrap(); + let eds = ExtendedDataSquare::from_raw(raw_eds, AppVersion::V2).unwrap(); let dah_json = include_str!("../test_data/shwap_samples/dah.json"); let dah: DataAvailabilityHeader = serde_json::from_str(dah_json).unwrap(); @@ -597,7 +608,7 @@ mod tests { raw_share(3, 0), raw_share(3, 1), raw_share(3, 2), raw_share(3, 3), ]; - let eds = ExtendedDataSquare::new(shares, "fake".to_string()).unwrap(); + let eds = ExtendedDataSquare::new(shares, "fake".to_string(), AppVersion::V2).unwrap(); assert_eq!( eds.row(0).unwrap(), @@ -750,13 +761,28 @@ mod tests { #[test] fn validation() { - ExtendedDataSquare::new(vec![], "fake".to_string()).unwrap_err(); - ExtendedDataSquare::new(vec![vec![]], "fake".to_string()).unwrap_err(); - ExtendedDataSquare::new(vec![vec![]; 4], "fake".to_string()).unwrap_err(); - - ExtendedDataSquare::new(vec![vec![0u8; SHARE_SIZE]; 4], "fake".to_string()).unwrap(); - ExtendedDataSquare::new(vec![vec![0u8; SHARE_SIZE]; 6], "fake".to_string()).unwrap_err(); - ExtendedDataSquare::new(vec![vec![0u8; SHARE_SIZE]; 16], "fake".to_string()).unwrap(); + ExtendedDataSquare::new(vec![], "fake".to_string(), AppVersion::V2).unwrap_err(); + ExtendedDataSquare::new(vec![vec![]], "fake".to_string(), AppVersion::V2).unwrap_err(); + ExtendedDataSquare::new(vec![vec![]; 4], "fake".to_string(), AppVersion::V2).unwrap_err(); + + ExtendedDataSquare::new( + vec![vec![0u8; SHARE_SIZE]; 4], + "fake".to_string(), + AppVersion::V2, + ) + .unwrap(); + ExtendedDataSquare::new( + vec![vec![0u8; SHARE_SIZE]; 6], + "fake".to_string(), + AppVersion::V2, + ) + .unwrap_err(); + ExtendedDataSquare::new( + vec![vec![0u8; SHARE_SIZE]; 16], + "fake".to_string(), + AppVersion::V2, + ) + .unwrap(); let share = |n| { [ @@ -766,50 +792,67 @@ mod tests { .concat() }; - ExtendedDataSquare::from_ods(vec![ - // row 0 - share(0), // ODS - ]) + ExtendedDataSquare::from_ods( + vec![ + // row 0 + share(0), // ODS + ], + AppVersion::V2, + ) .unwrap(); - ExtendedDataSquare::from_ods(vec![ - // row 0 - share(1), - share(2), - // row 1 - share(1), - share(3), - ]) + ExtendedDataSquare::from_ods( + vec![ + // row 0 + share(1), + share(2), + // row 1 + share(1), + share(3), + ], + AppVersion::V2, + ) .unwrap(); - ExtendedDataSquare::from_ods(vec![ - // row 0 - share(1), - share(2), - // row 1 - share(1), - share(1), // error: smaller namespace in 2nd column - ]) + ExtendedDataSquare::from_ods( + vec![ + // row 0 + share(1), + share(2), + // row 1 + share(1), + share(1), // error: smaller namespace in 2nd column + ], + AppVersion::V2, + ) .unwrap_err(); - ExtendedDataSquare::from_ods(vec![ - // row 0 - share(1), - share(1), - // row 1 - share(2), - share(1), // error: smaller namespace in 2nd row - ]) + ExtendedDataSquare::from_ods( + vec![ + // row 0 + share(1), + share(1), + // row 1 + share(2), + share(1), // error: smaller namespace in 2nd row + ], + AppVersion::V2, + ) .unwrap_err(); // not a power of 2 - ExtendedDataSquare::new(vec![share(1); 6 * 6], "fake".to_string()).unwrap_err(); + ExtendedDataSquare::new(vec![share(1); 6 * 6], "fake".to_string(), AppVersion::V2) + .unwrap_err(); // too big // we need to go to the next power of 2 or we just hit other checks - let square_width = MAX_EXTENDED_SQUARE_WIDTH * 2; - ExtendedDataSquare::new(vec![share(1); square_width.pow(2)], "fake".to_string()) - .unwrap_err(); + let square_width = max_extended_square_width(AppVersion::V2) * 2; + ExtendedDataSquare::new( + vec![share(1); square_width.pow(2)], + "fake".to_string(), + AppVersion::V2, + ) + .unwrap_err(); } #[test] @@ -824,9 +867,9 @@ mod tests { #[test] fn reconstruct_all() { - let eds = generate_eds(8 << (rand::random::() % 6)); + let eds = generate_eds(8 << (rand::random::() % 6), AppVersion::V2); - let blobs = Blob::reconstruct_all(eds.data_square()).unwrap(); + let blobs = Blob::reconstruct_all(eds.data_square(), AppVersion::V2).unwrap(); // first ods row has PFB's, one blob occupies 2 rows, and rest rows have 1 blob each let expected = eds.square_width() as usize / 2 - 2; assert_eq!(blobs.len(), expected); diff --git a/types/src/extended_header.rs b/types/src/extended_header.rs index e1305564..ab300c4e 100644 --- a/types/src/extended_header.rs +++ b/types/src/extended_header.rs @@ -13,10 +13,12 @@ use celestia_tendermint::{validator, Hash, Time}; use celestia_tendermint_proto::Protobuf; use serde::{Deserialize, Serialize}; +use crate::consts::appconsts::AppVersion; use crate::trust_level::DEFAULT_TRUST_LEVEL; use crate::validator_set::ValidatorSetExt; use crate::{ - bail_validation, bail_verification, DataAvailabilityHeader, Error, Result, ValidateBasic, + bail_validation, bail_verification, validation_error, DataAvailabilityHeader, Error, Result, + ValidateBasic, ValidateBasicWithAppVersion, }; /// Information about a tendermint validator. @@ -182,7 +184,12 @@ impl ExtendedHeader { &self.commit, )?; - self.dah.validate_basic()?; + let app_version = self.header.version.app; + let app_version = AppVersion::from_u64(app_version).ok_or_else(|| { + validation_error!("Invalid or unsupported AppVersion in header: {app_version}") + })?; + + self.dah.validate_basic(app_version)?; Ok(()) } diff --git a/types/src/fraud_proof.rs b/types/src/fraud_proof.rs index de302763..04d6e649 100644 --- a/types/src/fraud_proof.rs +++ b/types/src/fraud_proof.rs @@ -93,6 +93,7 @@ impl Serialize for Proof { #[cfg(test)] mod tests { + use crate::consts::appconsts::AppVersion; use crate::test_utils::{corrupt_eds, generate_dummy_eds, ExtendedHeaderGenerator}; use super::*; @@ -100,7 +101,7 @@ mod tests { #[test] fn befp_serde() { let mut gen = ExtendedHeaderGenerator::new(); - let mut eds = generate_dummy_eds(8); + let mut eds = generate_dummy_eds(8, AppVersion::V2); let (_, proof) = corrupt_eds(&mut gen, &mut eds); let proof = Proof::BadEncoding(proof); diff --git a/types/src/lib.rs b/types/src/lib.rs index 25676617..b7c87fb7 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -6,7 +6,7 @@ mod block; mod byzantine; pub mod consts; mod data_availability_header; -mod eds; +pub mod eds; mod error; mod extended_header; pub mod fraud_proof; @@ -32,6 +32,7 @@ mod validator_set; pub use crate::blob::{Blob, Commitment}; pub use crate::block::*; +pub use crate::consts::appconsts::AppVersion; pub use crate::data_availability_header::*; pub use crate::eds::{AxisType, ExtendedDataSquare}; pub use crate::error::*; diff --git a/types/src/row.rs b/types/src/row.rs index 5cd9d62b..d6ed4d50 100644 --- a/types/src/row.rs +++ b/types/src/row.rs @@ -261,7 +261,7 @@ impl From for CidGeneric { #[cfg(test)] mod tests { use super::*; - use crate::consts::appconsts::SHARE_SIZE; + use crate::consts::appconsts::{AppVersion, SHARE_SIZE}; use crate::test_utils::{generate_dummy_eds, generate_eds}; use crate::Blob; @@ -281,7 +281,7 @@ mod tests { #[test] fn index_calculation() { let shares = vec![vec![0; SHARE_SIZE]; 8 * 8]; - let eds = ExtendedDataSquare::new(shares, "codec".to_string()).unwrap(); + let eds = ExtendedDataSquare::new(shares, "codec".to_string(), AppVersion::V2).unwrap(); Row::new(1, &eds).unwrap(); Row::new(7, &eds).unwrap(); @@ -369,7 +369,7 @@ mod tests { #[test] fn test_roundtrip_verify() { for _ in 0..5 { - let eds = generate_dummy_eds(2 << (rand::random::() % 8)); + let eds = generate_dummy_eds(2 << (rand::random::() % 8), AppVersion::V2); let dah = DataAvailabilityHeader::from_eds(&eds); let index = rand::random::() % eds.square_width(); @@ -393,11 +393,14 @@ mod tests { #[test] fn reconstruct_all() { for _ in 0..3 { - let eds = generate_eds(8 << (rand::random::() % 6)); + let eds = generate_eds(8 << (rand::random::() % 6), AppVersion::V2); let rows: Vec<_> = (1..4).map(|row| Row::new(row, &eds).unwrap()).collect(); - let blobs = - Blob::reconstruct_all(rows.iter().flat_map(|row| row.shares.iter())).unwrap(); + let blobs = Blob::reconstruct_all( + rows.iter().flat_map(|row| row.shares.iter()), + AppVersion::V2, + ) + .unwrap(); assert_eq!(blobs.len(), 2); } diff --git a/types/src/row_namespace_data.rs b/types/src/row_namespace_data.rs index 89b6775e..9af9dae1 100644 --- a/types/src/row_namespace_data.rs +++ b/types/src/row_namespace_data.rs @@ -268,12 +268,10 @@ impl From for CidGeneric { #[cfg(test)] mod tests { - use crate::{ - test_utils::{generate_dummy_eds, generate_eds}, - Blob, - }; - use super::*; + use crate::consts::appconsts::AppVersion; + use crate::test_utils::{generate_dummy_eds, generate_eds}; + use crate::Blob; #[test] fn round_trip() { @@ -381,7 +379,7 @@ mod tests { fn test_roundtrip_verify() { // random for _ in 0..5 { - let eds = generate_dummy_eds(2 << (rand::random::() % 8)); + let eds = generate_dummy_eds(2 << (rand::random::() % 8), AppVersion::V2); let dah = DataAvailabilityHeader::from_eds(&eds); let namespace = eds.share(1, 1).unwrap().namespace(); @@ -396,7 +394,7 @@ mod tests { } // parity share - let eds = generate_dummy_eds(2 << (rand::random::() % 8)); + let eds = generate_dummy_eds(2 << (rand::random::() % 8), AppVersion::V2); let dah = DataAvailabilityHeader::from_eds(&eds); for (id, row) in eds .get_namespace_data(Namespace::PARITY_SHARE, &dah, 1) @@ -413,7 +411,7 @@ mod tests { #[test] fn reconstruct_all() { for _ in 0..3 { - let eds = generate_eds(8 << (rand::random::() % 6)); + let eds = generate_eds(8 << (rand::random::() % 6), AppVersion::V2); let dah = DataAvailabilityHeader::from_eds(&eds); let mut namespaces: Vec<_> = eds @@ -429,7 +427,7 @@ mod tests { assert_eq!(namespace_data.len(), 3); let shares = namespace_data.iter().flat_map(|(_, row)| row.shares.iter()); - let blobs = Blob::reconstruct_all(shares).unwrap(); + let blobs = Blob::reconstruct_all(shares, AppVersion::V2).unwrap(); assert_eq!(blobs.len(), 2); // rest of namespaces should have 1 blob each @@ -438,7 +436,7 @@ mod tests { assert_eq!(namespace_data.len(), 1); let shares = namespace_data.iter().flat_map(|(_, row)| row.shares.iter()); - let blobs = Blob::reconstruct_all(shares).unwrap(); + let blobs = Blob::reconstruct_all(shares, AppVersion::V2).unwrap(); assert_eq!(blobs.len(), 1); } } diff --git a/types/src/sample.rs b/types/src/sample.rs index 39a72c3c..5a055f88 100644 --- a/types/src/sample.rs +++ b/types/src/sample.rs @@ -325,6 +325,7 @@ impl From for CidGeneric { #[cfg(test)] mod tests { use super::*; + use crate::consts::appconsts::AppVersion; use crate::test_utils::generate_dummy_eds; #[test] @@ -342,7 +343,7 @@ mod tests { #[test] fn index_calculation() { - let eds = generate_dummy_eds(8); + let eds = generate_dummy_eds(8, AppVersion::V2); Sample::new(0, 0, AxisType::Row, &eds).unwrap(); Sample::new(7, 6, AxisType::Row, &eds).unwrap(); @@ -413,7 +414,7 @@ mod tests { #[test] fn test_roundtrip_verify() { for _ in 0..5 { - let eds = generate_dummy_eds(2 << (rand::random::() % 8)); + let eds = generate_dummy_eds(2 << (rand::random::() % 8), AppVersion::V2); let dah = DataAvailabilityHeader::from_eds(&eds); let row_index = rand::random::() % eds.square_width(); diff --git a/types/src/share.rs b/types/src/share.rs index ed282c65..40fa9dfd 100644 --- a/types/src/share.rs +++ b/types/src/share.rs @@ -209,6 +209,7 @@ impl From for RawShare { #[cfg(test)] mod tests { use super::*; + use crate::consts::appconsts::AppVersion; use crate::nmt::{NamespaceProof, NamespacedHash, NAMESPACED_HASH_SIZE}; use crate::Blob; use base64::prelude::*; @@ -219,7 +220,7 @@ mod tests { #[test] fn share_structure() { let ns = Namespace::new_v0(b"foo").unwrap(); - let blob = Blob::new(ns, vec![7; 512]).unwrap(); + let blob = Blob::new(ns, vec![7; 512], AppVersion::V2).unwrap(); let shares = blob.to_shares().unwrap(); diff --git a/types/src/test_utils.rs b/types/src/test_utils.rs index f4f534ca..00c75d8a 100644 --- a/types/src/test_utils.rs +++ b/types/src/test_utils.rs @@ -11,8 +11,8 @@ use rand::RngCore; use crate::block::{CommitExt, GENESIS_HEIGHT}; pub use crate::byzantine::test_utils::corrupt_eds; use crate::consts::appconsts::{ - CONTINUATION_SPARSE_SHARE_CONTENT_SIZE, FIRST_SPARSE_SHARE_CONTENT_SIZE, SHARE_INFO_BYTES, - SHARE_SIZE, + AppVersion, CONTINUATION_SPARSE_SHARE_CONTENT_SIZE, FIRST_SPARSE_SHARE_CONTENT_SIZE, + SHARE_INFO_BYTES, SHARE_SIZE, }; use crate::consts::version; use crate::hash::{Hash, HashExt}; @@ -361,7 +361,7 @@ pub fn unverify(header: &mut ExtendedHeader) { } /// Generate a properly encoded [`ExtendedDataSquare`] with random data. -pub fn generate_dummy_eds(square_width: usize) -> ExtendedDataSquare { +pub fn generate_dummy_eds(square_width: usize, app_version: AppVersion) -> ExtendedDataSquare { let ns = Namespace::const_v0(rand::random()); let ods_width = square_width / 2; @@ -376,7 +376,7 @@ pub fn generate_dummy_eds(square_width: usize) -> ExtendedDataSquare { }) .collect(); - ExtendedDataSquare::from_ods(shares).unwrap() + ExtendedDataSquare::from_ods(shares, app_version).unwrap() } /// Generate a properly encoded [`ExtendedDataSquare`] with random data. @@ -391,7 +391,7 @@ pub fn generate_dummy_eds(square_width: usize) -> ExtendedDataSquare { /// is padded with tail padding namespace. /// /// Minimum supported square_width is 8. -pub fn generate_eds(square_width: usize) -> ExtendedDataSquare { +pub fn generate_eds(square_width: usize, app_version: AppVersion) -> ExtendedDataSquare { assert!(square_width >= 8); let ods_width = square_width / 2; @@ -426,7 +426,7 @@ pub fn generate_eds(square_width: usize) -> ExtendedDataSquare { // first blob is bigger so that it spans over 2 rows let blob_shares = (rand::random::() % (ods_width - 1)) + ods_width + 1; let data = random_bytes(blob_len(blob_shares)); - let blob = Blob::new(namespaces[0], data).unwrap(); + let blob = Blob::new(namespaces[0], data, app_version).unwrap(); shares.extend(blob.to_shares().unwrap().iter().map(Share::to_vec)); // namespace padding @@ -439,7 +439,7 @@ pub fn generate_eds(square_width: usize) -> ExtendedDataSquare { for ns in &namespaces { let blob_shares = (rand::random::() % (ods_width - 1)) + 1; let data = random_bytes(blob_len(blob_shares)); - let blob = Blob::new(*ns, data).unwrap(); + let blob = Blob::new(*ns, data, app_version).unwrap(); shares.extend(blob.to_shares().unwrap().iter().map(Share::to_vec)); let padding_ns = if ns != namespaces.last().unwrap() { @@ -453,7 +453,7 @@ pub fn generate_eds(square_width: usize) -> ExtendedDataSquare { ); } - ExtendedDataSquare::from_ods(shares).unwrap() + ExtendedDataSquare::from_ods(shares, app_version).unwrap() } fn blob_len(shares: usize) -> usize { diff --git a/types/src/validate.rs b/types/src/validate.rs index e92422a9..5bf39018 100644 --- a/types/src/validate.rs +++ b/types/src/validate.rs @@ -1,3 +1,4 @@ +use crate::consts::appconsts::AppVersion; use crate::ValidationError; /// A trait to perform basic validation of the data consistency. @@ -5,3 +6,9 @@ pub trait ValidateBasic { /// Perform a basic validation of the data consistency. fn validate_basic(&self) -> Result<(), ValidationError>; } + +/// A trait to perform basic validation of the data consistency. +pub trait ValidateBasicWithAppVersion { + /// Perform a basic validation of the data consistency. + fn validate_basic(&self, app_version: AppVersion) -> Result<(), ValidationError>; +}