feat(proto,types,rpc)!: celestia node v0.20.4 upgrade

ci/Dockerfile.bridge

@@ -6,8 +6,8 @@ FROM docker.io/alpine:3.19.1
 RUN apk update && apk add --no-cache bash jq dasel
 
 # Copy in the binary
-COPY --from=ghcr.io/celestiaorg/celestia-node:v0.18.3-mocha /bin/celestia /bin/celestia
-COPY --from=ghcr.io/celestiaorg/celestia-node:v0.18.3-mocha /bin/cel-key /bin/cel-key
+COPY --from=ghcr.io/celestiaorg/celestia-node:v0.20.4 /bin/celestia /bin/celestia
+COPY --from=ghcr.io/celestiaorg/celestia-node:v0.20.4 /bin/cel-key /bin/cel-key
 
 COPY ./run-bridge.sh /opt/entrypoint.sh
 

ci/Dockerfile.validator

@@ -8,7 +8,7 @@ ENV CELESTIA_HOME=/root
 RUN apk update && apk add --no-cache bash jq
 
 # Copy in the binary
-COPY --from=ghcr.io/celestiaorg/celestia-app:v2.3.0 /bin/celestia-appd /bin/celestia-appd
+COPY --from=ghcr.io/celestiaorg/celestia-app:v3.0.2 /bin/celestia-appd /bin/celestia-appd
 
 COPY ./run-validator.sh /opt/entrypoint.sh
 

ci/docker-compose.yml

@@ -26,6 +26,7 @@ services:
       - NODE_ID=0
       # setting SKIP_AUTH to true disables the use of JWT for authentication
       - SKIP_AUTH=true
+      - CELESTIA_ENABLE_QUIC=1
     ports:
       - 26658:26658
     volumes:
@@ -42,6 +43,7 @@ services:
       # provide an id for the bridge node (default: 0)
       # each node should have a next natural number starting from 0
       - NODE_ID=1
+      - CELESTIA_ENABLE_QUIC=1
     ports:
       - 36658:26658
     volumes:
@@ -60,6 +62,7 @@ services:
   #     # provide an id for the bridge node (default: 0)
   #     # each node should have a next natural number starting from 0
   #     - NODE_ID=2
+  #     - CELESTIA_ENABLE_QUIC=1
   #   ports:
   #     - 46658:26658
   #   volumes:

ci/run-validator.sh

@@ -34,7 +34,7 @@ wait_for_block() {
   local block_num="$1"
   local block_hash=""
 
-  # Wait for the block to be created 
+  # Wait for the block to be created
   while [[ -z "$block_hash" ]]; do
     # `|| echo` fallbacks to an empty string in case it's not ready
     block_hash="$(celestia-appd query block "$block_num" 2>/dev/null | jq '.block_id.hash' || echo)"
@@ -133,14 +133,8 @@ setup_private_validator() {
   # If you encounter: `sed: -I or -i may not be used with stdin` on MacOS you can mitigate by installing gnu-sed
   # https://gist.github.com/andre3k1/e3a1a7133fded5de5a9ee99c87c6fa0d?permalink_comment_id=3082272#gistcomment-3082272
   sed -i'.bak' 's|"tcp://127.0.0.1:26657"|"tcp://0.0.0.0:26657"|g' "$CONFIG_DIR/config/config.toml"
-  sed -i'.bak' 's|"null"|"kv"|g' "$CONFIG_DIR/config/config.toml"
-
-  # reduce the time of commiting the proposed block
-  # bringing this value too low results in errors
-  sed -i'.bak' 's|^timeout_commit.*|timeout_commit = "1s"|g' "$CONFIG_DIR/config/config.toml"
-
-  # Set app version to 1
-  sed -i'.bak' 's|"app_version": "2"|"app_version": "1"|g' "$CONFIG_DIR/config/genesis.json"
+  # enable transaction indexing
+  sed -i'.bak' 's|indexer = .*|indexer = "kv"|g' "$CONFIG_DIR/config/config.toml"
 }
 
 main() {
@@ -150,7 +144,7 @@ main() {
   provision_bridge_nodes &
   # Start the celestia-app
   echo "Configuration finished. Running a validator node..."
-  celestia-appd start --api.enable --grpc.enable
+  celestia-appd start --api.enable --grpc.enable --force-no-bbr
 }
 
 main

node/tests/shwap.rs

@@ -33,7 +33,8 @@ async fn shwap_sampling_forward() {
                 break height;
             }
         };
-        let new_head = timeout(Duration::from_secs(2), get_new_head).await.unwrap();
+        // timeout is double of the block time on CI
+        let new_head = timeout(Duration::from_secs(9), get_new_head).await.unwrap();
 
         // wait for height to be sampled
         let wait_height_sampled = async {
@@ -86,7 +87,7 @@ async fn shwap_sampling_backward() {
         .unwrap();
 
     // take just first N headers because batch size can be big
-    let mut headers_to_sample: HashSet<_> = (from_height..to_height).rev().take(50).collect();
+    let mut headers_to_sample: HashSet<_> = (from_height..to_height).rev().take(25).collect();
 
     // wait for all heights to be sampled
     timeout(Duration::from_secs(10), async {

proto/build.rs

@@ -17,6 +17,7 @@ const OPTION_TIMESTAMP: &str = r#"#[serde(with = "crate::serializers::option_tim
 const OPTION_PROTOBUF_DURATION: &str =
     r#"#[serde(with = "crate::serializers::option_protobuf_duration")]"#;
 const NULL_DEFAULT: &str = r#"#[serde(with = "crate::serializers::null_default")]"#;
+const VEC_SKIP_IF_EMPTY: &str = r#"#[serde(skip_serializing_if = "::std::vec::Vec::is_empty")]"#;
 const BYTES_SKIP_IF_EMPTY: &str = r#"#[serde(skip_serializing_if = "bytes::Bytes::is_empty")]"#;
 
 #[rustfmt::skip]
@@ -82,6 +83,7 @@ static CUSTOM_FIELD_ATTRIBUTES: &[(&str, &str)] = &[
     (".proof.pb.Proof.nodes", VEC_BASE64STRING),
     (".proto.blob.v1.BlobProto.data", BASE64STRING),
     (".proto.blob.v1.BlobProto.namespace_id", BASE64STRING),
+    (".proto.blob.v1.BlobProto.signer", VEC_SKIP_IF_EMPTY),
     (".proto.blob.v1.BlobProto.signer", BASE64STRING),
     (".shwap.RowNamespaceData.shares", NULL_DEFAULT),
     (".shwap.Share", BASE64STRING),
@@ -111,7 +113,6 @@ const PROTO_FILES: &[&str] = &[
     "vendor/celestia/blob/v1/params.proto",
     "vendor/celestia/blob/v1/query.proto",
     "vendor/celestia/blob/v1/tx.proto",
-    "vendor/celestia/core/v1/blob/blob.proto",
     "vendor/celestia/core/v1/da/data_availability_header.proto",
     "vendor/celestia/core/v1/proof/proof.proto",
     "vendor/cosmos/auth/v1beta1/auth.proto",

rpc/Cargo.toml

@@ -23,7 +23,6 @@ celestia-types.workspace = true
 prost.workspace = true
 
 async-trait = "0.1.80"
-futures = "0.3.31"
 jsonrpsee = { version = "0.24.2", features = ["client-core", "macros"] }
 serde = { version = "1.0.203", features = ["derive"] }
 thiserror = "1.0.61"
@@ -34,6 +33,9 @@ http = "1.1.0"
 jsonrpsee = { version = "0.24.2", features = ["http-client", "ws-client"] }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
+anyhow = "1.0.86"
+dotenvy = "0.15.7"
+futures = "0.3.31"
 libp2p = { workspace = true, features = [
   "tokio",
   "macros",
@@ -42,8 +44,6 @@ libp2p = { workspace = true, features = [
   "yamux",
 ] }
 nmt-rs.workspace = true
-anyhow = "1.0.86"
-dotenvy = "0.15.7"
 rand = "0.8.5"
 tokio = { version = "1.38.0", features = ["rt", "macros"] }
 tracing = "0.1.40"

rpc/src/share.rs

@@ -5,9 +5,8 @@ use std::marker::{Send, Sync};
 
 use celestia_types::consts::appconsts::AppVersion;
 use celestia_types::nmt::Namespace;
-use celestia_types::row_namespace_data::NamespacedShares;
+use celestia_types::row_namespace_data::NamespaceData;
 use celestia_types::{ExtendedDataSquare, ExtendedHeader, RawShare, Share, ShareProof};
-use futures::FutureExt;
 use jsonrpsee::core::client::{ClientT, Error};
 use jsonrpsee::proc_macros::rpc;
 use serde::{Deserialize, Serialize};
@@ -17,7 +16,7 @@ use serde::{Deserialize, Serialize};
 #[serde(rename_all = "PascalCase")]
 pub struct GetRangeResponse {
     /// Shares contained in given range.
-    pub shares: Vec<RawShare>,
+    pub shares: Vec<Share>,
     /// Proof of inclusion of the shares.
     pub proof: ShareProof,
 }
@@ -29,40 +28,38 @@ mod rpc {
     #[rpc(client)]
     pub trait Share {
         #[method(name = "share.GetEDS")]
-        async fn share_get_eds(
-            &self,
-            root: &ExtendedHeader,
-        ) -> Result<RawExtendedDataSquare, Error>;
+        async fn share_get_eds(&self, height: u64) -> Result<RawExtendedDataSquare, Error>;
 
         #[method(name = "share.GetRange")]
         async fn share_get_range(
             &self,
             height: u64,
-            start: usize,
-            end: usize,
+            start: u64,
+            end: u64,
         ) -> Result<GetRangeResponse, Error>;
 
         #[method(name = "share.GetShare")]
-        async fn share_get_share(
-            &self,
-            root: &ExtendedHeader,
-            row: u64,
-            col: u64,
-        ) -> Result<RawShare, Error>;
+        async fn share_get_share(&self, height: u64, row: u64, col: u64)
+            -> Result<RawShare, Error>;
 
-        #[method(name = "share.GetSharesByNamespace")]
-        async fn share_get_shares_by_namespace(
+        #[method(name = "share.GetNamespaceData")]
+        async fn share_get_namespace_data(
             &self,
-            root: &ExtendedHeader,
+            height: u64,
             namespace: Namespace,
-        ) -> Result<NamespacedShares, Error>;
+        ) -> Result<NamespaceData, Error>;
 
         #[method(name = "share.SharesAvailable")]
-        async fn share_shares_available(&self, root: &ExtendedHeader) -> Result<(), Error>;
+        async fn share_shares_available(&self, height: u64) -> Result<(), Error>;
     }
 }
 
 /// Client implementation for the `Share` RPC API.
+///
+/// Please note that celestia-node requires just the block height for most of those API's.
+/// This trait instead requires [`ExtendedHeader`] to perform validation of the returned types.
+// NOTE: we use EH anyhwhere where Share is returned because it's gonna be required in future,
+// to check if shares are allowed to have version 1 in corresponding app version
 pub trait ShareClient: ClientT {
     /// GetEDS gets the full EDS identified by the given root.
     fn share_get_eds<'a, 'b, 'fut>(
@@ -81,26 +78,28 @@ pub trait ShareClient: ClientT {
                 Error::Custom(e)
             })?;
 
-            let raw_eds = rpc::ShareClient::share_get_eds(self, root).await?;
+            let raw_eds = rpc::ShareClient::share_get_eds(self, root.height().value()).await?;
 
             ExtendedDataSquare::from_raw(raw_eds, app_version)
                 .map_err(|e| Error::Custom(e.to_string()))
         }
     }
 
     /// GetRange gets a list of shares and their corresponding proof.
+    ///
+    /// The start and end index ignores parity shares and corresponds to ODS.
     fn share_get_range<'a, 'b, 'fut>(
         &'a self,
-        height: u64,
-        start: usize,
-        end: usize,
+        root: &'b ExtendedHeader,
+        start: u64,
+        end: u64,
     ) -> impl Future<Output = Result<GetRangeResponse, Error>> + Send + 'fut
     where
         'a: 'fut,
         'b: 'fut,
         Self: Sized + Sync + 'fut,
     {
-        rpc::ShareClient::share_get_range(self, height, start, end)
+        rpc::ShareClient::share_get_range(self, root.height().value(), start, end)
     }
 
     /// GetShare gets a Share by coordinates in EDS.
@@ -115,46 +114,52 @@ pub trait ShareClient: ClientT {
         'b: 'fut,
         Self: Sized + Sync + 'fut,
     {
-        rpc::ShareClient::share_get_share(self, root, row, col).map(move |res| {
-            res.and_then(|shr| {
-                if row < root.dah.square_width() as u64 / 2
-                    && col < root.dah.square_width() as u64 / 2
-                {
-                    Share::from_raw(&shr.data)
-                } else {
-                    Share::parity(&shr.data)
-                }
-                .map_err(|e| Error::Custom(e.to_string()))
-            })
-        })
+        async move {
+            let share =
+                rpc::ShareClient::share_get_share(self, root.height().value(), row, col).await?;
+            if is_ods_square(row, col, root.dah.square_width()) {
+                Share::from_raw(&share.data)
+            } else {
+                Share::parity(&share.data)
+            }
+            .map_err(|e| Error::Custom(e.to_string()))
+        }
     }
 
-    /// GetSharesByNamespace gets all shares from an EDS within the given namespace. Shares are returned in a row-by-row order if the namespace spans multiple rows.
-    fn share_get_shares_by_namespace<'a, 'b, 'fut>(
+    /// GetNamespaceData gets all shares from an EDS within the given namespace.
+    ///
+    /// Shares are returned in a row-by-row order if the namespace spans multiple rows.
+    ///
+    /// PARITY and TAIL PADDING namespaces are not allowed.
+    fn share_get_namespace_data<'a, 'b, 'fut>(
         &'a self,
         root: &'b ExtendedHeader,
         namespace: Namespace,
-    ) -> impl Future<Output = Result<NamespacedShares, Error>> + Send + 'fut
+    ) -> impl Future<Output = Result<NamespaceData, Error>> + Send + 'fut
     where
         'a: 'fut,
         'b: 'fut,
         Self: Sized + Sync + 'fut,
     {
-        rpc::ShareClient::share_get_shares_by_namespace(self, root, namespace)
+        rpc::ShareClient::share_get_namespace_data(self, root.height().value(), namespace)
     }
 
     /// SharesAvailable subjectively validates if Shares committed to the given Root are available on the Network.
-    fn share_shares_available<'a, 'b, 'fut>(
+    fn share_shares_available<'a, 'fut>(
         &'a self,
-        root: &'b ExtendedHeader,
+        height: u64,
     ) -> impl Future<Output = Result<(), Error>> + Send + 'fut
     where
         'a: 'fut,
-        'b: 'fut,
         Self: Sized + Sync + 'fut,
     {
-        rpc::ShareClient::share_shares_available(self, root)
+        rpc::ShareClient::share_shares_available(self, height)
     }
 }
 
 impl<T> ShareClient for T where T: ClientT {}
+
+fn is_ods_square(row: u64, column: u64, square_width: u16) -> bool {
+    let ods_width = square_width / 2;
+    row < ods_width as u64 && column < ods_width as u64
+}