Allow Jupiter to run under Risc0 (#16)

* Make compatible with risc0.

Introduce the CompactHeader type, which allows us to circumvent
informalsystems/tendermint-rs#1309.
Update to Tendermint 0.32, which allows removal of unnecessary data
copies.

Feature gate `CelestiaService` and its netowkr/io dependencies
behind the "native" feature, re-organizing files as necessary.

* Update to latest sdk. Add CI

* Remove outdated da_app files

Cargo.toml

@@ -6,34 +6,42 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-sovereign-sdk = { git = "https://github.com/Sovereign-Labs/sovereign.git", rev = "5e43c3ee9b5785abdca33b21c86fd38dbd9285e0" }
-tendermint = "0.27"
+sovereign-sdk = { git = "https://github.com/Sovereign-Labs/sovereign.git", rev = "e848baea380880d7c95d3d5cfbf7486c51fe3b79" }
+tendermint = "0.32"
+tendermint-proto = "0.32"
 
 prost = "0.11"
 prost-types = "0.11"
 borsh = { version = "0.10.3", features = ["bytes"] }
 
 # Convenience
-reqwest = { version = "0.11.13", features = ["blocking"] }
-tokio = { version = "1", features = ["full"] }
+reqwest = { version = "0.11.13", features = ["blocking"], optional = true }
+tokio = { version = "1", features = ["full"], optional = true }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_cbor = "0.11.2"
 hex = { version = "0.4.3", features = ["serde"] }
 hex-literal = "0.3.4"
 base64 = "0.13.1"
 anyhow = "1.0.62"
-jsonrpsee = { version = "0.16.2", features = ["http-client"] }
+jsonrpsee = { version = "0.16.2", features = ["http-client"], optional = true }
 tracing = "0.1.37"
-tracing-subscriber = "0.3.16"
 
+nmt-rs = { git = "https://github.com/Sovereign-Labs/nmt-rs.git", rev = "aec2dcdc279b381162537f5b20ce43d1d46dc42f", features = ["serde", "borsh"] }
 
-#nmt-rs = { path = "../nmt-rs", features = ["serde", "borsh"] }
-nmt-rs = { git = "https://github.com/Sovereign-Labs/nmt-rs.git", rev = "36bffad64cf257264069e3b04679d945d0a0af36", features = ["serde", "borsh"] }
+[dev-dependencies]
+postcard = { version = "1", features = ["use-std"]}
 
 [build-dependencies]
 prost-build = { version = "0.11" }
 
 
-#[patch.'https://github.com/Sovereign-Labs/sovereign.git']
-#sovereign-sdk = { path = "../sovereign/sdk" }
+[patch.crates-io]
+# Patch tendermint until the "0.32" release lands. We need 0.32 to avoid a lot of unnecessary data copying during header verification
+tendermint = { git = "https://github.com/informalsystems/tendermint-rs.git", rev = "e014de927abed7c5fcbf8186780a61b5c9c1e775" }
+tendermint-proto = { git = "https://github.com/informalsystems/tendermint-rs.git", rev = "e014de927abed7c5fcbf8186780a61b5c9c1e775" }
+
+[features]
+default = ["native"]
+native = ["dep:tokio", "dep:reqwest", "dep:jsonrpsee"]
+verifier = []

src/celestia.rs

@@ -1,4 +1,4 @@
-use std::ops::Range;
+use std::{cell::RefCell, ops::Range};
 
 use borsh::{BorshDeserialize, BorshSerialize};
 use nmt_rs::NamespacedHash;
@@ -7,16 +7,21 @@ use serde::{Deserialize, Serialize};
 use sovereign_sdk::core::traits::{
  AddressTrait as Address, BlockHeaderTrait as BlockHeader, CanonicalHash,
 };
+pub use tendermint::block::Header as TendermintHeader;
+use tendermint::{crypto::default::Sha256, merkle::simple_hash_from_byte_vectors, Hash};
+use tendermint_proto::Protobuf;
 use tracing::debug;
 
+pub use tendermint_proto::v0_34 as celestia_tm_version;
+
 const NAMESPACED_HASH_LEN: usize = 48;
 
 use crate::{
- da_app::{address::CelestiaAddress, TmHash},
- da_service::PFB_NAMESPACE,
  pfb::{BlobTx, MsgPayForBlobs, Tx},
  shares::{read_varint, Blob, BlobRefIterator, NamespaceGroup},
  utils::BoxError,
+ verifier::PFB_NAMESPACE,
+ verifier::{address::CelestiaAddress, TmHash},
 };
 
 #[derive(Deserialize, Serialize, PartialEq, Debug, Clone)]
@@ -25,6 +30,158 @@ pub struct MarshalledDataAvailabilityHeader {
  pub column_roots: Vec<String>,
 }
 
+#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, BorshDeserialize, BorshSerialize)]
+pub struct PartialBlockId {
+ pub hash: ProtobufHash,
+ pub part_set_header: Vec<u8>,
+}
+
+/// A partially serialized tendermint header. Only fields which are actually inspected by
+/// Jupiter are included in their raw form. Other fields are pre-encoded as protobufs.
+///
+/// This type was first introduced as a way to circumvent a bug in tendermint-rs which prevents
+/// a tendermint::block::Header from being deserialized in most formats except JSON. However
+/// it also provides a significant efficiency benefit over the standard tendermint type, which
+/// performs a complete protobuf serialization every time `.hash()` is called.
+#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize)]
+pub struct CompactHeader {
+ /// Header version
+ pub version: Vec<u8>,
+
+ /// Chain ID
+ pub chain_id: Vec<u8>,
+
+ /// Current block height
+ pub height: Vec<u8>,
+
+ /// Current timestamp
+ pub time: Vec<u8>,
+
+ /// Previous block info
+ pub last_block_id: Vec<u8>,
+
+ /// Commit from validators from the last block
+ pub last_commit_hash: Vec<u8>,
+
+ /// Merkle root of transaction hashes
+ pub data_hash: Option<ProtobufHash>,
+
+ /// Validators for the current block
+ pub validators_hash: Vec<u8>,
+
+ /// Validators for the next block
+ pub next_validators_hash: Vec<u8>,
+
+ /// Consensus params for the current block
+ pub consensus_hash: Vec<u8>,
+
+ /// State after txs from the previous block
+ pub app_hash: Vec<u8>,
+
+ /// Root hash of all results from the txs from the previous block
+ pub last_results_hash: Vec<u8>,
+
+ /// Hash of evidence included in the block
+ pub evidence_hash: Vec<u8>,
+
+ /// Original proposer of the block
+ pub proposer_address: Vec<u8>,
+}
+
+trait EncodeTm34 {
+ fn encode_to_tm34_protobuf(&self) -> Result<Vec<u8>, BoxError>;
+}
+
+impl From<TendermintHeader> for CompactHeader {
+ fn from(value: TendermintHeader) -> Self {
+ let data_hash = if let Some(h) = value.data_hash {
+ match h {
+ Hash::Sha256(value) => Some(ProtobufHash(value)),
+ Hash::None => None,
+ }
+ } else {
+ None
+ };
+ Self {
+ version: Protobuf::<celestia_tm_version::version::Consensus>::encode_vec(
+ &value.version,
+ )
+ .unwrap(),
+ chain_id: value.chain_id.encode_vec().unwrap(),
+ height: value.height.encode_vec().unwrap(),
+ time: value.time.encode_vec().unwrap(),
+ last_block_id: Protobuf::<celestia_tm_version::types::BlockId>::encode_vec(
+ &value.last_block_id.unwrap_or_default(),
+ )
+ .unwrap(),
+ last_commit_hash: value
+ .last_commit_hash
+ .unwrap_or_default()
+ .encode_vec()
+ .unwrap(),
+ data_hash,
+ validators_hash: value.validators_hash.encode_vec().unwrap(),
+ next_validators_hash: value.next_validators_hash.encode_vec().unwrap(),
+ consensus_hash: value.consensus_hash.encode_vec().unwrap(),
+ app_hash: value.app_hash.encode_vec().unwrap(),
+ last_results_hash: value
+ .last_results_hash
+ .unwrap_or_default()
+ .encode_vec()
+ .unwrap(),
+ evidence_hash: value
+ .evidence_hash
+ .unwrap_or_default()
+ .encode_vec()
+ .unwrap(),
+ proposer_address: value.proposer_address.encode_vec().unwrap(),
+ }
+ }
+}
+
+#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize, BorshDeserialize, BorshSerialize)]
+pub struct ProtobufHash(pub [u8; 32]);
+
+pub fn protobuf_encode(hash: &Option<ProtobufHash>) -> Vec<u8> {
+ match hash {
+ Some(ProtobufHash(value)) => prost::Message::encode_to_vec(&value.to_vec()),
+ None => prost::Message::encode_to_vec(&vec![]),
+ }
+}
+
+impl CompactHeader {
+ /// Hash this header
+ // TODO: this function can be made even more efficient. Rather than computing the block hash,
+ // we could provide the hash as a non-deterministic input and simply verify the correctness of the
+ // fields that we care about.
+ pub fn hash(&self) -> Hash {
+ // Note that if there is an encoding problem this will
+ // panic (as the golang code would):
+ // https://github.com/tendermint/tendermint/blob/134fe2896275bb926b49743c1e25493f6b24cc31/types/block.go#L393
+ // https://github.com/tendermint/tendermint/blob/134fe2896275bb926b49743c1e25493f6b24cc31/types/encoding_helper.go#L9:6
+
+ let encoded_data_hash = protobuf_encode(&self.data_hash);
+ let fields_bytes = vec![
+ &self.version,
+ &self.chain_id,
+ &self.height,
+ &self.time,
+ &self.last_block_id,
+ &self.last_commit_hash,
+ &encoded_data_hash,
+ &self.validators_hash,
+ &self.next_validators_hash,
+ &self.consensus_hash,
+ &self.app_hash,
+ &self.last_results_hash,
+ &self.evidence_hash,
+ &self.proposer_address,
+ ];
+
+ Hash::Sha256(simple_hash_from_byte_vectors::<Sha256>(&fields_bytes))
+ }
+}
+
 #[derive(
  PartialEq, Debug, Clone, Deserialize, serde::Serialize, BorshDeserialize, BorshSerialize,
 )]
@@ -59,6 +216,8 @@ impl TryFrom<MarshalledDataAvailabilityHeader> for DataAvailabilityHeader {
  }
 }
 
+/// The response from the celestia `/header` endpoint. Must be converted to a
+/// [`CelestiaHeader`] before use.
 #[derive(Deserialize, Serialize, PartialEq, Debug, Clone)]
 pub struct CelestiaHeaderResponse {
  pub header: tendermint::block::Header,
@@ -74,10 +233,20 @@ pub struct NamespacedSharesResponse {
 #[derive(Debug, PartialEq, Clone, Deserialize, serde::Serialize)]
 pub struct CelestiaHeader {
  pub dah: DataAvailabilityHeader,
- pub header: tendermint::block::Header,
+ pub header: CompactHeader,
+ #[serde(skip)]
+ cached_prev_hash: RefCell<Option<TmHash>>,
 }
 
 impl CelestiaHeader {
+ pub fn new(dah: DataAvailabilityHeader, header: CompactHeader) -> Self {
+ Self {
+ dah,
+ header,
+ cached_prev_hash: RefCell::new(None),
+ }
+ }
+
  pub fn square_size(&self) -> usize {
  self.dah.row_roots.len()
  }
@@ -100,13 +269,22 @@ pub struct BlobWithSender {
 impl BlockHeader for CelestiaHeader {
  type Hash = TmHash;
 
- fn prev_hash(&self) -> &Self::Hash {
- self.header
- .last_block_id
- .as_ref()
+ fn prev_hash(&self) -> Self::Hash {
+ // Try to return the cached value
+ if let Some(hash) = self.cached_prev_hash.borrow().as_ref() {
+ return hash.clone();
+ }
+ // If we reach this point, we know that the cach is empty - so there can't be any outstanding references to its value.
+ // That means its safe to borrow the cache mutably and populate it.
+ let mut cached_hash = self.cached_prev_hash.borrow_mut();
+ let hash =
+ <tendermint::block::Id as Protobuf<celestia_tm_version::types::BlockId>>::decode(
+ self.header.last_block_id.as_ref(),
+ )
  .expect("must not call prev_hash on block with no predecessor")
- .hash
- .as_ref()
+ .hash;
+ *cached_hash = Some(TmHash(hash.clone()));
+ TmHash(hash)
  }
 }
 
@@ -216,3 +394,33 @@ fn next_pfb(mut data: &mut BlobRefIterator) -> Result<(MsgPayForBlobs, TxPositio
  },
  ))
 }
+
+#[cfg(test)]
+mod tests {
+ use crate::{CelestiaHeaderResponse, CompactHeader};
+
+ const HEADER_RESPONSE_JSON: &[u8] = include_bytes!("./header_response.json");
+
+ #[test]
+ fn test_compact_header_serde() {
+ let original_header: CelestiaHeaderResponse =
+ serde_json::from_slice(HEADER_RESPONSE_JSON).unwrap();
+
+ let header: CompactHeader = original_header.header.into();
+
+ let serialized_header = postcard::to_stdvec(&header).unwrap();
+ let deserialized_header: CompactHeader = postcard::from_bytes(&serialized_header).unwrap();
+ assert_eq!(deserialized_header, header)
+ }
+
+ #[test]
+ fn test_compact_header_hash() {
+ let original_header: CelestiaHeaderResponse =
+ serde_json::from_slice(HEADER_RESPONSE_JSON).unwrap();
+
+ let tm_header = original_header.header.clone();
+ let compact_header: CompactHeader = original_header.header.into();
+
+ assert_eq!(tm_header.hash(), compact_header.hash());
+ }
+}