feat: blob metadata (#144)

.github/workflows/ci.yml

@@ -162,9 +162,6 @@ jobs:
           ignore-unpublished-changes: true
 
   build-docker-images:
-    # needs:
-    #   - cargo-verifications
-    #   - e2e-tests
     strategy:
       matrix:
         arch: [
@@ -314,3 +311,30 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           slack_webhook: ${{ secrets.SLACK_WEBHOOK_NOTIFY_BUILD }}
+
+  publish:
+    needs:
+      - cargo-verifications
+      - publish-crates-check
+    # Only do this job if publishing a release
+    if: github.event_name == 'release' && github.event.action == 'published'
+    runs-on: buildjet-4vcpu-ubuntu-2204
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_VERSION }}
+
+      - name: Verify tag version
+        run: |
+          curl -sSLf "$DASEL_VERSION" -L -o dasel && chmod +x dasel
+          mv ./dasel /usr/local/bin/dasel
+          ./.github/workflows/scripts/verify_tag.sh ${{ github.ref_name }} Cargo.toml
+      - name: Publish crate
+        uses: FuelLabs/publish-crates@v1
+        with:
+          publish-delay: 30000
+          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}

Cargo.toml

@@ -4,6 +4,7 @@ members = [
   "committer",
   "e2e",
   "packages/clock",
+  "packages/encoding",
   "packages/eth",
   "packages/fuel",
   "packages/metrics",
@@ -30,8 +31,9 @@ ports = { path = "./packages/ports", default-features = false }
 storage = { path = "./packages/storage", default-features = false }
 services = { path = "./packages/services", default-features = false }
 clock = { path = "./packages/clock", default-features = false }
-
+fuel-block-committer-encoding = { path = "./packages/encoding", default-features = false }
 actix-web = { version = "4", default-features = false }
+bitvec = { version = "1.0", default-features = false }
 bytesize = { version = "1.3", default-features = false }
 alloy = { version = "0.3.6", default-features = false }
 proptest = { version = "1.0", default-features = false }
@@ -45,10 +47,14 @@ clap = { version = "4.5", default-features = false }
 config = { version = "0.14", default-features = false }
 delegate = { version = "0.13", default-features = false }
 flate2 = { version = "1.0", default-features = false }
+postcard = { version = "1.0", default-features = false }
 fs_extra = { version = "1.3", default-features = false }
 fuel-core-chain-config = { version = "0.40", default-features = false }
 fuel-core-client = { version = "0.40", default-features = false }
-fuel-core-types = { version = "0.40", default-features = false, features = ["da-compression"] }
+fuel-core-types = { version = "0.40", default-features = false, features = [
+  "da-compression",
+] }
+static_assertions = { version = "1.1", default-features = false }
 fuel-crypto = { version = "0.58.2", default-features = false }
 futures = { version = "0.3", default-features = false }
 hex = { version = "0.4", default-features = false }

committer/Cargo.toml

@@ -10,15 +10,16 @@ publish = { workspace = true }
 rust-version = { workspace = true }
 
 [dependencies]
-num_cpus = { workspace = true }
 actix-web = { workspace = true, features = ["macros"] }
 clap = { workspace = true, features = ["default", "derive"] }
 clock = { workspace = true }
 config = { workspace = true, features = ["toml", "async"] }
 eth = { workspace = true }
 fuel = { workspace = true }
+fuel-block-committer-encoding = { workspace = true }
 humantime = { workspace = true }
 metrics = { workspace = true }
+num_cpus = { workspace = true }
 ports = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }

committer/src/config.rs

@@ -7,8 +7,8 @@ use std::{
 
 use clap::{command, Parser};
 use eth::Address;
+use fuel_block_committer_encoding::bundle::CompressionLevel;
 use serde::Deserialize;
-use services::CompressionLevel;
 use storage::DbConfig;
 use url::Url;
 

committer/src/setup.rs

@@ -1,7 +1,8 @@
 use std::{num::NonZeroU32, time::Duration};
 
 use clock::SystemClock;
-use eth::{AwsConfig, Eip4844BlobEncoder, KmsKeys};
+use eth::{AwsConfig, BlobEncoder, KmsKeys};
+use fuel_block_committer_encoding::bundle;
 use metrics::{
     prometheus::{IntGauge, Registry},
     HealthChecker, RegistersMetrics,
@@ -73,8 +74,8 @@ pub fn block_bundler(
     registry: &Registry,
 ) -> tokio::task::JoinHandle<()> {
     let bundler_factory = services::BundlerFactory::new(
-        Eip4844BlobEncoder,
-        config.app.bundle.compression_level,
+        BlobEncoder,
+        bundle::Encoder::new(config.app.bundle.compression_level),
         config.app.bundle.optimization_step,
     );
 

packages/encoding/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "fuel-block-committer-encoding"
+version = { workspace = true }
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+publish = true
+
+[dependencies]
+# alloy only really needed when generating proofs, and since the core doesn't
+# need to do that we've gated it behing the `kzg` feature flag.
+alloy = { workspace = true, features = ["consensus", "eips"], optional = true }
+anyhow = { workspace = true }
+bitvec = { workspace = true, features = ["default"] }
+c-kzg = { workspace = true }
+flate2 = { workspace = true, features = ["default"] }
+hex = { workspace = true }
+itertools = { workspace = true, features = ["use_std"] }
+postcard = { workspace = true, features = ["alloc"] }
+serde = { workspace = true }
+static_assertions = { workspace = true }
+
+[dev-dependencies]
+alloy = { workspace = true, features = ["consensus", "eips", "kzg"] }
+fuel-block-committer-encoding = { workspace = true, features = [
+  "default",
+  "kzg",
+] }
+itertools = { workspace = true, features = ["use_alloc"] }
+proptest = { workspace = true, features = ["default"] }
+rand = { workspace = true, features = ["std", "std_rng", "small_rng"] }
+test-case = { workspace = true }
+
+[features]
+default = []
+kzg = ["alloy/kzg"]

packages/encoding/src/blob.rs

@@ -0,0 +1,51 @@
+mod decoder;
+mod encoder;
+mod header;
+
+pub use decoder::Decoder;
+pub use encoder::Encoder;
+pub use header::*;
+
+use crate::constants::BYTES_PER_BLOB;
+
+pub type Blob = Box<[u8; BYTES_PER_BLOB]>;
+
+#[cfg(feature = "kzg")]
+pub fn generate_sidecar(
+    blobs: impl IntoIterator<Item = Blob>,
+) -> anyhow::Result<alloy::consensus::BlobTransactionSidecar> {
+    let blobs = blobs
+        .into_iter()
+        .map(|blob| alloy::eips::eip4844::Blob::from(*blob))
+        .collect::<Vec<_>>();
+    let mut commitments = Vec::with_capacity(blobs.len());
+    let mut proofs = Vec::with_capacity(blobs.len());
+    let settings = alloy::consensus::EnvKzgSettings::default();
+
+    for blob in &blobs {
+        // SAFETY: same size
+        let blob =
+            unsafe { core::mem::transmute::<&alloy::eips::eip4844::Blob, &c_kzg::Blob>(blob) };
+        let commitment = c_kzg::KzgCommitment::blob_to_kzg_commitment(blob, settings.get())?;
+        let proof =
+            c_kzg::KzgProof::compute_blob_kzg_proof(blob, &commitment.to_bytes(), settings.get())?;
+
+        // SAFETY: same size
+        unsafe {
+            commitments.push(core::mem::transmute::<
+                c_kzg::Bytes48,
+                alloy::eips::eip4844::Bytes48,
+            >(commitment.to_bytes()));
+            proofs.push(core::mem::transmute::<
+                c_kzg::Bytes48,
+                alloy::eips::eip4844::Bytes48,
+            >(proof.to_bytes()));
+        }
+    }
+
+    Ok(alloy::consensus::BlobTransactionSidecar::new(
+        blobs,
+        commitments,
+        proofs,
+    ))
+}