Merge branch 'main' into coda/pref-streaming

.github/workflows/lint-workspace.yaml

@@ -14,6 +14,7 @@ on:
       - "xmtp_mls/**"
       - "xmtp_proto/**"
       - "xmtp_v2/**"
+      - "xmtp_debug/**"
       - "Cargo.toml"
       - "Cargo.lock"
       - "rust-toolchain"

.github/workflows/push-xbg.yml

@@ -0,0 +1,49 @@
+name: Push XDBG Image
+
+on:
+  push:
+    branches:
+      - main
+
+  workflow_dispatch:
+
+jobs:
+  push_to_registry:
+    name: Push Docker Image to GitHub Packages
+    runs-on: warp-ubuntu-latest-x64-16x
+    permissions:
+      contents: read
+      packages: write
+    outputs:
+      digest: ${{ steps.push.outputs.digest }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Log in to the container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository_owner }}/xdbg
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=tag
+            type=ref,event=pr
+            type=sha
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        id: push
+        with:
+          context: .
+          file: ./dev/xdbg/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

dev/docker/docker-compose.yml

@@ -69,13 +69,13 @@ services:
     # note: the SHA here is tied to the XTMPD_CONTRACTS_*_ADDRESSes
     # if you bump the version of anvil-xmtpd you will have to change the contracts
     # you can find them inside the anvil-xmtpd image via `docker exec libxmtp-chain-1 cat contracts.env`
-    image: ghcr.io/xmtp/anvil-xmtpd:sha-0b3421b
+    image: ghcr.io/xmtp/anvil-xmtpd:sha-10808fb
     command: ["--host", "0.0.0.0"]
 
   repnode:
     platform: linux/amd64
     # note: avoid using :latest while xmtpd is under development to avoid breaking changes
-    image: ghcr.io/xmtp/xmtpd:sha-0b3421b
+    image: ghcr.io/xmtp/xmtpd:sha-10808fb
     environment:
       XMTPD_DB_WRITER_CONNECTION_STRING: "postgres://postgres:xmtp@replicationdb:5432/postgres?sslmode=disable"
       XMTPD_CONTRACTS_RPC_URL: "http://chain:8545"
@@ -84,9 +84,11 @@ services:
       XMTPD_CONTRACTS_IDENTITY_UPDATES_ADDRESS: 0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0
       XMTPD_SIGNER_PRIVATE_KEY: 0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d
       XMTPD_PAYER_PRIVATE_KEY: 0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d
-      XMTPD_MLS_VALIDATION_GRPC_ADDRESS: "validation:50051"
+      XMTPD_MLS_VALIDATION_GRPC_ADDRESS: "http://validation:50051"
       XMTPD_PAYER_ENABLE: true
       XMTPD_REPLICATION_ENABLE: true
+      XMTPD_INDEXER_ENABLE: true
+      XMTPD_SYNC_ENABLE: true
     depends_on:
       chain:
         condition: service_started

dev/xdbg/Dockerfile

@@ -0,0 +1,10 @@
+FROM rust:1-bullseye as builder
+WORKDIR /code
+COPY . .
+COPY .git /.git
+RUN cargo build --release --package xdbg
+
+FROM debian:bullseye-slim
+COPY --from=builder /code/target/release/xdbg /usr/local/bin/xdbg
+ENV RUST_LOG=info
+ENTRYPOINT ["xdbg"]

xmtp_debug/src/app.rs

@@ -21,6 +21,7 @@ mod store;
 /// Types shared between App Functions
 mod types;
 
+use clap::CommandFactory;
 use color_eyre::eyre::{self, Result};
 use directories::ProjectDirs;
 use std::{fs, path::PathBuf, sync::Arc};
@@ -95,6 +96,11 @@ impl App {
         } = opts;
         debug!(fdlimit = get_fdlimit());
 
+        if cmd.is_none() && !clear {
+            AppOpts::command().print_help()?;
+            eyre::bail!("No subcommand was specified");
+        }
+
         if let Some(cmd) = cmd {
             match cmd {
                 Generate(g) => generate::Generate::new(g, backend, db).run().await,

xmtp_debug/src/app/clients.rs

@@ -57,10 +57,7 @@ async fn new_client_inner(
     wallet: &LocalWallet,
     db_path: Option<PathBuf>,
 ) -> Result<crate::DbgClient> {
-    let url = url::Url::from(network.clone());
-    let is_secure = url.scheme() == "https";
-    trace!(url = %url, is_secure, "create grpc");
-    let api = crate::GrpcClient::create(url.as_str().to_string(), is_secure).await?;
+    let api = network.connect().await?;
 
     let nonce = 1;
     let inbox_id = generate_inbox_id(&wallet.get_address(), &nonce).unwrap();
@@ -125,9 +122,7 @@ async fn existing_client_inner(
     network: &args::BackendOpts,
     db_path: PathBuf,
 ) -> Result<crate::DbgClient> {
-    let url = url::Url::from(network.clone());
-    let is_secure = url.scheme() == "https";
-    let api = crate::GrpcClient::create(url.as_str().to_string(), is_secure).await?;
+    let api = network.connect().await?;
 
     let store = EncryptedMessageStore::new(
         StorageOption::Persistent(db_path.clone().into_os_string().into_string().unwrap()),

xmtp_debug/src/args.rs

@@ -3,6 +3,7 @@ use std::path::PathBuf;
 
 use clap::{Args, Parser, Subcommand, ValueEnum};
 use clap_verbosity_flag::{InfoLevel, Verbosity};
+use color_eyre::eyre;
 use xxhash_rust::xxh3;
 mod types;
 pub use types::*;
@@ -56,7 +57,7 @@ pub struct Generate {
     #[arg(long, short)]
     pub amount: usize,
     /// Specify amount of random identities to invite to group
-    #[arg(long, short)]
+    #[arg(long)]
     pub invite: Option<usize>,
     #[command(flatten)]
     pub message_opts: MessageGenerateOpts,
@@ -193,6 +194,76 @@ pub struct BackendOpts {
         conflicts_with = "constant-backend"
     )]
     pub url: Option<url::Url>,
+    #[arg(
+        short,
+        long,
+        group = "custom-backend",
+        conflicts_with = "constant-backend"
+    )]
+    pub payer_url: Option<url::Url>,
+    /// Enable the decentralization backend
+    #[arg(short, long)]
+    pub d14n: bool,
+}
+
+impl BackendOpts {
+    pub fn payer_url(&self) -> eyre::Result<url::Url> {
+        use BackendKind::*;
+
+        if let Some(p) = &self.payer_url {
+            return Ok(p.clone());
+        }
+
+        match (self.backend, self.d14n) {
+            (Dev, false) => eyre::bail!("No payer for V3"),
+            (Production, false) => eyre::bail!("No payer for V3"),
+            (Local, false) => eyre::bail!("No payer for V3"),
+            (Dev, true) => Ok((*crate::constants::XMTP_DEV_PAYER).clone()),
+            (Production, true) => Ok((*crate::constants::XMTP_PRODUCTION_PAYER).clone()),
+            (Local, true) => Ok((*crate::constants::XMTP_LOCAL_PAYER).clone()),
+        }
+    }
+
+    pub fn network_url(&self) -> url::Url {
+        use BackendKind::*;
+
+        if let Some(n) = &self.url {
+            return n.clone();
+        }
+
+        match (self.backend, self.d14n) {
+            (Dev, false) => (*crate::constants::XMTP_DEV).clone(),
+            (Production, false) => (*crate::constants::XMTP_PRODUCTION).clone(),
+            (Local, false) => (*crate::constants::XMTP_LOCAL).clone(),
+            (Dev, true) => (*crate::constants::XMTP_DEV_D14N).clone(),
+            (Production, true) => (*crate::constants::XMTP_PRODUCTION_D14N).clone(),
+            (Local, true) => (*crate::constants::XMTP_LOCAL_D14N).clone(),
+        }
+    }
+
+    pub async fn connect(&self) -> eyre::Result<Box<dyn xmtp_mls::XmtpApi>> {
+        let network = self.network_url();
+        let is_secure = network.scheme() == "https";
+
+        if self.d14n {
+            let payer = self.payer_url()?;
+            trace!(url = %network, payer = %payer, is_secure, "create grpc");
+
+            Ok(Box::new(
+                xmtp_api_grpc::replication_client::ClientV4::create(
+                    network.as_str().to_string(),
+                    payer.as_str().to_string(),
+                    is_secure,
+                )
+                .await?,
+            ))
+        } else {
+            trace!(url = %network, is_secure, "create grpc");
+            Ok(Box::new(
+                crate::GrpcClient::create(network.as_str().to_string(), is_secure).await?,
+            ))
+        }
+    }
 }
 
 impl<'a> From<&'a BackendOpts> for u64 {
@@ -202,10 +273,13 @@ impl<'a> From<&'a BackendOpts> for u64 {
         if let Some(ref url) = value.url {
             xxh3::xxh3_64(url.as_str().as_bytes())
         } else {
-            match value.backend {
-                Production => 2,
-                Dev => 1,
-                Local => 0,
+            match (value.backend, value.d14n) {
+                (Production, false) => 2,
+                (Dev, false) => 1,
+                (Local, false) => 0,
+                (Production, true) => 5,
+                (Dev, true) => 4,
+                (Local, true) => 3,
             }
         }
     }
@@ -219,8 +293,10 @@ impl From<BackendOpts> for u64 {
 
 impl From<BackendOpts> for url::Url {
     fn from(value: BackendOpts) -> Self {
-        let BackendOpts { backend, url } = value;
-        url.unwrap_or(backend.into())
+        let BackendOpts {
+            backend, url, d14n, ..
+        } = value;
+        url.unwrap_or(backend.to_network_url(d14n))
     }
 }
 
@@ -232,6 +308,20 @@ pub enum BackendKind {
     Local,
 }
 
+impl BackendKind {
+    fn to_network_url(self, d14n: bool) -> url::Url {
+        use BackendKind::*;
+        match (self, d14n) {
+            (Dev, false) => (*crate::constants::XMTP_DEV).clone(),
+            (Production, false) => (*crate::constants::XMTP_PRODUCTION).clone(),
+            (Local, false) => (*crate::constants::XMTP_LOCAL).clone(),
+            (Dev, true) => (*crate::constants::XMTP_DEV_D14N).clone(),
+            (Production, true) => (*crate::constants::XMTP_PRODUCTION_D14N).clone(),
+            (Local, true) => (*crate::constants::XMTP_LOCAL_D14N).clone(),
+        }
+    }
+}
+
 impl From<BackendKind> for url::Url {
     fn from(value: BackendKind) -> Self {
         use BackendKind::*;

xmtp_debug/src/constants.rs

@@ -9,5 +9,18 @@ pub static XMTP_DEV: LazyLock<Url> =
     LazyLock::new(|| Url::parse("https://grpc.dev.xmtp.network:443").unwrap());
 pub static XMTP_LOCAL: LazyLock<Url> =
     LazyLock::new(|| Url::parse("http://localhost:5556").unwrap());
+
+pub static XMTP_PRODUCTION_D14N: LazyLock<Url> = LazyLock::new(|| Url::parse("").unwrap());
+pub static XMTP_DEV_D14N: LazyLock<Url> =
+    LazyLock::new(|| Url::parse("https://grpc.testnet.xmtp.network:443").unwrap());
+pub static XMTP_LOCAL_D14N: LazyLock<Url> =
+    LazyLock::new(|| Url::parse("http://localhost:5050").unwrap());
+
+pub static XMTP_PRODUCTION_PAYER: LazyLock<Url> = LazyLock::new(|| Url::parse("").unwrap());
+pub static XMTP_DEV_PAYER: LazyLock<Url> =
+    LazyLock::new(|| Url::parse("https://payer.testnet.xmtp.network:443").unwrap());
+pub static XMTP_LOCAL_PAYER: LazyLock<Url> =
+    LazyLock::new(|| Url::parse("http://localhost:5050").unwrap());
+
 pub static TMPDIR: LazyLock<TempDir> = LazyLock::<TempDir>::new(|| TempDir::new().unwrap());
 pub const STORAGE_PREFIX: &str = "xdbg";

xmtp_debug/src/main.rs

@@ -7,8 +7,10 @@ use clap::Parser;
 use color_eyre::eyre::Result;
 
 use xmtp_api_grpc::grpc_api_helper::Client as GrpcClient;
+use xmtp_mls::XmtpApi;
 
-pub type DbgClient = xmtp_mls::client::Client<GrpcClient>;
+// pub type DbgClient = xmtp_mls::client::Client<GrpcClient>;
+type DbgClient = xmtp_mls::client::Client<Box<dyn XmtpApi>>;
 
 #[macro_use]
 extern crate tracing;