diff --git a/.dockerignore b/.dockerignore index 81c7945a3..27f5edd07 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,5 @@ **/target/** **/node_modules **/.build/** -**/build/** \ No newline at end of file +**/build/** +rust-toolchain.toml \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 37729ccfe..4b9f26df9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6457,15 +6457,16 @@ dependencies = [ "ethers", "ethers-core 2.0.10", "futures", + "hex", "log", "thiserror", "timeago", "tokio", "url", "walletconnect", - "xmtp", "xmtp_api_grpc", "xmtp_cryptography", + "xmtp_mls", "xmtp_proto", ] diff --git a/Dockerfile b/Dockerfile index c833bb4ab..49164c57f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,17 @@ FROM ghcr.io/xmtp/rust:latest -USER xmtp -RUN ~xmtp/.cargo/bin/rustup toolchain install stable -RUN ~xmtp/.cargo/bin/rustup component add rustfmt -RUN ~xmtp/.cargo/bin/rustup component add clippy +RUN sudo apt update && sudo apt install -y pkg-config openssl WORKDIR /workspaces/libxmtp COPY --chown=xmtp:xmtp . . -ENV PATH=~xmtp/.cargo/bin:$PATH -ENV USER=xmtp -RUN ~xmtp/.cargo/bin/cargo check -RUN ~xmtp/.cargo/bin/cargo fmt --check -RUN ~xmtp/.cargo/bin/cargo clippy --all-features --no-deps -RUN ~xmtp/.cargo/bin/cargo clippy --all-features --no-deps --manifest-path xmtp/Cargo.toml +RUN cargo check +RUN cargo fmt --check +RUN cargo clippy --all-features --no-deps +RUN cargo clippy --all-features --no-deps --manifest-path xmtp/Cargo.toml # some tests are setup as integration tests 👀 xmtp_mls -RUN for crate in xmtp xmtp_api_grpc xmtp_api_grpc_gateway xmtp_cryptography xmtp_proto xmtp_v2; do cd ${crate}; ~xmtp/.cargo/bin/cargo test; done +RUN for crate in xmtp_cryptography xmtp_proto xmtp_v2; do cd ${crate}; cargo test; done LABEL org.label-schema.build-date=$BUILD_DATE \ org.label-schema.name="rustdev" \ diff --git a/examples/cli/Cargo.toml b/examples/cli/Cargo.toml index 2242528eb..77b31d96c 100644 --- a/examples/cli/Cargo.toml +++ b/examples/cli/Cargo.toml @@ -9,7 +9,7 @@ name = "cli-client" path = "cli-client.rs" [dependencies] -clap = {version = "4.3.0", features=["derive"]} +clap = { version = "4.4.6", features = ["derive"] } ethers = "2.0.4" ethers-core = "2.0.4" env_logger = "0.10.0" @@ -18,10 +18,13 @@ log = "0.4.17" thiserror = "1.0.40" tokio = "1.28.1" url = "2.3.1" -walletconnect = {git="https://github.com/nlordell/walletconnect-rs.git", features=["qr", "transport"]} -xmtp = { path = "../../xmtp", features = ["grpc", "native"] } -xmtp_cryptography = { path = "../../xmtp_cryptography"} +walletconnect = { git = "https://github.com/nlordell/walletconnect-rs.git", features = [ + "qr", + "transport", +] } +xmtp_mls = { path = "../../xmtp_mls", features = ["grpc", "native"] } +xmtp_cryptography = { path = "../../xmtp_cryptography" } xmtp_api_grpc = { path = "../../xmtp_api_grpc" } xmtp_proto = { path = "../../xmtp_proto", features = ["proto_full", "grpc"] } timeago = "0.4.1" - +hex = "0.4.3" diff --git a/examples/cli/README.md b/examples/cli/README.md index 89369634b..b7c5ff42b 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -2,18 +2,18 @@ ![Status](https://img.shields.io/badge/Project_status-Alpha-orange) -This is a demo XMTP v3-alpha console client (CLI) that you can use to send and receive messages via the command line. Specifically, you can use it to try out [double ratchet messaging](https://github.com/xmtp/libxmtp/blob/main/README.md#double-ratchet-messaging) and [installation key bundles](https://github.com/xmtp/libxmtp/blob/main/README.md#installation-key-bundles) enabled by XMTP v3-alpha. +This is a demo XMTP MLS console client (CLI) that you can use to send and receive messages via the command line. > **Important** > This software is in **alpha** status and ready for you to start experimenting with. Expect frequent changes as we add features and iterate based on feedback. -## Send a double ratchet message +## Create a MLS group Use the CLI to send a [double ratchet message](https://github.com/xmtp/libxmtp/blob/main/README.md#double-ratchet-messaging) between test wallets on the XMTP `dev` network. 1. Go to the `examples/cli` directory. -2. Create a sender wallet account (user1). Create an [installation key bundle](https://github.com/xmtp/libxmtp/blob/main/README.md#installation-key-bundles) and store it in the database. Grant the installation key bundle permission to message on behalf of the sender address. This will allow the CLI to message on behalf of the sender address. +2. Create a sender wallet account (user1). Create an [XMTP identity](../../xmtp_mls/IDENTITY.md) and store it in the database. Grant the installation key bundle permission to message on behalf of the sender address. This will allow the CLI to message on behalf of the sender address. ```bash ./xli.sh --db user1.db3 register @@ -31,16 +31,27 @@ Use the CLI to send a [double ratchet message](https://github.com/xmtp/libxmtp/b ./xli.sh --db user2.db3 info ``` -5. Send a message into the conversation. The message is sent using [one session between each installation](https://github.com/xmtp/libxmtp/blob/main/README.md#installation-key-bundles) associated with the sender and recipient. The message is encrypted using a per-message encryption key derived using the [double ratchet algorithm](https://github.com/xmtp/libxmtp/blob/main/README.md#double-ratchet-messaging). +5. Create a new group and take note of the group ID. ```bash - ./xli.sh --db user1.db3 send "hello" + ./xli.sh --db user1.db3 create-group ``` -6. List conversations and messages. +6. Add user 2 to the group ```bash - ./xli.sh --db user1.db3 list-conversations + ./xli.sh --db user1.db3 add-group-member $GROUP_ID $USER_2_WALLET_ADDRESS + ``` + +7. Send a message + ```bash + ./xli.sh --db user1.db3 send $GROUP_ID "hello world" + ``` + +8. Have User 2 read the message + + ```bash + ./xli.sh --db user2.db3 list-group-messages $GROUP_ID ``` If you want to run the CLI against localhost, go to the root directory and run `dev/up` to start a local server. Then run the CLI commands using the `--local` flag. diff --git a/examples/cli/alice.db b/examples/cli/alice.db new file mode 100644 index 000000000..41265f2e7 Binary files /dev/null and b/examples/cli/alice.db differ diff --git a/examples/cli/cli-client.rs b/examples/cli/cli-client.rs index 15557f0be..31033702b 100644 --- a/examples/cli/cli-client.rs +++ b/examples/cli/cli-client.rs @@ -4,31 +4,29 @@ XLI is a Commandline client using XMTPv3. extern crate ethers; extern crate log; -extern crate xmtp; +extern crate xmtp_mls; use std::{fs, path::PathBuf, time::Duration}; use clap::{Parser, Subcommand}; use log::{error, info}; use thiserror::Error; -use xmtp::{ - builder::{AccountStrategy, ClientBuilderError}, - client::ClientError, - conversation::{Conversation, ConversationError, ListMessagesOptions}, - conversations::Conversations, - storage::{ - now, EncryptedMessageStore, EncryptionKey, MessageState, StorageError, StorageOption, - }, - InboxOwner, -}; use xmtp_api_grpc::grpc_api_helper::Client as ApiClient; use xmtp_cryptography::{ signature::{RecoverableSignature, SignatureError}, utils::{rng, seeded_rng, LocalWallet}, }; -use xmtp_proto::api_client::XmtpApiClient; -type Client = xmtp::client::Client; -type ClientBuilder = xmtp::builder::ClientBuilder; +use xmtp_mls::{ + builder::{ClientBuilderError, IdentityStrategy}, + client::ClientError, + groups::MlsGroup, + storage::{EncryptedMessageStore, EncryptionKey, StorageError, StorageOption}, + utils::time::now_ns, + InboxOwner, Network, +}; +use xmtp_proto::api_client::{XmtpApiClient, XmtpMlsClient}; +type Client = xmtp_mls::client::Client; +type ClientBuilder = xmtp_mls::builder::ClientBuilder; /// A fictional versioning CLI #[derive(Debug, Parser)] // requires `derive` feature @@ -40,7 +38,7 @@ struct Cli { /// Sets a custom config file #[arg(long, value_name = "FILE", global = true)] db: Option, - #[clap(long, default_value_t = false)] + #[clap(long, default_value_t = true)] local: bool, } @@ -51,19 +49,28 @@ enum Commands { #[clap(long = "seed", default_value_t = 0)] wallet_seed: u64, }, + CreateGroup {}, // List conversations on the registered wallet - ListConversations {}, - /// Information about the account that owns the DB - Info {}, + ListGroups {}, /// Send Message Send { - #[arg(value_name = "ADDR")] - addr: String, + #[arg(value_name = "Group ID")] + group_id: String, #[arg(value_name = "Message")] msg: String, }, - Recv {}, - ListContacts {}, + ListGroupMessages { + #[arg(value_name = "Group ID")] + group_id: String, + }, + AddGroupMember { + #[arg(value_name = "Group ID")] + group_id: String, + #[arg(value_name = "Wallet Address")] + wallet_address: String, + }, + /// Information about the account that owns the DB + Info {}, Clear {}, } @@ -71,14 +78,12 @@ enum Commands { enum CliError { #[error("signature failed to generate")] Signature(#[from] SignatureError), - #[error("stored error occured")] - MessageStore(#[from] StorageError), #[error("client error")] ClientError(#[from] ClientError), #[error("clientbuilder error")] ClientBuilder(#[from] ClientBuilderError), - #[error("ConversationError: {0}")] - ConversationError(#[from] ConversationError), + #[error("storage error")] + StorageError(#[from] StorageError), #[error("generic:{0}")] Generic(String), } @@ -135,77 +140,126 @@ async fn main() { } Commands::Info {} => { info!("Info"); - let client = create_client(&cli, AccountStrategy::CachedOnly("nil".into())) + let client = create_client(&cli, IdentityStrategy::CachedOnly) .await .unwrap(); - info!("Address is: {}", client.wallet_address()); - info!("Installation_id: {}", client.installation_id()); + info!("Address is: {}", client.account_address()); + info!( + "Installation_id: {}", + hex::encode(client.installation_public_key()) + ); } - Commands::ListConversations {} => { - info!("List Conversations"); - let client = create_client(&cli, AccountStrategy::CachedOnly("nil".into())) + Commands::ListGroups {} => { + info!("List Groups"); + let client = create_client(&cli, IdentityStrategy::CachedOnly) .await .unwrap(); - recv(&client).await.unwrap(); - let convo_list = Conversations::list(&client, true).await.unwrap(); + client + .sync_welcomes() + .await + .expect("failed to sync welcomes"); + + // recv(&client).await.unwrap(); + let group_list = client + .find_groups(None, None, None, None) + .expect("failed to list groups"); - for (index, convo) in convo_list.iter().enumerate() { + for group in group_list.iter() { + group.sync().await.expect("error syncing group"); info!( - "====== [{}] Convo with {} ======{}{}", - index, - convo.peer_address(), - "\n", - format_messages(convo).await.unwrap() + "\n====== Group {} ======\n====================== Members ======================\n{}", + hex::encode(group.group_id.clone()), + group + .members() + .unwrap() + .into_iter() + .map(|m| m.wallet_address) + .collect::>() + .join("\n"), ); } } - Commands::Send { addr, msg } => { + Commands::Send { group_id, msg } => { info!("Send"); - let client = create_client(&cli, AccountStrategy::CachedOnly("nil".into())) + let client = create_client(&cli, IdentityStrategy::CachedOnly) .await .unwrap(); - info!("Address is: {}", client.wallet_address()); - send(client, addr, msg).await.unwrap(); + info!("Address is: {}", client.account_address()); + let group = get_group(&client, hex::decode(group_id).expect("group id decode")) + .await + .expect("failed to get group"); + send(group, msg.clone()).await.unwrap(); } - Commands::Recv {} => { + Commands::ListGroupMessages { group_id } => { info!("Recv"); - let client = create_client(&cli, AccountStrategy::CachedOnly("nil".into())) + let client = create_client(&cli, IdentityStrategy::CachedOnly) .await .unwrap(); - info!("Address is: {}", client.wallet_address()); - recv(&client).await.unwrap(); + + let group = get_group(&client, hex::decode(group_id).expect("group id decode")) + .await + .expect("failed to get group"); + + let messages = + format_messages(&group, client.account_address()).expect("failed to get messages"); + info!( + "====== Group {} ======\n{}", + hex::encode(group.group_id), + messages + ) } - Commands::ListContacts {} => { - let client = create_client(&cli, AccountStrategy::CachedOnly("nil".into())) + Commands::AddGroupMember { + group_id, + wallet_address, + } => { + let client = create_client(&cli, IdentityStrategy::CachedOnly) .await .unwrap(); - let contacts = client.get_contacts(&client.wallet_address()).await.unwrap(); - for (index, contact) in contacts.iter().enumerate() { - info!(" [{}] Contact: {:?}", index, contact.installation_id()); - } + let group = get_group(&client, hex::decode(group_id).expect("group id decode")) + .await + .expect("failed to get group"); + + group + .add_members(vec![wallet_address.clone()]) + .await + .expect("failed to add member"); + + info!( + "Successfully added {} to group {}", + wallet_address, group_id + ); } + Commands::CreateGroup {} => { + let client = create_client(&cli, IdentityStrategy::CachedOnly) + .await + .unwrap(); + + let group = client.create_group().expect("failed to create group"); + info!("Created group {}", hex::encode(group.group_id)) + } + Commands::Clear {} => { fs::remove_file(cli.db.unwrap()).unwrap(); } } } -async fn create_client(cli: &Cli, account: AccountStrategy) -> Result { +async fn create_client(cli: &Cli, account: IdentityStrategy) -> Result { let msg_store = get_encrypted_store(&cli.db).unwrap(); let mut builder = ClientBuilder::new(account).store(msg_store); if cli.local { builder = builder - .network(xmtp::Network::Local("http://localhost:5556")) + .network(Network::Local("http://localhost:5556")) .api_client( ApiClient::create("http://localhost:5556".into(), false) .await .unwrap(), ); } else { - builder = builder.network(xmtp::Network::Dev).api_client( + builder = builder.network(Network::Dev).api_client( ApiClient::create("https://dev.xmtp.network:5556".into(), true) .await .unwrap(), @@ -222,10 +276,10 @@ async fn register(cli: &Cli, wallet_seed: &u64) -> Result<(), CliError> { Wallet::LocalWallet(LocalWallet::new(&mut seeded_rng(*wallet_seed))) }; - let mut client = create_client(cli, AccountStrategy::CreateIfNotFound(w)).await?; - info!("Address is: {}", client.wallet_address()); + let client = create_client(cli, IdentityStrategy::CreateIfNotFound(w)).await?; + info!("Address is: {}", client.account_address()); - if let Err(e) = client.init().await { + if let Err(e) = client.register_identity().await { error!("Initialization Failed: {}", e.to_string()); panic!("Could not init"); }; @@ -233,39 +287,49 @@ async fn register(cli: &Cli, wallet_seed: &u64) -> Result<(), CliError> { Ok(()) } -async fn send(client: Client, addr: &str, msg: &str) -> Result<(), CliError> { - let conversation = Conversation::new(&client, addr.to_string()).unwrap(); - conversation.send_text(msg).await.unwrap(); - info!("Message successfully sent"); +async fn get_group(client: &Client, group_id: Vec) -> Result, CliError> { + client.sync_welcomes().await?; + let group = client.group(group_id)?; + group + .sync() + .await + .map_err(|_| CliError::Generic("failed to sync group".to_string()))?; - Ok(()) + Ok(group) } -async fn recv(client: &Client) -> Result<(), CliError> { - Conversations::receive(client)?; +async fn send<'c>(group: MlsGroup<'c, ApiClient>, msg: String) -> Result<(), CliError> { + group + .send_message(msg.into_bytes().as_slice()) + .await + .unwrap(); + info!( + "Message successfully sent to group {}", + hex::encode(group.group_id) + ); + Ok(()) } -async fn format_messages<'c, A: XmtpApiClient>( - convo: &Conversation<'c, A>, +fn format_messages<'c, A: XmtpApiClient + XmtpMlsClient>( + convo: &MlsGroup<'c, A>, + my_wallet_address: String, ) -> Result { let mut output: Vec = vec![]; - let opts = ListMessagesOptions::default(); - for msg in convo.list_messages(&opts).await? { - let contents = msg.get_text().map_err(|e| e.to_string())?; - let is_inbound = msg.state == MessageState::Received as i32; - let direction = if is_inbound { - String::from(" -------->") + for msg in convo.find_messages(None, None, None, None).unwrap() { + let contents = msg.decrypted_message_bytes; + let sender = if msg.sender_wallet_address == my_wallet_address { + "Me".to_string() } else { - String::from("<-------- ") + msg.sender_wallet_address }; let msg_line = format!( - "[{:>15} ] {} {}", - pretty_delta(now() as u64, msg.created_at as u64), - direction, - contents + "[{:>15} ] {}: {}", + pretty_delta(now_ns() as u64, msg.sent_at_ns as u64), + sender, + String::from_utf8(contents).unwrap() ); output.push(msg_line); } @@ -297,5 +361,6 @@ fn get_encrypted_store(db: &Option) -> Result String { let f = timeago::Formatter::new(); - f.convert(Duration::from_nanos(now - then)) + let diff = if now > then { now - then } else { then - now }; + f.convert(Duration::from_nanos(diff)) } diff --git a/xmtp_mls/src/client.rs b/xmtp_mls/src/client.rs index 6a379fd9e..e0a21915d 100644 --- a/xmtp_mls/src/client.rs +++ b/xmtp_mls/src/client.rs @@ -13,11 +13,15 @@ use crate::{ api_client_wrapper::{ApiClientWrapper, IdentityUpdate}, groups::MlsGroup, identity::Identity, - storage::{group::GroupMembershipState, DbConnection, EncryptedMessageStore, StorageError}, + storage::{ + group::{GroupMembershipState, StoredGroup}, + DbConnection, EncryptedMessageStore, StorageError, + }, types::Address, utils::topic::get_welcome_topic, verified_key_package::{KeyPackageVerificationError, VerifiedKeyPackage}, xmtp_openmls_provider::XmtpOpenMlsProvider, + Fetch, }; #[derive(Clone, Copy, Default, Debug)] @@ -100,6 +104,15 @@ where Ok(group) } + pub fn group(&self, group_id: Vec) -> Result, ClientError> { + let conn = &mut self.store.conn()?; + let stored_group: Option = conn.fetch(&group_id)?; + match stored_group { + Some(group) => Ok(MlsGroup::new(self, group.id, group.created_at_ns)), + None => Err(ClientError::Generic("group not found".to_string())), + } + } + pub fn find_groups( &self, allowed_states: Option>, diff --git a/xmtp_mls/src/groups/mod.rs b/xmtp_mls/src/groups/mod.rs index 071ab6543..485128bc7 100644 --- a/xmtp_mls/src/groups/mod.rs +++ b/xmtp_mls/src/groups/mod.rs @@ -435,6 +435,20 @@ where Ok(()) } + pub async fn add_members(&self, wallet_addresses: Vec) -> Result<(), GroupError> { + let conn = &mut self.client.store.conn()?; + let key_packages = self + .client + .get_key_packages_for_wallet_addresses(wallet_addresses) + .await?; + let intent_data: Vec = AddMembersIntentData::new(key_packages).try_into()?; + let intent = + NewGroupIntent::new(IntentKind::AddMembers, self.group_id.clone(), intent_data); + intent.store(conn)?; + + self.sync_with_conn(conn).await + } + pub async fn add_members_by_installation_id( &self, installation_ids: Vec>, @@ -449,7 +463,7 @@ where NewGroupIntent::new(IntentKind::AddMembers, self.group_id.clone(), intent_data); intent.store(conn)?; - self.sync(conn).await + self.sync_with_conn(conn).await } pub async fn remove_members_by_installation_id( @@ -465,7 +479,7 @@ where ); intent.store(conn)?; - self.sync(conn).await + self.sync_with_conn(conn).await } pub async fn key_update(&self) -> Result<(), GroupError> { @@ -473,10 +487,15 @@ where let intent = NewGroupIntent::new(IntentKind::KeyUpdate, self.group_id.clone(), vec![]); intent.store(conn)?; - self.sync(conn).await + self.sync_with_conn(conn).await + } + + pub async fn sync(&self) -> Result<(), GroupError> { + let conn = &mut self.client.store.conn()?; + self.sync_with_conn(conn).await } - pub async fn sync(&self, conn: &mut DbConnection) -> Result<(), GroupError> { + async fn sync_with_conn(&self, conn: &mut DbConnection) -> Result<(), GroupError> { self.publish_intents(conn).await?; if let Err(e) = self.receive().await { log::warn!("receive error {:?}", e);