From 6e431f4eae2cfe0f74fe09fa026802d8803d7817 Mon Sep 17 00:00:00 2001 From: Christian Berkhoff Date: Wed, 23 Oct 2024 18:14:03 -0700 Subject: [PATCH 1/6] Test support for sharding --- ipa-core/src/config.rs | 24 +- ipa-core/src/net/client/mod.rs | 29 +- ipa-core/src/net/test.rs | 497 +++++++++++++++++++++++++++------ ipa-core/src/net/transport.rs | 113 ++++---- 4 files changed, 511 insertions(+), 152 deletions(-) diff --git a/ipa-core/src/config.rs b/ipa-core/src/config.rs index 50bd90f4b..30d1b70a8 100644 --- a/ipa-core/src/config.rs +++ b/ipa-core/src/config.rs @@ -532,8 +532,11 @@ mod tests { }; const URI_1: &str = "http://localhost:3000"; + const URI_1S: &str = "http://localhost:6000"; const URI_2: &str = "http://localhost:3001"; + const URI_2S: &str = "http://localhost:6001"; const URI_3: &str = "http://localhost:3002"; + const URI_3S: &str = "http://localhost:6002"; #[test] fn parse_config() { @@ -541,18 +544,27 @@ mod tests { let uri1 = URI_1.parse::().unwrap(); let id1 = HelperIdentity::try_from(1usize).unwrap(); - let value1 = &conf.network.peers()[id1]; - assert_eq!(value1.url, uri1); + let ring_value1 = &conf.leaders_ring().network.peers()[id1]; + assert_eq!(ring_value1.url, uri1); + let uri1s = URI_1S.parse::().unwrap(); + let sharding_value1 = conf.get_shards_for_helper(id1).network.get_peer(0).unwrap(); + assert_eq!(sharding_value1.url, uri1s); let uri2 = URI_2.parse::().unwrap(); let id2 = HelperIdentity::try_from(2usize).unwrap(); - let value2 = &conf.network.peers()[id2]; - assert_eq!(value2.url, uri2); + let ring_value2 = &conf.leaders_ring().network.peers()[id2]; + assert_eq!(ring_value2.url, uri2); + let uri2s = URI_2S.parse::().unwrap(); + let sharding_value2 = conf.get_shards_for_helper(id2).network.get_peer(0).unwrap(); + assert_eq!(sharding_value2.url, uri2s); let uri3 = URI_3.parse::().unwrap(); let id3 = HelperIdentity::try_from(3usize).unwrap(); - let value3 = &conf.network.peers()[id3]; - assert_eq!(value3.url, uri3); + let ring_value3 = &conf.leaders_ring().network.peers()[id3]; + assert_eq!(ring_value3.url, uri3); + let uri3s = URI_3S.parse::().unwrap(); + let sharding_value3 = conf.get_shards_for_helper(id3).network.get_peer(0).unwrap(); + assert_eq!(sharding_value3.url, uri3s); } #[test] diff --git a/ipa-core/src/net/client/mod.rs b/ipa-core/src/net/client/mod.rs index 3693431f2..0b47fb03e 100644 --- a/ipa-core/src/net/client/mod.rs +++ b/ipa-core/src/net/client/mod.rs @@ -25,7 +25,7 @@ use pin_project::pin_project; use rustls::RootCertStore; use tracing::error; -use super::{ConnectionFlavor, Helper}; +use super::{ConnectionFlavor, Helper, Shard}; use crate::{ config::{ ClientConfig, HyperClientConfigurator, NetworkConfig, OwnedCertificate, OwnedPrivateKey, @@ -469,6 +469,33 @@ impl MpcHelperClient { } } +impl MpcHelperClient { + /// This is a mirror of [`MpcHelperClient::from_config`] but for Shards. This creates + /// set of Shard clients in the supplied helper network configuration, which can be used to + /// talk to each of the shards in this helper. + /// + /// `identity` configures whether and how the client will authenticate to the server. It is for + /// the shard making the calls, so the same one is used for all three of the clients. + #[must_use] + #[allow(clippy::missing_panics_doc)] + pub fn shards_from_conf( + runtime: &IpaRuntime, + conf: &NetworkConfig, + identity: &ClientIdentity, + ) -> Vec { + conf.peers_iter() + .map(|peer_conf| { + Self::new( + runtime.clone(), + &conf.client, + peer_conf.clone(), + identity.clone_with_key(), + ) + }) + .collect() + } +} + fn make_http_connector() -> HttpConnector { let mut connector = HttpConnector::new(); // IPA uses HTTP2 and it is sensitive to those delays especially in high-latency network diff --git a/ipa-core/src/net/test.rs b/ipa-core/src/net/test.rs index e62bccce6..7ceda7cd3 100644 --- a/ipa-core/src/net/test.rs +++ b/ipa-core/src/net/test.rs @@ -10,33 +10,127 @@ #![allow(clippy::missing_panics_doc)] use std::{ array, + iter::zip, net::{SocketAddr, TcpListener}, }; use once_cell::sync::Lazy; use rustls_pki_types::CertificateDer; -use super::transport::MpcHttpTransport; +use super::{transport::MpcHttpTransport, ConnectionFlavor, Shard}; use crate::{ config::{ ClientConfig, HpkeClientConfig, HpkeServerConfig, NetworkConfig, PeerConfig, ServerConfig, TlsConfig, }, executor::{IpaJoinHandle, IpaRuntime}, - helpers::{HandlerBox, HelperIdentity, RequestHandler}, + helpers::{repeat_n, HandlerBox, HelperIdentity, RequestHandler, TransportIdentity}, hpke::{Deserializable as _, IpaPublicKey}, net::{ClientIdentity, Helper, MpcHelperClient, MpcHelperServer}, + sharding::ShardIndex, sync::Arc, test_fixture::metrics::MetricsHandle, }; -pub const DEFAULT_TEST_PORTS: [u16; 3] = [3000, 3001, 3002]; +/// Simple struct to keep default port configuration organized. +pub struct Ports { + ring: C, + shards: C, +} + +/// A **single** ring with 3 hosts, each with a ring and sharding port. +pub const DEFAULT_TEST_PORTS: Ports<[u16; 3]> = Ports { + ring: [3000, 3001, 3002], + shards: [6000, 6001, 6002], +}; + +/// Configuration of a server that can be reached via socket or port. +pub struct AddressableTestServer { + /// Contains the ports + pub config: ServerConfig, + /// Sockets are created if no port was specified. + pub socket: Option, +} + +/// A bunch of addressable servers. These can be part of a Shard or MPC network. +#[derive(Default)] +pub struct TestServers { + pub configs: Vec, +} + +impl TestServers { + /// Creates a bunch of addressable servers from the given configurations plus the optional + /// ports. Meant to be used during the definition of an MPC ring. + fn new(configs: Vec, sockets: Vec>) -> Self { + assert_eq!(configs.len(), sockets.len()); + let mut new_configs = Vec::with_capacity(configs.len()); + for s in zip(configs, sockets) { + new_configs.push(AddressableTestServer { + config: s.0, + socket: s.1, + }); + } + TestServers { + configs: new_configs, + } + } + + /// Adds a test server. These increments are useful during shard network definition. + fn push_shard(&mut self, config: AddressableTestServer) { + self.configs.push(config); + } + + pub fn iter(&self) -> impl Iterator { + self.configs.iter() + } +} +/// Either a single Ring on MPC connection or all of the shards in a Helper. +pub struct TestNetwork { + pub network: NetworkConfig, // Contains Clients config + pub servers: TestServers, +} + +impl TestNetwork { + #[must_use] + + /// Gets a ref to the first shard in this network. + pub fn get_first_shard(&self) -> &AddressableTestServer { + self.servers.configs.first().unwrap() + } + + /// Gets a mut ref to the first shard in this network. + pub fn get_first_shard_mut(&mut self) -> &mut AddressableTestServer { + self.servers.configs.get_mut(0).unwrap() + } +} + +/// Uber container for test configuration. Provides access to a vec of MPC rings and 3 sharding +/// networks (one for each Helper) pub struct TestConfig { pub disable_https: bool, - pub network: NetworkConfig, - pub servers: [ServerConfig; 3], - pub sockets: Option<[TcpListener; 3]>, + pub rings: Vec>, + pub sharding_networks: Vec>, +} + +impl TestConfig { + /// Gets a ref to the first ring in the network. This ring is important because it's the one + /// that's reached out by the report collector on behalf of all the shards in the helper. + #[must_use] + pub fn leaders_ring(&self) -> &TestNetwork { + &self.rings[0] + } + + /// Gets a ref to the entire shard network for a specific helper. + #[must_use] + pub fn get_shards_for_helper(&self, id: HelperIdentity) -> &TestNetwork { + self.sharding_networks.get(id.as_index()).unwrap() + } + + /// Gets a mut ref to the entire shard network for a specific helper. + pub fn get_shards_for_helper_mut(&mut self, id: HelperIdentity) -> &mut TestNetwork { + self.sharding_networks.get_mut(id.as_index()).unwrap() + } } impl TestConfig { @@ -75,11 +169,11 @@ fn server_config_insecure_http(port: u16, matchkey_encryption: bool) -> ServerCo #[must_use] pub fn server_config_https( - id: HelperIdentity, + cert_index: usize, port: u16, matchkey_encryption: bool, ) -> ServerConfig { - let (certificate, private_key) = get_test_certificate_and_key(id); + let (certificate, private_key) = get_test_certificate_and_key(cert_index); ServerConfig { port: Some(port), disable_https: false, @@ -91,30 +185,40 @@ pub fn server_config_https( } } -#[derive(Default)] pub struct TestConfigBuilder { - ports: Option<[u16; 3]>, + /// Can be empty meaning that free ports should be obtained from the operating system. + /// One ring per shard in a helper (sharding factor). For each ring we need 3 shard and + /// 3 mpc ports. + ports_by_ring: Vec>>, + /// Describes the number of shards per helper + sharding_factor: usize, disable_https: bool, use_http1: bool, disable_matchkey_encryption: bool, } -impl TestConfigBuilder { - #[must_use] - pub fn with_http_and_default_test_ports() -> Self { +impl Default for TestConfigBuilder { + fn default() -> Self { Self { - ports: Some(DEFAULT_TEST_PORTS), - disable_https: true, + ports_by_ring: vec![], + sharding_factor: 1, + disable_https: false, use_http1: false, disable_matchkey_encryption: false, } } +} +impl TestConfigBuilder { #[must_use] - pub fn with_open_ports() -> Self { + pub fn with_http_and_default_test_ports() -> Self { Self { - ports: None, - disable_https: false, + ports_by_ring: vec![Ports { + ring: DEFAULT_TEST_PORTS.ring.to_vec(), + shards: DEFAULT_TEST_PORTS.shards.to_vec(), + }], + sharding_factor: 1, + disable_https: true, use_http1: false, disable_matchkey_encryption: false, } @@ -126,6 +230,13 @@ impl TestConfigBuilder { self } + #[must_use] + pub fn with_ports_by_ring(mut self, value: Vec>>) -> Self { + self.sharding_factor = value.len(); + self.ports_by_ring = value; + self + } + #[must_use] pub fn with_use_http1_option(mut self, value: bool) -> Self { self.use_http1 = value; @@ -140,31 +251,75 @@ impl TestConfigBuilder { self } - #[must_use] - pub fn build(self) -> TestConfig { - let mut sockets = None; - let ports = self.ports.unwrap_or_else(|| { - let socks = array::from_fn(|_| TcpListener::bind("localhost:0").unwrap()); - let ports = socks - .each_ref() - .map(|sock| sock.local_addr().unwrap().port()); - sockets = Some(socks); - ports + /// Creates 3 MPC test servers, assigning ports from OS if necessary. + fn build_mpc_servers( + &self, + optional_ports: Option>, + ring_index: usize, + ) -> TestServers { + let mut sockets = vec![None, None, None]; + let ports = optional_ports.unwrap_or_else(|| { + sockets = (0..3) + .map(|_| Some(TcpListener::bind("localhost:0").unwrap())) + .collect(); + sockets + .iter() + .map(|sock| sock.as_ref().unwrap().local_addr().unwrap().port()) + .collect() }); - let (scheme, certs) = if self.disable_https { - ("http", [None, None, None]) + // Ports will always be defined after this. Socks only if there were no ports set. + let configs = if self.disable_https { + ports + .into_iter() + .map(|ports| server_config_insecure_http(ports, !self.disable_matchkey_encryption)) + .collect() } else { - ("https", TEST_CERTS_DER.clone().map(Some)) + let start_idx = ring_index * 3; + (start_idx..start_idx + 3) + .map(|id| server_config_https(id, ports[id], !self.disable_matchkey_encryption)) + .collect() }; - let peers = certs - .into_iter() - .enumerate() - .map(|(i, cert)| PeerConfig { - url: format!("{scheme}://localhost:{}", ports[i]) - .parse() - .unwrap(), - certificate: cert, - hpke_config: if self.disable_matchkey_encryption { + TestServers::new(configs, sockets) + } + + /// Defines a test network with a single MPC ring (3 helpers). + fn build_ring_network( + &self, + servers: TestServers, + scheme: &str, + certs: Vec>>, + ) -> TestNetwork { + let (peers, client_config) = self.build_network(&servers, scheme, certs); + let network = NetworkConfig::::new_mpc(peers, client_config); + TestNetwork { network, servers } + } + + /// Defines a test network with the N shards on a helper. + fn build_shard_network( + &self, + servers: TestServers, + scheme: &str, + certs: Vec>>, + ) -> TestNetwork { + let (peers, client_config) = self.build_network(&servers, scheme, certs); + let network = NetworkConfig::::new_shards(peers, client_config); + TestNetwork { network, servers } + } + + fn build_network( + &self, + servers: &TestServers, + scheme: &str, + certs: Vec>>, + ) -> (Vec, ClientConfig) { + let peers = zip(servers.configs.iter(), certs) + .map(|(addr_server, certificate)| { + let port = addr_server + .config + .port + .expect("Port should have been defined already"); + let url = format!("{scheme}://localhost:{port}").parse().unwrap(); + let hpke_config = if self.disable_matchkey_encryption { None } else { Some(HpkeClientConfig::new( @@ -173,30 +328,76 @@ impl TestConfigBuilder { ) .unwrap(), )) - }, + }; + PeerConfig { + url, + certificate, + hpke_config, + } }) - .collect::>(); - let network = NetworkConfig::::new_mpc( - peers, - self.use_http1 - .then(ClientConfig::use_http1) - .unwrap_or_default(), - ); - let servers = if self.disable_https { - ports.map(|ports| server_config_insecure_http(ports, !self.disable_matchkey_encryption)) - } else { - HelperIdentity::make_three() - .map(|id| server_config_https(id, ports[id], !self.disable_matchkey_encryption)) - }; + .collect(); + + let client_config = self + .use_http1 + .then(ClientConfig::use_http1) + .unwrap_or_default(); + (peers, client_config) + } + + /// Creates a test network with shards + #[must_use] + pub fn build(self) -> TestConfig { + // first we build all the rings and then we connect all the shards. + // The following will accumulate the shards as we create the rings. + let mut sharding_networks_shards = vec![ + TestServers::default(), + TestServers::default(), + TestServers::default(), + ]; + let rings: Vec> = (0..self.sharding_factor) + .map(|s| { + let ring_ports = self.ports_by_ring.get(s).map(|p| p.ring.clone()); + let shards_ports = self.ports_by_ring.get(s).map(|p| p.shards.clone()); + + // We create the servers in the MPC ring and connect them (in a TestNetwork). + let ring_servers = self.build_mpc_servers(ring_ports, s); + let (scheme, certs) = if self.disable_https { + ("http", [None, None, None]) + } else { + ("https", get_certs_der_row(s)) + }; + let ring_network = self.build_ring_network(ring_servers, scheme, certs.to_vec()); + + // We create the sharding servers and accumulate them but don't connect them yet + let shard_servers = self.build_mpc_servers(shards_ports, s); + for (i, s) in shard_servers.configs.into_iter().enumerate() { + sharding_networks_shards[i].push_shard(s); + } + + ring_network + }) + .collect(); + + let sharding_networks = sharding_networks_shards + .into_iter() + .enumerate() + .map(|(i, s)| { + let (scheme, certs) = if self.disable_https { + ("http", repeat_n(None, self.sharding_factor).collect()) + } else { + ("https", get_certs_der_col(i).to_vec()) + }; + self.build_shard_network(s, scheme, certs) + }) + .collect(); + TestConfig { - network, - servers, - sockets, disable_https: self.disable_https, + rings, + sharding_networks, } } } - pub struct TestServer { pub addr: SocketAddr, pub handle: IpaJoinHandle<()>, @@ -270,42 +471,31 @@ impl TestServerBuilder { } pub async fn build(self) -> TestServer { - let identity = if self.disable_https { - ClientIdentity::Header(HelperIdentity::ONE) - } else { - get_test_identity(HelperIdentity::ONE) - }; - let test_config = TestConfig::builder() + let identities = create_ids(self.disable_https, HelperIdentity::ONE, ShardIndex::FIRST); + let mut test_config = TestConfig::builder() .with_disable_https_option(self.disable_https) .with_use_http1_option(self.use_http1) // TODO: add disble_matchkey here .build(); - let TestConfig { - network: network_config, - servers: [server_config, _, _], - sockets: Some([server_socket, _, _]), - .. - } = test_config - else { - panic!("TestConfig should have allocated ports"); - }; + let leaders_ring = test_config.rings.pop().unwrap(); + let first_server = leaders_ring.servers.configs.into_iter().next().unwrap(); let clients = MpcHelperClient::from_conf( &IpaRuntime::current(), - &network_config, - &identity.clone_with_key(), + &leaders_ring.network, + &identities.helper.clone_with_key(), ); let handler = self.handler.as_ref().map(HandlerBox::owning_ref); let client = clients[0].clone(); let (transport, server) = MpcHttpTransport::new( IpaRuntime::current(), HelperIdentity::ONE, - server_config, - network_config.clone(), + first_server.config, + leaders_ring.network.clone(), &clients, handler, ); let (addr, handle) = server - .start_on(&IpaRuntime::current(), Some(server_socket), self.metrics) + .start_on(&IpaRuntime::current(), first_server.socket, self.metrics) .await; TestServer { addr, @@ -318,17 +508,40 @@ impl TestServerBuilder { } } -fn get_test_certificate_and_key(id: HelperIdentity) -> (&'static [u8], &'static [u8]) { +pub struct ClientIdentities { + pub helper: ClientIdentity, + pub shard: ClientIdentity, +} + +#[must_use] +pub fn create_ids(disable_https: bool, id: HelperIdentity, ix: ShardIndex) -> ClientIdentities { + if disable_https { + ClientIdentities { + helper: ClientIdentity::Header(id), + shard: ClientIdentity::Header(ix), + } + } else { + get_client_test_identity(id, ix) + } +} + +fn get_test_certificate_and_key(id: usize) -> (&'static [u8], &'static [u8]) { (TEST_CERTS[id], TEST_KEYS[id]) } +/// Creating a cert client identity. Using the same certificate for both shard and mpc. #[must_use] -pub fn get_test_identity(id: HelperIdentity) -> ClientIdentity { - let (mut certificate, mut private_key) = get_test_certificate_and_key(id); - ClientIdentity::from_pkcs8(&mut certificate, &mut private_key).unwrap() +pub fn get_client_test_identity(id: HelperIdentity, ix: ShardIndex) -> ClientIdentities { + let pos = ix.as_index() * 3 + id.as_index(); + let (mut certificate, mut private_key) = get_test_certificate_and_key(pos); + let (mut scertificate, mut sprivate_key) = get_test_certificate_and_key(pos); + ClientIdentities { + helper: ClientIdentity::from_pkcs8(&mut certificate, &mut private_key).unwrap(), + shard: ClientIdentity::from_pkcs8(&mut scertificate, &mut sprivate_key).unwrap(), + } } -pub const TEST_CERTS: [&[u8]; 3] = [ +pub const TEST_CERTS: [&[u8]; 6] = [ b"\ -----BEGIN CERTIFICATE----- MIIBZjCCAQ2gAwIBAgIIGGCAUnB4cZcwCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ @@ -364,14 +577,62 @@ BAQDAgKkMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAKBggqhkjOPQQD AgNHADBEAiB+K2yadiLIDR7ZvDpyMIXP70gL3CXp7JmVmh8ygFtbjQIgU16wnFBy jn+NXYPeKEWnkCcVKjFED6MevGnOgrJylgY= -----END CERTIFICATE----- +", + b" +-----BEGIN CERTIFICATE----- +MIIBZDCCAQugAwIBAgIIFeKzq6ypfYgwCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ +bG9jYWxob3N0MB4XDTI0MTAwNjIyMTEzOFoXDTI1MDEwNTIyMTEzOFowFDESMBAG +A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAECKdJUHmm +Mmqtvhu4PpWwwZnu+LFjaE8Y9guDNIXN+O9kulFl1hLVMx6WLpoScrLYlvHrQvcq +/BTG24EOKAeaRqNHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE +AwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwCgYIKoZIzj0EAwID +RwAwRAIgBO2SBoLmPikfcovOFpjA8jpY+JuSybeISUKD2GAsXQICIEChXm7/UJ7p +86qXEVsjN2N1pyRd6rUNxLyCaV87ZmfS +-----END CERTIFICATE----- +", + b" +-----BEGIN CERTIFICATE----- +MIIBZTCCAQugAwIBAgIIXTgB/bkN/aUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ +bG9jYWxob3N0MB4XDTI0MTAwNjIyMTIwM1oXDTI1MDEwNTIyMTIwM1owFDESMBAG +A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEyzSofZIX +XgLUKGumrN3SEXOMOAKXcl1VshTBzvyVwxxnD01WVLgS80/TELEltT8SMj1Cgu7I +tkDx3EVPjq4pOKNHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE +AwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwCgYIKoZIzj0EAwID +SAAwRQIhAN93g0zfB/4VyhNOaY1uCb4af4qMxcz1wp0yZ7HKAyWqAiBVPgv4X7aR +JMepVZwIWJrVhnxdcmzOuONoeLZPZraFpw== +-----END CERTIFICATE----- +", + b" +-----BEGIN CERTIFICATE----- +MIIBZTCCAQugAwIBAgIIXTgB/bkN/aUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ +bG9jYWxob3N0MB4XDTI0MTAwNjIyMTIwM1oXDTI1MDEwNTIyMTIwM1owFDESMBAG +A1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEyzSofZIX +XgLUKGumrN3SEXOMOAKXcl1VshTBzvyVwxxnD01WVLgS80/TELEltT8SMj1Cgu7I +tkDx3EVPjq4pOKNHMEUwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQE +AwICpDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwCgYIKoZIzj0EAwID +SAAwRQIhAN93g0zfB/4VyhNOaY1uCb4af4qMxcz1wp0yZ7HKAyWqAiBVPgv4X7aR +JMepVZwIWJrVhnxdcmzOuONoeLZPZraFpw== +-----END CERTIFICATE----- ", ]; -pub static TEST_CERTS_DER: Lazy<[CertificateDer; 3]> = Lazy::new(|| { +pub static TEST_CERTS_DER: Lazy<[CertificateDer; 6]> = Lazy::new(|| { TEST_CERTS.map(|mut pem| rustls_pemfile::certs(&mut pem).flatten().next().unwrap()) }); -pub const TEST_KEYS: [&[u8]; 3] = [ +#[must_use] +pub fn get_certs_der_row(ring_index: usize) -> [Option>; 3] { + let r: [usize; 3] = array::from_fn(|i| i + ring_index * 3); + r.map(|i| Some(TEST_CERTS_DER[i].clone())) +} + +#[must_use] +pub fn get_certs_der_col(helper: usize) -> [Option>; 2] { + let r: [usize; 2] = array::from_fn(|i| i * 3 + helper); + r.map(|i| Some(TEST_CERTS_DER[i].clone())) +} + +pub const TEST_KEYS: [&[u8]; 6] = [ b"\ -----BEGIN PRIVATE KEY----- MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgHmPeGcv6Dy9QWPHD @@ -392,6 +653,27 @@ MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgDfOsXGbO9T6e9mPb u9BeVKo7j/DyX4j3XcqrOYnIwOOhRANCAASEORA/IDvqRGiJpddoyocRa+9HEG2B 6P8vfTTV28Ph7n9YBgJodGd29Kt7Dy2IdCjy7PsOik5KGZ4Ee+a+juKk -----END PRIVATE KEY----- +", + b"\ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgWlbBJGC40HwzwMsd +3a6o6x75HZgRnktVwBoi6/84nPmhRANCAAQIp0lQeaYyaq2+G7g+lbDBme74sWNo +Txj2C4M0hc3472S6UWXWEtUzHpYumhJystiW8etC9yr8FMbbgQ4oB5pG +-----END PRIVATE KEY----- +", + b"\ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgi9TsF4lX49P+GIER +DjyUhMiyRZ52EsD00dGPRA4XJbahRANCAATLNKh9khdeAtQoa6as3dIRc4w4Apdy +XVWyFMHO/JXDHGcPTVZUuBLzT9MQsSW1PxIyPUKC7si2QPHcRU+Orik4 +-----END PRIVATE KEY----- +", + b"\ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgs8cH8I4hrdrqDN/d +p1HENqJEFXMwcERH5JFyW/B6D/ChRANCAAT+nXv66H0vd2omUjYwWDYbGkIiGc6S +jzcyiSIULkaelVYvnEQBYefjGLQwvwbifmMrQ+hfQUT9FNbGRQ788pK9 +-----END PRIVATE KEY----- ", ]; @@ -404,3 +686,48 @@ const TEST_HPKE_PUBLIC_KEY: &str = "\ const TEST_HPKE_PRIVATE_KEY: &str = "\ a0778c3e9960576cbef4312a3b7ca34137880fd588c11047bd8b6a8b70b5a151 "; + +#[cfg(all(test, unit_test))] +mod tests { + use super::TestConfigBuilder; + use crate::{helpers::HelperIdentity, net::test::Ports}; + + /// This simple test makes sure that testing networks are created properly. + /// The network itself won't be excersized as that's tested elsewhere. + #[test] + fn create_4_shard_http_network() { + let ports: Vec>> = vec![ + Ports { + ring: vec![10000, 10001, 10002], + shards: vec![10005, 10006, 10007], + }, + Ports { + ring: vec![10010, 10011, 10012], + shards: vec![10015, 10016, 10017], + }, + Ports { + ring: vec![10020, 10021, 10022], + shards: vec![10025, 10026, 10027], + }, + Ports { + ring: vec![10030, 10031, 10032], + shards: vec![10035, 10036, 10037], + }, + ]; + // Providing ports and no https certs to keep this test fast + let conf = TestConfigBuilder::default() + .with_disable_https_option(true) + .with_ports_by_ring(ports) + .build(); + + assert!(conf.disable_https); + assert_eq!(conf.rings.len(), 4); + assert_eq!(conf.sharding_networks.len(), 3); + let shards_2 = conf.get_shards_for_helper(HelperIdentity::TWO); + assert_eq!(shards_2.get_first_shard().config.port, Some(10006)); + let second_helper_third_shard_configs = &shards_2.servers.configs[2]; + assert_eq!(second_helper_third_shard_configs.config.port, Some(10026)); + let leader_ring_configs = &conf.leaders_ring().servers.configs; + assert_eq!(leader_ring_configs[2].config.port, Some(10002)); + } +} diff --git a/ipa-core/src/net/transport.rs b/ipa-core/src/net/transport.rs index a0fcd92a3..29738df3a 100644 --- a/ipa-core/src/net/transport.rs +++ b/ipa-core/src/net/transport.rs @@ -348,7 +348,7 @@ impl Transport for ShardHttpTransport { #[cfg(all(test, web_test, descriptive_gate))] mod tests { - use std::{iter::zip, net::TcpListener, task::Poll}; + use std::{iter::zip, task::Poll}; use bytes::Bytes; use futures::stream::{poll_immediate, StreamExt}; @@ -361,7 +361,6 @@ mod tests { use super::*; use crate::{ - config::{NetworkConfig, ServerConfig}, ff::{FieldType, Fp31, Serializable}, helpers::{ make_owned_handler, @@ -369,7 +368,7 @@ mod tests { }, net::{ client::ClientIdentity, - test::{get_test_identity, TestConfig, TestConfigBuilder, TestServer}, + test::{create_ids, TestConfig, TestConfigBuilder, TestServer}, }, secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, test_fixture::Reconstruct, @@ -446,55 +445,62 @@ mod tests { } // TODO(651): write a test for an error while reading the body (after error handling is finalized) - - async fn make_helpers( - sockets: [TcpListener; 3], - server_config: [ServerConfig; 3], - network_config: &NetworkConfig, - disable_https: bool, - ) -> [HelperApp; 3] { + async fn make_helpers(mut conf: ShardedConfig) -> [HelperApp; 3] { + let leaders_ring = conf.rings.pop().unwrap(); join_all( - zip(HelperIdentity::make_three(), zip(sockets, server_config)).map( - |(id, (socket, server_config))| async move { - let identity = if disable_https { - ClientIdentity::Header(id) - } else { - get_test_identity(id) - }; - let (setup, handler) = AppSetup::new(AppConfig::default()); + zip(HelperIdentity::make_three(), leaders_ring.servers.configs).map( + |(id, mut addr_server)| { + let (setup, mpc_handler) = AppSetup::new(AppConfig::default()); + let shard_ix = ShardIndex::FIRST; + let identities = create_ids(conf.disable_https, id, shard_ix); + + // Ring config let clients = MpcHelperClient::from_conf( &IpaRuntime::current(), - network_config, - &identity, + &leaders_ring.network, + &identities.helper, ); let (transport, server) = MpcHttpTransport::new( IpaRuntime::current(), id, - server_config.clone(), - network_config.clone(), + addr_server.config.clone(), + leaders_ring.network.clone(), &clients, - Some(handler), + Some(mpc_handler), + ); + + // Shard Config + let helper_shards = conf.get_shards_for_helper(id); + let addr_shard = helper_shards.get_first_shard(); + let shard_network_config = helper_shards.network.clone(); + let shard_clients = MpcHelperClient::::shards_from_conf( + &IpaRuntime::current(), + &shard_network_config, + &identities.shard, ); - // TODO: Following is just temporary until Shard Transport is actually used. - let shard_clients_config = network_config.client.clone(); - let shard_server_config = server_config; - let shard_network_config = - NetworkConfig::new_shards(vec![], shard_clients_config); - let (shard_transport, _shard_server) = ShardHttpTransport::new( + let (shard_transport, shard_server) = ShardHttpTransport::new( IpaRuntime::current(), - ShardIndex::FIRST, - shard_server_config, + shard_ix, + addr_shard.config.clone(), shard_network_config, - vec![], - None, + shard_clients, + None, // This will come online once we go into Query Workflow ); - // --- - server - .start_on(&IpaRuntime::current(), Some(socket), ()) - .await; - - setup.connect(transport, shard_transport) + let helper_shards = conf.get_shards_for_helper_mut(id); + let addr_shard = helper_shards.get_first_shard_mut(); + let ring_socket = addr_server.socket.take(); + let sharding_socket = addr_shard.socket.take(); + + async move { + server + .start_on(&IpaRuntime::current(), ring_socket, ()) + .await; + shard_server + .start_on(&IpaRuntime::current(), sharding_socket, ()) + .await; + setup.connect(transport, shard_transport) + } }, ), ) @@ -504,38 +510,25 @@ mod tests { .unwrap() } - async fn test_three_helpers(mut conf: TestConfig) { + async fn test_three_helpers(conf: ShardedConfig) { let clients = MpcHelperClient::from_conf( &IpaRuntime::current(), - &conf.network, + &conf.leaders_ring().network, &ClientIdentity::None, ); - let _helpers = make_helpers( - conf.sockets.take().unwrap(), - conf.servers, - &conf.network, - conf.disable_https, - ) - .await; - + let _helpers = make_helpers(conf).await; test_multiply(&clients).await; } #[tokio::test(flavor = "multi_thread")] async fn happy_case_twice() { - let mut conf = TestConfigBuilder::with_open_ports().build(); + let conf = TestConfigBuilder::default().build(); let clients = MpcHelperClient::from_conf( &IpaRuntime::current(), - &conf.network, + &conf.leaders_ring().network, &ClientIdentity::None, ); - let _helpers = make_helpers( - conf.sockets.take().unwrap(), - conf.servers, - &conf.network, - conf.disable_https, - ) - .await; + let _helpers = make_helpers(conf).await; test_multiply(&clients).await; test_multiply(&clients).await; @@ -585,7 +578,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn three_helpers_http() { - let conf = TestConfigBuilder::with_open_ports() + let conf = TestConfigBuilder::default() .with_disable_https_option(true) .build(); test_three_helpers(conf).await; @@ -593,7 +586,7 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn three_helpers_https() { - let conf = TestConfigBuilder::with_open_ports().build(); + let conf = TestConfigBuilder::default().build(); test_three_helpers(conf).await; } } From f0cb5070daf100ff8115b379f394b2c72e9e1197 Mon Sep 17 00:00:00 2001 From: Christian Berkhoff Date: Thu, 24 Oct 2024 09:39:24 -0700 Subject: [PATCH 2/6] Fix: rename TestConfig --- ipa-core/src/net/transport.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipa-core/src/net/transport.rs b/ipa-core/src/net/transport.rs index 29738df3a..095e1bdb2 100644 --- a/ipa-core/src/net/transport.rs +++ b/ipa-core/src/net/transport.rs @@ -445,7 +445,7 @@ mod tests { } // TODO(651): write a test for an error while reading the body (after error handling is finalized) - async fn make_helpers(mut conf: ShardedConfig) -> [HelperApp; 3] { + async fn make_helpers(mut conf: TestConfig) -> [HelperApp; 3] { let leaders_ring = conf.rings.pop().unwrap(); join_all( zip(HelperIdentity::make_three(), leaders_ring.servers.configs).map( @@ -510,7 +510,7 @@ mod tests { .unwrap() } - async fn test_three_helpers(conf: ShardedConfig) { + async fn test_three_helpers(conf: TestConfig) { let clients = MpcHelperClient::from_conf( &IpaRuntime::current(), &conf.leaders_ring().network, From d691b0643a93f1b1dbc624d321bcd958e3ed407b Mon Sep 17 00:00:00 2001 From: Christian Berkhoff Date: Mon, 28 Oct 2024 13:06:00 -0700 Subject: [PATCH 3/6] Addressing comments --- ipa-core/src/net/mod.rs | 10 +- ipa-core/src/net/test.rs | 577 ++++++++++++++++++---------------- ipa-core/src/net/transport.rs | 119 +++---- ipa-core/src/sharding.rs | 28 ++ 4 files changed, 395 insertions(+), 339 deletions(-) diff --git a/ipa-core/src/net/mod.rs b/ipa-core/src/net/mod.rs index e0fdca35a..c71c609ea 100644 --- a/ipa-core/src/net/mod.rs +++ b/ipa-core/src/net/mod.rs @@ -115,15 +115,15 @@ pub fn parse_certificate_and_private_key_bytes( mod tests { use std::io::ErrorKind; - use crate::net::test; + use super::test::get_test_certificate_and_key; + use crate::sharding::ShardedHelperIdentity; const NOTHING: &[u8] = b" "; const GARBAGE: &[u8] = b"ksjdhfskjdfhsdf"; #[test] fn parse_cert_pk_happy_path() { - let mut c = test::TEST_CERTS[0]; - let mut pk = test::TEST_KEYS[0]; + let (mut c, mut pk) = get_test_certificate_and_key(ShardedHelperIdentity::ONE_FIRST); super::parse_certificate_and_private_key_bytes(&mut c, &mut pk).unwrap(); } @@ -131,7 +131,7 @@ mod tests { #[should_panic(expected = "No certificates found")] fn parse_cert_pk_no_cert() { let mut c = NOTHING; - let mut pk = test::TEST_KEYS[0]; + let (_, mut pk) = get_test_certificate_and_key(ShardedHelperIdentity::ONE_FIRST); let r = super::parse_certificate_and_private_key_bytes(&mut c, &mut pk); assert_eq!(r.as_ref().unwrap_err().kind(), ErrorKind::Other); r.unwrap(); @@ -140,7 +140,7 @@ mod tests { #[test] #[should_panic(expected = "No private key")] fn parse_cert_pk_no_pk() { - let mut c = test::TEST_CERTS[0]; + let (mut c, _) = get_test_certificate_and_key(ShardedHelperIdentity::ONE_FIRST); let mut pk = NOTHING; let r = super::parse_certificate_and_private_key_bytes(&mut c, &mut pk); assert_eq!(r.as_ref().unwrap_err().kind(), ErrorKind::Other); diff --git a/ipa-core/src/net/test.rs b/ipa-core/src/net/test.rs index 7ceda7cd3..28035c726 100644 --- a/ipa-core/src/net/test.rs +++ b/ipa-core/src/net/test.rs @@ -9,9 +9,9 @@ #![allow(clippy::missing_panics_doc)] use std::{ - array, - iter::zip, + collections::HashSet, net::{SocketAddr, TcpListener}, + ops::Index, }; use once_cell::sync::Lazy; @@ -27,122 +27,153 @@ use crate::{ helpers::{repeat_n, HandlerBox, HelperIdentity, RequestHandler, TransportIdentity}, hpke::{Deserializable as _, IpaPublicKey}, net::{ClientIdentity, Helper, MpcHelperClient, MpcHelperServer}, - sharding::ShardIndex, + sharding::{ShardIndex, ShardedHelperIdentity}, sync::Arc, test_fixture::metrics::MetricsHandle, }; /// Simple struct to keep default port configuration organized. -pub struct Ports { - ring: C, - shards: C, +#[derive(Clone)] +pub struct Ports { + ring: [u16; 3], + shards: [u16; 3], } /// A **single** ring with 3 hosts, each with a ring and sharding port. -pub const DEFAULT_TEST_PORTS: Ports<[u16; 3]> = Ports { +pub const DEFAULT_TEST_PORTS: Ports = Ports { ring: [3000, 3001, 3002], shards: [6000, 6001, 6002], }; /// Configuration of a server that can be reached via socket or port. pub struct AddressableTestServer { + pub id: ShardedHelperIdentity, /// Contains the ports pub config: ServerConfig, /// Sockets are created if no port was specified. pub socket: Option, } -/// A bunch of addressable servers. These can be part of a Shard or MPC network. -#[derive(Default)] -pub struct TestServers { - pub configs: Vec, -} - -impl TestServers { - /// Creates a bunch of addressable servers from the given configurations plus the optional - /// ports. Meant to be used during the definition of an MPC ring. - fn new(configs: Vec, sockets: Vec>) -> Self { - assert_eq!(configs.len(), sockets.len()); - let mut new_configs = Vec::with_capacity(configs.len()); - for s in zip(configs, sockets) { - new_configs.push(AddressableTestServer { - config: s.0, - socket: s.1, - }); - } - TestServers { - configs: new_configs, - } - } - - /// Adds a test server. These increments are useful during shard network definition. - fn push_shard(&mut self, config: AddressableTestServer) { - self.configs.push(config); +/// Creates a new socket from the OS if no port is given. +fn create_port(optional_port: Option) -> (Option, u16) { + if let Some(port) = optional_port { + (None, port) + } else { + let socket = TcpListener::bind("localhost:0").unwrap(); + let port = socket.local_addr().unwrap().port(); + (Some(socket), port) } +} - pub fn iter(&self) -> impl Iterator { - self.configs.iter() +impl AddressableTestServer { + /// Creates a new Test Server with the given Id. If no port is given, one will be obtained from + /// the OS. + fn new( + id: ShardedHelperIdentity, + optional_port: Option, + conf: &TestConfigBuilder, + ) -> Self { + let (socket, port) = create_port(optional_port); + let config = if conf.disable_https { + server_config_insecure_http(port, !conf.disable_matchkey_encryption) + } else { + server_config_https(id, port, !conf.disable_matchkey_encryption) + }; + Self { id, config, socket } } } /// Either a single Ring on MPC connection or all of the shards in a Helper. pub struct TestNetwork { pub network: NetworkConfig, // Contains Clients config - pub servers: TestServers, + pub servers: Vec, +} + +impl TestNetwork { + /// Helper function that creates [`PeerConfig`] + fn create_peers( + servers: &[AddressableTestServer], + conf: &TestConfigBuilder, + ) -> Vec { + servers + .iter() + .map(|addr_server| { + let port = addr_server + .config + .port + .expect("Port should have been defined already"); + let (scheme, certificate) = if conf.disable_https { + ("http", None) + } else { + ("https", Some(TEST_CERTS_DER[addr_server.id].clone())) + }; + let url = format!("{scheme}://localhost:{port}").parse().unwrap(); + let hpke_config = if conf.disable_matchkey_encryption { + None + } else { + Some(HpkeClientConfig::new( + IpaPublicKey::from_bytes( + &hex::decode(TEST_HPKE_PUBLIC_KEY.trim()).unwrap(), + ) + .unwrap(), + )) + }; + PeerConfig { + url, + certificate, + hpke_config, + } + }) + .collect() + } } impl TestNetwork { #[must_use] - /// Gets a ref to the first shard in this network. pub fn get_first_shard(&self) -> &AddressableTestServer { - self.servers.configs.first().unwrap() + self.servers.first().unwrap() } /// Gets a mut ref to the first shard in this network. pub fn get_first_shard_mut(&mut self) -> &mut AddressableTestServer { - self.servers.configs.get_mut(0).unwrap() + self.servers.get_mut(0).unwrap() } } -/// Uber container for test configuration. Provides access to a vec of MPC rings and 3 sharding -/// networks (one for each Helper) -pub struct TestConfig { - pub disable_https: bool, - pub rings: Vec>, - pub sharding_networks: Vec>, -} - -impl TestConfig { - /// Gets a ref to the first ring in the network. This ring is important because it's the one - /// that's reached out by the report collector on behalf of all the shards in the helper. - #[must_use] - pub fn leaders_ring(&self) -> &TestNetwork { - &self.rings[0] - } - - /// Gets a ref to the entire shard network for a specific helper. - #[must_use] - pub fn get_shards_for_helper(&self, id: HelperIdentity) -> &TestNetwork { - self.sharding_networks.get(id.as_index()).unwrap() - } - - /// Gets a mut ref to the entire shard network for a specific helper. - pub fn get_shards_for_helper_mut(&mut self, id: HelperIdentity) -> &mut TestNetwork { - self.sharding_networks.get_mut(id.as_index()).unwrap() - } -} - -impl TestConfig { - #[must_use] - pub fn builder() -> TestConfigBuilder { - TestConfigBuilder::default() +impl TestNetwork { + /// Creates 3 mpc test servers and creates a network. + fn new_mpc(ix: ShardIndex, ports: Vec>, conf: &TestConfigBuilder) -> Self { + let servers: Vec<_> = HelperIdentity::make_three() + .into_iter() + .zip(ports) + .map(|(id, p)| { + let sid = ShardedHelperIdentity::new(id, ix); + AddressableTestServer::new(sid, p, conf) + }) + .collect(); + let peers = Self::create_peers(servers.as_slice(), conf); + let client_config = conf.create_client_config(); + let network = NetworkConfig::::new_mpc(peers, client_config); + TestNetwork { network, servers } } } -impl Default for TestConfig { - fn default() -> Self { - Self::builder().build() +impl TestNetwork { + /// Creates all the shards for a helper and creates a network. + fn new_shards(id: HelperIdentity, ports: Vec>, conf: &TestConfigBuilder) -> Self { + let servers: Vec<_> = (0..conf.shard_count) + .map(ShardIndex) + .zip(ports) + .map(|(ix, p)| { + let sid = ShardedHelperIdentity::new(id, ix); + AddressableTestServer::new(sid, p, conf) + }) + .collect(); + let peers = Self::create_peers(servers.as_slice(), conf); + let client_config = conf.create_client_config(); + let network = NetworkConfig::::new_shards(peers, client_config); + TestNetwork { network, servers } } } @@ -168,12 +199,12 @@ fn server_config_insecure_http(port: u16, matchkey_encryption: bool) -> ServerCo } #[must_use] -pub fn server_config_https( - cert_index: usize, +fn server_config_https( + id: ShardedHelperIdentity, port: u16, matchkey_encryption: bool, ) -> ServerConfig { - let (certificate, private_key) = get_test_certificate_and_key(cert_index); + let (certificate, private_key) = get_test_certificate_and_key(id); ServerConfig { port: Some(port), disable_https: false, @@ -185,23 +216,88 @@ pub fn server_config_https( } } +/// Uber container for test configuration. Provides access to a vec of MPC rings and 3 sharding +/// networks (one for each Helper) +pub struct TestConfig { + pub disable_https: bool, + pub rings: Vec>, + pub shards: Vec>, +} + +impl TestConfig { + /// Gets a ref to the first ring in the network. This ring is important because it's the one + /// that's reached out by the report collector on behalf of all the shards in the helper. + #[must_use] + pub fn leaders_ring(&self) -> &TestNetwork { + &self.rings[0] + } + + /// Gets a ref to the entire shard network for a specific helper. + #[must_use] + pub fn get_shards_for_helper(&self, id: HelperIdentity) -> &TestNetwork { + self.shards.get(id.as_index()).unwrap() + } + + /// Gets a mut ref to the entire shard network for a specific helper. + pub fn get_shards_for_helper_mut(&mut self, id: HelperIdentity) -> &mut TestNetwork { + self.shards.get_mut(id.as_index()).unwrap() + } + + /// Creates a new [`TestConfig`] using the provided configuration. + fn new(conf: &TestConfigBuilder) -> Self { + let rings = (0..conf.shard_count) + .map(ShardIndex) + .map(|s| { + let ports = conf.get_ports_for_shard_index(s); + TestNetwork::::new_mpc(s, ports, conf) + }) + .collect(); + let shards = HelperIdentity::make_three() + .into_iter() + .map(|id| { + let ports = conf.get_ports_for_helper_identity(id); + TestNetwork::::new_shards(id, ports, conf) + }) + .collect(); + Self { + disable_https: conf.disable_https, + rings, + shards, + } + } +} + +impl TestConfig { + #[must_use] + pub fn builder() -> TestConfigBuilder { + TestConfigBuilder::default() + } +} + +impl Default for TestConfig { + fn default() -> Self { + Self::builder().build() + } +} + pub struct TestConfigBuilder { - /// Can be empty meaning that free ports should be obtained from the operating system. - /// One ring per shard in a helper (sharding factor). For each ring we need 3 shard and - /// 3 mpc ports. - ports_by_ring: Vec>>, - /// Describes the number of shards per helper - sharding_factor: usize, + /// Can be None, meaning that free ports should be obtained from the operating system. + /// One ring per shard in a helper (see [`shard_count`]). For each ring we need 3 shard + /// (A `Vec`) and 3 mpc ports. + ports_by_ring: Option>, + /// Describes the number of shards per helper. This is directly related to [`ports_by_ring`]. + shard_count: u32, disable_https: bool, use_http1: bool, disable_matchkey_encryption: bool, } impl Default for TestConfigBuilder { + /// Non-sharded, HTTPS and get ports from OS. fn default() -> Self { Self { - ports_by_ring: vec![], - sharding_factor: 1, + ports_by_ring: None, + shard_count: 1, disable_https: false, use_http1: false, disable_matchkey_encryption: false, @@ -213,11 +309,8 @@ impl TestConfigBuilder { #[must_use] pub fn with_http_and_default_test_ports() -> Self { Self { - ports_by_ring: vec![Ports { - ring: DEFAULT_TEST_PORTS.ring.to_vec(), - shards: DEFAULT_TEST_PORTS.shards.to_vec(), - }], - sharding_factor: 1, + ports_by_ring: Some(vec![DEFAULT_TEST_PORTS]), + shard_count: 1, disable_https: true, use_http1: false, disable_matchkey_encryption: false, @@ -230,10 +323,19 @@ impl TestConfigBuilder { self } + /// Sets the ports the test network should use. + /// # Panics + /// If a duplicate port is given. #[must_use] - pub fn with_ports_by_ring(mut self, value: Vec>>) -> Self { - self.sharding_factor = value.len(); - self.ports_by_ring = value; + pub fn with_ports_by_ring(mut self, value: Vec) -> Self { + self.shard_count = value.len().try_into().unwrap(); + let mut uniqueness_set = HashSet::new(); + for ps in &value { + for p in ps.ring.iter().chain(ps.shards.iter()) { + assert!(uniqueness_set.insert(p), "Found duplicate port {p}"); + } + } + self.ports_by_ring = Some(value); self } @@ -251,151 +353,40 @@ impl TestConfigBuilder { self } - /// Creates 3 MPC test servers, assigning ports from OS if necessary. - fn build_mpc_servers( - &self, - optional_ports: Option>, - ring_index: usize, - ) -> TestServers { - let mut sockets = vec![None, None, None]; - let ports = optional_ports.unwrap_or_else(|| { - sockets = (0..3) - .map(|_| Some(TcpListener::bind("localhost:0").unwrap())) - .collect(); - sockets - .iter() - .map(|sock| sock.as_ref().unwrap().local_addr().unwrap().port()) - .collect() - }); - // Ports will always be defined after this. Socks only if there were no ports set. - let configs = if self.disable_https { - ports - .into_iter() - .map(|ports| server_config_insecure_http(ports, !self.disable_matchkey_encryption)) - .collect() - } else { - let start_idx = ring_index * 3; - (start_idx..start_idx + 3) - .map(|id| server_config_https(id, ports[id], !self.disable_matchkey_encryption)) - .collect() - }; - TestServers::new(configs, sockets) - } - - /// Defines a test network with a single MPC ring (3 helpers). - fn build_ring_network( - &self, - servers: TestServers, - scheme: &str, - certs: Vec>>, - ) -> TestNetwork { - let (peers, client_config) = self.build_network(&servers, scheme, certs); - let network = NetworkConfig::::new_mpc(peers, client_config); - TestNetwork { network, servers } + /// Creates a HTTP1 or HTTP2 client config. + pub fn create_client_config(&self) -> ClientConfig { + self.use_http1 + .then(ClientConfig::use_http1) + .unwrap_or_default() } - /// Defines a test network with the N shards on a helper. - fn build_shard_network( - &self, - servers: TestServers, - scheme: &str, - certs: Vec>>, - ) -> TestNetwork { - let (peers, client_config) = self.build_network(&servers, scheme, certs); - let network = NetworkConfig::::new_shards(peers, client_config); - TestNetwork { network, servers } + /// Get all the MPC ports in a ring specified by the shard index. + fn get_ports_for_shard_index(&self, ix: ShardIndex) -> Vec> { + if let Some(ports_by_ring) = &self.ports_by_ring { + let ports = ports_by_ring[ix.as_index()].clone(); + ports.ring.into_iter().map(Some).collect() + } else { + repeat_n(None, 3).collect() + } } - fn build_network( - &self, - servers: &TestServers, - scheme: &str, - certs: Vec>>, - ) -> (Vec, ClientConfig) { - let peers = zip(servers.configs.iter(), certs) - .map(|(addr_server, certificate)| { - let port = addr_server - .config - .port - .expect("Port should have been defined already"); - let url = format!("{scheme}://localhost:{port}").parse().unwrap(); - let hpke_config = if self.disable_matchkey_encryption { - None - } else { - Some(HpkeClientConfig::new( - IpaPublicKey::from_bytes( - &hex::decode(TEST_HPKE_PUBLIC_KEY.trim()).unwrap(), - ) - .unwrap(), - )) - }; - PeerConfig { - url, - certificate, - hpke_config, - } - }) - .collect(); - - let client_config = self - .use_http1 - .then(ClientConfig::use_http1) - .unwrap_or_default(); - (peers, client_config) + /// Get all the shard ports in a helper. + fn get_ports_for_helper_identity(&self, id: HelperIdentity) -> Vec> { + if let Some(ports_by_ring) = &self.ports_by_ring { + ports_by_ring + .iter() + .map(|r| r.shards[id.as_index()]) + .map(Some) + .collect() + } else { + repeat_n(None, self.shard_count.try_into().unwrap()).collect() + } } - /// Creates a test network with shards + /// Creates a test network with shards. #[must_use] - pub fn build(self) -> TestConfig { - // first we build all the rings and then we connect all the shards. - // The following will accumulate the shards as we create the rings. - let mut sharding_networks_shards = vec![ - TestServers::default(), - TestServers::default(), - TestServers::default(), - ]; - let rings: Vec> = (0..self.sharding_factor) - .map(|s| { - let ring_ports = self.ports_by_ring.get(s).map(|p| p.ring.clone()); - let shards_ports = self.ports_by_ring.get(s).map(|p| p.shards.clone()); - - // We create the servers in the MPC ring and connect them (in a TestNetwork). - let ring_servers = self.build_mpc_servers(ring_ports, s); - let (scheme, certs) = if self.disable_https { - ("http", [None, None, None]) - } else { - ("https", get_certs_der_row(s)) - }; - let ring_network = self.build_ring_network(ring_servers, scheme, certs.to_vec()); - - // We create the sharding servers and accumulate them but don't connect them yet - let shard_servers = self.build_mpc_servers(shards_ports, s); - for (i, s) in shard_servers.configs.into_iter().enumerate() { - sharding_networks_shards[i].push_shard(s); - } - - ring_network - }) - .collect(); - - let sharding_networks = sharding_networks_shards - .into_iter() - .enumerate() - .map(|(i, s)| { - let (scheme, certs) = if self.disable_https { - ("http", repeat_n(None, self.sharding_factor).collect()) - } else { - ("https", get_certs_der_col(i).to_vec()) - }; - self.build_shard_network(s, scheme, certs) - }) - .collect(); - - TestConfig { - disable_https: self.disable_https, - rings, - sharding_networks, - } + pub fn build(&self) -> TestConfig { + TestConfig::new(self) } } pub struct TestServer { @@ -471,14 +462,15 @@ impl TestServerBuilder { } pub async fn build(self) -> TestServer { - let identities = create_ids(self.disable_https, HelperIdentity::ONE, ShardIndex::FIRST); + let identities = + ClientIdentities::new(self.disable_https, ShardedHelperIdentity::ONE_FIRST); let mut test_config = TestConfig::builder() .with_disable_https_option(self.disable_https) .with_use_http1_option(self.use_http1) // TODO: add disble_matchkey here .build(); let leaders_ring = test_config.rings.pop().unwrap(); - let first_server = leaders_ring.servers.configs.into_iter().next().unwrap(); + let first_server = leaders_ring.servers.into_iter().next().unwrap(); let clients = MpcHelperClient::from_conf( &IpaRuntime::current(), &leaders_ring.network, @@ -513,35 +505,58 @@ pub struct ClientIdentities { pub shard: ClientIdentity, } -#[must_use] -pub fn create_ids(disable_https: bool, id: HelperIdentity, ix: ShardIndex) -> ClientIdentities { - if disable_https { - ClientIdentities { - helper: ClientIdentity::Header(id), - shard: ClientIdentity::Header(ix), +impl ClientIdentities { + #[must_use] + pub fn new(disable_https: bool, id: ShardedHelperIdentity) -> Self { + if disable_https { + ClientIdentities { + helper: ClientIdentity::Header(id.helper_identity), + shard: ClientIdentity::Header(id.shard_index), + } + } else { + get_client_test_identity(id) } - } else { - get_client_test_identity(id, ix) } } -fn get_test_certificate_and_key(id: usize) -> (&'static [u8], &'static [u8]) { +impl Index for [&'static [u8]; S] { + type Output = &'static [u8]; + + fn index(&self, index: ShardedHelperIdentity) -> &Self::Output { + let pos = index.as_index(); + self.get(pos) + .unwrap_or_else(|| panic!("The computed index {pos} is outside of {S}")) + } +} + +impl Index for Lazy<[CertificateDer<'static>; S]> { + type Output = CertificateDer<'static>; + + fn index(&self, index: ShardedHelperIdentity) -> &Self::Output { + let pos = index.as_index(); + self.get(pos) + .unwrap_or_else(|| panic!("The computed index {pos} is outside of {S}")) + } +} + +pub(super) fn get_test_certificate_and_key( + id: ShardedHelperIdentity, +) -> (&'static [u8], &'static [u8]) { (TEST_CERTS[id], TEST_KEYS[id]) } /// Creating a cert client identity. Using the same certificate for both shard and mpc. #[must_use] -pub fn get_client_test_identity(id: HelperIdentity, ix: ShardIndex) -> ClientIdentities { - let pos = ix.as_index() * 3 + id.as_index(); - let (mut certificate, mut private_key) = get_test_certificate_and_key(pos); - let (mut scertificate, mut sprivate_key) = get_test_certificate_and_key(pos); +pub fn get_client_test_identity(id: ShardedHelperIdentity) -> ClientIdentities { + let (mut certificate, mut private_key) = get_test_certificate_and_key(id); + let (mut scertificate, mut sprivate_key) = get_test_certificate_and_key(id); ClientIdentities { helper: ClientIdentity::from_pkcs8(&mut certificate, &mut private_key).unwrap(), shard: ClientIdentity::from_pkcs8(&mut scertificate, &mut sprivate_key).unwrap(), } } -pub const TEST_CERTS: [&[u8]; 6] = [ +const TEST_CERTS: [&[u8]; 6] = [ b"\ -----BEGIN CERTIFICATE----- MIIBZjCCAQ2gAwIBAgIIGGCAUnB4cZcwCgYIKoZIzj0EAwIwFDESMBAGA1UEAwwJ @@ -616,23 +631,11 @@ JMepVZwIWJrVhnxdcmzOuONoeLZPZraFpw== ", ]; -pub static TEST_CERTS_DER: Lazy<[CertificateDer; 6]> = Lazy::new(|| { +static TEST_CERTS_DER: Lazy<[CertificateDer; 6]> = Lazy::new(|| { TEST_CERTS.map(|mut pem| rustls_pemfile::certs(&mut pem).flatten().next().unwrap()) }); -#[must_use] -pub fn get_certs_der_row(ring_index: usize) -> [Option>; 3] { - let r: [usize; 3] = array::from_fn(|i| i + ring_index * 3); - r.map(|i| Some(TEST_CERTS_DER[i].clone())) -} - -#[must_use] -pub fn get_certs_der_col(helper: usize) -> [Option>; 2] { - let r: [usize; 2] = array::from_fn(|i| i * 3 + helper); - r.map(|i| Some(TEST_CERTS_DER[i].clone())) -} - -pub const TEST_KEYS: [&[u8]; 6] = [ +const TEST_KEYS: [&[u8]; 6] = [ b"\ -----BEGIN PRIVATE KEY----- MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgHmPeGcv6Dy9QWPHD @@ -689,29 +692,33 @@ a0778c3e9960576cbef4312a3b7ca34137880fd588c11047bd8b6a8b70b5a151 #[cfg(all(test, unit_test))] mod tests { - use super::TestConfigBuilder; - use crate::{helpers::HelperIdentity, net::test::Ports}; + use super::{get_test_certificate_and_key, TestConfigBuilder}; + use crate::{ + helpers::HelperIdentity, + net::test::{Ports, TEST_CERTS, TEST_KEYS}, + sharding::ShardedHelperIdentity, + }; /// This simple test makes sure that testing networks are created properly. /// The network itself won't be excersized as that's tested elsewhere. #[test] fn create_4_shard_http_network() { - let ports: Vec>> = vec![ + let ports: Vec = vec![ Ports { - ring: vec![10000, 10001, 10002], - shards: vec![10005, 10006, 10007], + ring: [10000, 10001, 10002], + shards: [10005, 10006, 10007], }, Ports { - ring: vec![10010, 10011, 10012], - shards: vec![10015, 10016, 10017], + ring: [10010, 10011, 10012], + shards: [10015, 10016, 10017], }, Ports { - ring: vec![10020, 10021, 10022], - shards: vec![10025, 10026, 10027], + ring: [10020, 10021, 10022], + shards: [10025, 10026, 10027], }, Ports { - ring: vec![10030, 10031, 10032], - shards: vec![10035, 10036, 10037], + ring: [10030, 10031, 10032], + shards: [10035, 10036, 10037], }, ]; // Providing ports and no https certs to keep this test fast @@ -722,12 +729,32 @@ mod tests { assert!(conf.disable_https); assert_eq!(conf.rings.len(), 4); - assert_eq!(conf.sharding_networks.len(), 3); + assert_eq!(conf.shards.len(), 3); let shards_2 = conf.get_shards_for_helper(HelperIdentity::TWO); assert_eq!(shards_2.get_first_shard().config.port, Some(10006)); - let second_helper_third_shard_configs = &shards_2.servers.configs[2]; + let second_helper_third_shard_configs = &shards_2.servers[2]; assert_eq!(second_helper_third_shard_configs.config.port, Some(10026)); - let leader_ring_configs = &conf.leaders_ring().servers.configs; + let leader_ring_configs = &conf.leaders_ring().servers; assert_eq!(leader_ring_configs[2].config.port, Some(10002)); } + + #[test] + #[should_panic(expected = "Found duplicate port 10001")] + fn overlapping_ports() { + let ports: Vec = vec![Ports { + ring: [10000, 10001, 10002], + shards: [10001, 10006, 10007], + }]; + let _ = TestConfigBuilder::default() + .with_disable_https_option(true) + .with_ports_by_ring(ports) + .build(); + } + + #[test] + fn get_assets_by_index() { + let (c, k) = get_test_certificate_and_key(ShardedHelperIdentity::ONE_FIRST); + assert_eq!(TEST_KEYS[0], k); + assert_eq!(TEST_CERTS[0], c); + } } diff --git a/ipa-core/src/net/transport.rs b/ipa-core/src/net/transport.rs index 095e1bdb2..0aedcb937 100644 --- a/ipa-core/src/net/transport.rs +++ b/ipa-core/src/net/transport.rs @@ -351,7 +351,10 @@ mod tests { use std::{iter::zip, task::Poll}; use bytes::Bytes; - use futures::stream::{poll_immediate, StreamExt}; + use futures::{ + future::join, + stream::{poll_immediate, StreamExt}, + }; use futures_util::future::{join_all, try_join_all}; use generic_array::GenericArray; use once_cell::sync::Lazy; @@ -368,9 +371,10 @@ mod tests { }, net::{ client::ClientIdentity, - test::{create_ids, TestConfig, TestConfigBuilder, TestServer}, + test::{ClientIdentities, TestConfig, TestConfigBuilder, TestServer}, }, secret_sharing::{replicated::semi_honest::AdditiveShare, IntoShares}, + sharding::ShardedHelperIdentity, test_fixture::Reconstruct, AppConfig, AppSetup, HelperApp, }; @@ -447,63 +451,60 @@ mod tests { // TODO(651): write a test for an error while reading the body (after error handling is finalized) async fn make_helpers(mut conf: TestConfig) -> [HelperApp; 3] { let leaders_ring = conf.rings.pop().unwrap(); - join_all( - zip(HelperIdentity::make_three(), leaders_ring.servers.configs).map( - |(id, mut addr_server)| { - let (setup, mpc_handler) = AppSetup::new(AppConfig::default()); - let shard_ix = ShardIndex::FIRST; - let identities = create_ids(conf.disable_https, id, shard_ix); - - // Ring config - let clients = MpcHelperClient::from_conf( - &IpaRuntime::current(), - &leaders_ring.network, - &identities.helper, - ); - let (transport, server) = MpcHttpTransport::new( - IpaRuntime::current(), - id, - addr_server.config.clone(), - leaders_ring.network.clone(), - &clients, - Some(mpc_handler), - ); - - // Shard Config - let helper_shards = conf.get_shards_for_helper(id); - let addr_shard = helper_shards.get_first_shard(); - let shard_network_config = helper_shards.network.clone(); - let shard_clients = MpcHelperClient::::shards_from_conf( - &IpaRuntime::current(), - &shard_network_config, - &identities.shard, - ); - let (shard_transport, shard_server) = ShardHttpTransport::new( - IpaRuntime::current(), - shard_ix, - addr_shard.config.clone(), - shard_network_config, - shard_clients, - None, // This will come online once we go into Query Workflow - ); - - let helper_shards = conf.get_shards_for_helper_mut(id); - let addr_shard = helper_shards.get_first_shard_mut(); - let ring_socket = addr_server.socket.take(); - let sharding_socket = addr_shard.socket.take(); - - async move { - server - .start_on(&IpaRuntime::current(), ring_socket, ()) - .await; - shard_server - .start_on(&IpaRuntime::current(), sharding_socket, ()) - .await; - setup.connect(transport, shard_transport) - } - }, - ), - ) + join_all(zip(HelperIdentity::make_three(), leaders_ring.servers).map( + |(id, mut addr_server)| { + let (setup, mpc_handler) = AppSetup::new(AppConfig::default()); + let sid = ShardedHelperIdentity::new(id, ShardIndex::FIRST); + let identities = ClientIdentities::new(conf.disable_https, sid); + + // Ring config + let clients = MpcHelperClient::from_conf( + &IpaRuntime::current(), + &leaders_ring.network, + &identities.helper, + ); + let (transport, server) = MpcHttpTransport::new( + IpaRuntime::current(), + id, + addr_server.config.clone(), + leaders_ring.network.clone(), + &clients, + Some(mpc_handler), + ); + + // Shard Config + let helper_shards = conf.get_shards_for_helper(id); + let addr_shard = helper_shards.get_first_shard(); + let shard_network_config = helper_shards.network.clone(); + let shard_clients = MpcHelperClient::::shards_from_conf( + &IpaRuntime::current(), + &shard_network_config, + &identities.shard, + ); + let (shard_transport, shard_server) = ShardHttpTransport::new( + IpaRuntime::current(), + sid.shard_index, + addr_shard.config.clone(), + shard_network_config, + shard_clients, + None, // This will come online once we go into Query Workflow + ); + + let helper_shards = conf.get_shards_for_helper_mut(id); + let addr_shard = helper_shards.get_first_shard_mut(); + let ring_socket = addr_server.socket.take(); + let sharding_socket = addr_shard.socket.take(); + + async move { + join( + server.start_on(&IpaRuntime::current(), ring_socket, ()), + shard_server.start_on(&IpaRuntime::current(), sharding_socket, ()), + ) + .await; + setup.connect(transport, shard_transport) + } + }, + )) .await .try_into() .ok() diff --git a/ipa-core/src/sharding.rs b/ipa-core/src/sharding.rs index afd051988..f2986f3ab 100644 --- a/ipa-core/src/sharding.rs +++ b/ipa-core/src/sharding.rs @@ -4,6 +4,34 @@ use std::{ ops::{Index, IndexMut}, }; +use crate::helpers::{HelperIdentity, TransportIdentity}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ShardedHelperIdentity { + pub helper_identity: HelperIdentity, + pub shard_index: ShardIndex, +} + +impl ShardedHelperIdentity { + pub const ONE_FIRST: ShardedHelperIdentity = ShardedHelperIdentity { + helper_identity: HelperIdentity::ONE, + shard_index: ShardIndex::FIRST, + }; + + #[must_use] + pub fn new(helper_identity: HelperIdentity, shard_index: ShardIndex) -> Self { + Self { + helper_identity, + shard_index, + } + } + + #[must_use] + pub fn as_index(&self) -> usize { + self.shard_index.as_index() * 3 + self.helper_identity.as_index() + } +} + /// A unique zero-based index of the helper shard. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct ShardIndex(pub u32); From 797b35def73a0e09565fc06361ede353ac955a9b Mon Sep 17 00:00:00 2001 From: Christian Berkhoff Date: Mon, 28 Oct 2024 15:37:33 -0700 Subject: [PATCH 4/6] Clippy --- ipa-core/src/net/test.rs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/ipa-core/src/net/test.rs b/ipa-core/src/net/test.rs index 28035c726..14177d524 100644 --- a/ipa-core/src/net/test.rs +++ b/ipa-core/src/net/test.rs @@ -24,7 +24,7 @@ use crate::{ TlsConfig, }, executor::{IpaJoinHandle, IpaRuntime}, - helpers::{repeat_n, HandlerBox, HelperIdentity, RequestHandler, TransportIdentity}, + helpers::{HandlerBox, HelperIdentity, RequestHandler, TransportIdentity}, hpke::{Deserializable as _, IpaPublicKey}, net::{ClientIdentity, Helper, MpcHelperClient, MpcHelperServer}, sharding::{ShardIndex, ShardedHelperIdentity}, @@ -366,7 +366,7 @@ impl TestConfigBuilder { let ports = ports_by_ring[ix.as_index()].clone(); ports.ring.into_iter().map(Some).collect() } else { - repeat_n(None, 3).collect() + crate::helpers::repeat_n(None, 3).collect() } } @@ -379,7 +379,7 @@ impl TestConfigBuilder { .map(Some) .collect() } else { - repeat_n(None, self.shard_count.try_into().unwrap()).collect() + crate::helpers::repeat_n(None, self.shard_count.try_into().unwrap()).collect() } } @@ -696,7 +696,7 @@ mod tests { use crate::{ helpers::HelperIdentity, net::test::{Ports, TEST_CERTS, TEST_KEYS}, - sharding::ShardedHelperIdentity, + sharding::{ShardIndex, ShardedHelperIdentity}, }; /// This simple test makes sure that testing networks are created properly. @@ -757,4 +757,23 @@ mod tests { assert_eq!(TEST_KEYS[0], k); assert_eq!(TEST_CERTS[0], c); } + + #[test] + fn get_default_ports() { + let builder = TestConfigBuilder::with_http_and_default_test_ports(); + assert_eq!( + vec![Some(3000), Some(3001), Some(3002)], + builder.get_ports_for_shard_index(ShardIndex(0)) + ); + assert_eq!( + vec![Some(6001)], + builder.get_ports_for_helper_identity(HelperIdentity::TWO) + ); + } + + #[test] + fn get_os_ports() { + let builder = TestConfigBuilder::default(); + assert_eq!(3, builder.get_ports_for_shard_index(ShardIndex(0)).len()); + } } From 29a8a4cacba0842e16fd1694ab43a713c195cf84 Mon Sep 17 00:00:00 2001 From: Christian Berkhoff Date: Mon, 28 Oct 2024 16:20:13 -0700 Subject: [PATCH 5/6] Cannot use repeat_n --- ipa-core/src/net/test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipa-core/src/net/test.rs b/ipa-core/src/net/test.rs index 14177d524..1e31a2f33 100644 --- a/ipa-core/src/net/test.rs +++ b/ipa-core/src/net/test.rs @@ -366,7 +366,7 @@ impl TestConfigBuilder { let ports = ports_by_ring[ix.as_index()].clone(); ports.ring.into_iter().map(Some).collect() } else { - crate::helpers::repeat_n(None, 3).collect() + vec![None; 3] } } @@ -379,7 +379,7 @@ impl TestConfigBuilder { .map(Some) .collect() } else { - crate::helpers::repeat_n(None, self.shard_count.try_into().unwrap()).collect() + vec![None; self.shard_count.try_into().unwrap()] } } From 29258c5d02e4d1dc2a9d20afafa9b605866e3da3 Mon Sep 17 00:00:00 2001 From: Christian Berkhoff Date: Tue, 29 Oct 2024 10:52:19 -0700 Subject: [PATCH 6/6] Nit: Some --- ipa-core/src/net/test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipa-core/src/net/test.rs b/ipa-core/src/net/test.rs index 1e31a2f33..e39c0c109 100644 --- a/ipa-core/src/net/test.rs +++ b/ipa-core/src/net/test.rs @@ -47,6 +47,7 @@ pub const DEFAULT_TEST_PORTS: Ports = Ports { /// Configuration of a server that can be reached via socket or port. pub struct AddressableTestServer { + /// The identity of this server in the network. pub id: ShardedHelperIdentity, /// Contains the ports pub config: ServerConfig, @@ -375,8 +376,7 @@ impl TestConfigBuilder { if let Some(ports_by_ring) = &self.ports_by_ring { ports_by_ring .iter() - .map(|r| r.shards[id.as_index()]) - .map(Some) + .map(|r| Some(r.shards[id.as_index()])) .collect() } else { vec![None; self.shard_count.try_into().unwrap()]