chore: add support for running pb client without cgroups

docker/dev-full/docker-compose.yml

@@ -60,7 +60,6 @@ services:
     command: -c /etc/rivet-client/config.yaml
     environment:
       - RUST_BACKTRACE=1
-    privileged: true
     stop_grace_period: 0s
     volumes:
       - ./rivet-client:/etc/rivet-client:ro

docker/dev-full/rivet-client/config.yaml

@@ -1,22 +1,23 @@
 client:
-  runtime:
+  runner:
     flavor: isolate
+    # Cgroups require privileged containers
+    use_cgroup: false
   cluster:
     # This is safe to hardcode
     client_id: fc67e54e-5d6a-4726-ab23-77b0e54f068f
     # This is hardcoded to match the value in rivet-server/config.yaml
     datacenter_id: f288913c-735d-4188-bf9b-2fcf6eac7b9c
     api_endpoint: http://rivet-server:8080
     pegboard_endpoint: ws://rivet-server:8082
-  actor:
-    network:
-      bind_ip: 127.0.0.1
-      lan_ip: 127.0.0.1
-      # Point to lcoalhost since this is a dev instance
-      wan_ip: 127.0.0.1
-      # Corresponds to the port range configured in the `docker-compose.yml`
-      wan_port_range_min: 20000
-      wan_port_range_max: 20100
+  network:
+    bind_ip: 127.0.0.1
+    lan_ip: 127.0.0.1
+    # Point to lcoalhost since this is a dev instance
+    wan_ip: 127.0.0.1
+    # Corresponds to the port range configured in the `docker-compose.yml`
+    wan_port_range_min: 20000
+    wan_port_range_max: 20100
   cni:
     network_interface: eth0
   # Don't reserve any resources on dev instances

docker/dev-monolith/README.md

@@ -15,7 +15,7 @@ docker build -f docker/dev-monolith/Dockerfile -t rivet .
 ### Run via `docker run`
 
 ```bash
-docker run -v "$(pwd)/rivet-data:/data" -p 8080:8080 -p 9000:9000 -p 20000-20100:20000-20100 --privileged rivet
+docker run -v "$(pwd)/rivet-data:/data" -p 8080:8080 -p 9000:9000 -p 20000-20100:20000-20100 rivet
 ```
 
 ### Run via Docker Compose
@@ -24,7 +24,6 @@ docker run -v "$(pwd)/rivet-data:/data" -p 8080:8080 -p 9000:9000 -p 20000-20100
 services:
   rivet:
     image: rivet
-    privileged: true
     volumes:
       - rivet-data:/data
     ports:
@@ -63,7 +62,7 @@ vector top --url http://0.0.0.0:9510/graphql
 ## Testing
 
 ```bash
-docker build -f docker/dev-monolith/Dockerfile -t rivet . && docker run --name rivet --rm -v "/tmp/rivet-data:/data" -p 8080:8080 -p 9000:9000 -p 20000-20100:20000-20100 --privileged rivet
+docker build -f docker/dev-monolith/Dockerfile -t rivet . && docker run --name rivet --rm -v "/tmp/rivet-data:/data" -p 8080:8080 -p 9000:9000 -p 20000-20100:20000-20100 rivet
 ```
 
 ## Port collisions

docker/dev-monolith/rivet-client/config.yaml

@@ -1,23 +1,24 @@
 client:
-  runtime:
+  data_dir: /data/rivet-client
+  runner:
     flavor: isolate
-    data_dir: /data/rivet-client
+    # Cgroups require privileged containers
+    use_cgroup: false
   cluster:
     # This is safe to hardcode
     client_id: fc67e54e-5d6a-4726-ab23-77b0e54f068f
     # This is hardcoded to match the value in rivet-server/config.yaml
     datacenter_id: f288913c-735d-4188-bf9b-2fcf6eac7b9c
     api_endpoint: http://rivet-server:8080
     pegboard_endpoint: ws://rivet-server:8082
-  actor:
-    network:
-      bind_ip: 127.0.0.1
-      lan_ip: 127.0.0.1
-      # Point to lcoalhost since this is a dev instance
-      wan_ip: 127.0.0.1
-      # Corresponds to the port range configured in the `docker-compose.yml`
-      wan_port_range_min: 20000
-      wan_port_range_max: 20100
+  network:
+    bind_ip: 127.0.0.1
+    lan_ip: 127.0.0.1
+    # Point to lcoalhost since this is a dev instance
+    wan_ip: 127.0.0.1
+    # Corresponds to the recommended port range for the `docker run` command
+    wan_port_range_min: 20000
+    wan_port_range_max: 20100
   cni:
     network_interface: eth0
   # Don't reserve any resources on dev instances

docker/dev-monolith/scripts/setup_s6.ts

@@ -1,7 +1,6 @@
 #!/usr/bin/env -S deno run --allow-net --allow-env --allow-read --allow-write
 
 import { resolve } from "@std/path";
-import { exists } from "@std/fs";
 import dedent from "dedent";
 
 interface Service {

packages/infra/client/isolate-v8-runner/src/isolate.rs

@@ -40,7 +40,8 @@ pub fn run(actors_path: PathBuf, actor_id: Uuid, stop_rx: watch::Receiver<()>) -
 	let (shutdown_tx, shutdown_rx) = mpsc::sync_channel(1);
 
 	// Start log shipper
-	let (msg_tx, log_shipper_thread) = if let Some(vector_socket_addr) = &config.vector_socket_addr {
+	let (msg_tx, log_shipper_thread) = if let Some(vector_socket_addr) = &config.vector_socket_addr
+	{
 		let (msg_tx, msg_rx) = mpsc::sync_channel::<log_shipper::ReceivedMessage>(
 			log_shipper::MAX_BUFFER_BYTES / log_shipper::MAX_LINE_BYTES,
 		);

packages/infra/client/isolate-v8-runner/src/log_shipper.rs

@@ -1,6 +1,6 @@
 use std::{
 	io::{BufRead, Write},
-	net::{ TcpStream},
+	net::TcpStream,
 	sync::mpsc,
 	thread::JoinHandle,
 	time::{Duration, SystemTime, UNIX_EPOCH},

packages/infra/client/manager/src/actor/mod.rs

@@ -166,9 +166,10 @@ impl Actor {
 			protocol::ImageKind::DockerImage | protocol::ImageKind::OciBundle => {
 				runner::Handle::spawn_orphaned(
 					runner::Comms::Basic,
-					&ctx.config().runtime.container_runner_binary_path(),
+					&ctx.config().runner.container_runner_binary_path(),
 					ctx.actor_path(self.actor_id),
 					&runner_env,
+					ctx.config().runner.use_cgroup(),
 				)?
 			}
 			// Shared runner

packages/infra/client/manager/src/actor/setup.rs

@@ -525,15 +525,15 @@ impl Actor {
 				ctx,
 				self.actor_id,
 				&gg_ports,
-				ctx.config().actor.network.lan_port_range_min()
-					..=ctx.config().actor.network.lan_port_range_max()
+				ctx.config().network.lan_port_range_min()
+					..=ctx.config().network.lan_port_range_max()
 			),
 			bind_ports_inner(
 				ctx,
 				self.actor_id,
 				&host_ports,
-				ctx.config().actor.network.wan_port_range_min()
-					..=ctx.config().actor.network.wan_port_range_max()
+				ctx.config().network.wan_port_range_min()
+					..=ctx.config().network.wan_port_range_max()
 			),
 		)?;
 
@@ -558,7 +558,7 @@ impl Actor {
 							source: host_port,
 							// When no target port was selected, default to randomly selected host port
 							target: port.target.unwrap_or(host_port),
-							ip: ctx.config().actor.network.bind_ip,
+							ip: ctx.config().network.bind_ip,
 							protocol: port.protocol,
 						},
 					)
@@ -574,7 +574,7 @@ impl Actor {
 								source: host_port,
 								// When no target port was selected, default to randomly selected host port
 								target: port.target.unwrap_or(host_port),
-								ip: ctx.config().actor.network.bind_ip,
+								ip: ctx.config().network.bind_ip,
 								protocol: port.protocol,
 							},
 						)

packages/infra/client/manager/src/config.rs

@@ -22,16 +22,14 @@ impl Config {
 	/// This holds information that the server needs in order to orchestrate nodes.
 	pub fn build_client_config(&self) -> pegboard::client_config::ClientConfig {
 		pegboard::client_config::ClientConfig {
-			actor: pegboard::client_config::Actor {
-				network: pegboard::client_config::ActorNetwork {
-					bind_ip: self.client.actor.network.bind_ip.to_string(),
-					lan_ip: self.client.actor.network.lan_ip.to_string(),
-					wan_ip: self.client.actor.network.wan_ip.to_string(),
-					lan_port_range_min: self.client.actor.network.lan_port_range_min(),
-					lan_port_range_max: self.client.actor.network.lan_port_range_max(),
-					wan_port_range_min: self.client.actor.network.wan_port_range_min(),
-					wan_port_range_max: self.client.actor.network.wan_port_range_max(),
-				},
+			network: pegboard::client_config::Network {
+				bind_ip: self.client.network.bind_ip.to_string(),
+				lan_ip: self.client.network.lan_ip.to_string(),
+				wan_ip: self.client.network.wan_ip.to_string(),
+				lan_port_range_min: self.client.network.lan_port_range_min(),
+				lan_port_range_max: self.client.network.lan_port_range_max(),
+				wan_port_range_min: self.client.network.wan_port_range_min(),
+				wan_port_range_max: self.client.network.wan_port_range_max(),
 			},
 			reserved_resources: pegboard::client_config::ReservedResources {
 				cpu: self.client.reserved_resources.cpu(),
@@ -44,9 +42,10 @@ impl Config {
 #[derive(Clone, Deserialize)]
 #[serde(rename_all = "snake_case", deny_unknown_fields)]
 pub struct Client {
+	pub data_dir: Option<PathBuf>,
 	pub cluster: Cluster,
-	pub runtime: Runtime,
-	pub actor: Actor,
+	pub runner: Runner,
+	pub network: Network,
 	#[serde(default)]
 	pub cni: Cni,
 	#[serde(default)]
@@ -59,6 +58,14 @@ pub struct Client {
 	pub vector: Option<Vector>,
 }
 
+impl Client {
+	pub fn data_dir(&self) -> PathBuf {
+		self.data_dir
+			.clone()
+			.unwrap_or_else(|| Path::new("/var/lib/rivet-client").to_path_buf())
+	}
+}
+
 #[derive(Clone, Deserialize)]
 #[serde(rename_all = "snake_case", deny_unknown_fields)]
 pub struct Cluster {
@@ -70,18 +77,28 @@ pub struct Cluster {
 
 #[derive(Clone, Deserialize)]
 #[serde(rename_all = "snake_case", deny_unknown_fields)]
-pub struct Runtime {
+pub struct Runner {
 	pub flavor: protocol::ClientFlavor,
-	pub data_dir: Option<PathBuf>,
+
+	/// WebSocket Port for runners on this machine to connect to.
+	pub port: Option<u16>,
+
+	/// If true, a cgroup will be created for the runners.
+	///
+	/// This should only be disabled for testing & development.
+	pub use_cgroup: Option<bool>,
+
 	pub container_runner_binary_path: Option<PathBuf>,
 	pub isolate_runner_binary_path: Option<PathBuf>,
 }
 
-impl Runtime {
-	pub fn data_dir(&self) -> PathBuf {
-		self.data_dir
-			.clone()
-			.unwrap_or_else(|| Path::new("/var/lib/rivet-client").to_path_buf())
+impl Runner {
+	pub fn port(&self) -> u16 {
+		self.port.unwrap_or(7080)
+	}
+
+	pub fn use_cgroup(&self) -> bool {
+		self.use_cgroup.unwrap_or(true)
 	}
 
 	pub fn container_runner_binary_path(&self) -> PathBuf {
@@ -99,22 +116,7 @@ impl Runtime {
 
 #[derive(Clone, Deserialize)]
 #[serde(rename_all = "snake_case", deny_unknown_fields)]
-pub struct Actor {
-	pub network: ActorNetwork,
-
-	/// WebSocket Port for runners on this machine to connect to.
-	pub runner_port: Option<u16>,
-}
-
-impl Actor {
-	pub fn runner_port(&self) -> u16 {
-		self.runner_port.unwrap_or(7080)
-	}
-}
-
-#[derive(Clone, Deserialize)]
-#[serde(rename_all = "snake_case", deny_unknown_fields)]
-pub struct ActorNetwork {
+pub struct Network {
 	/// Address to serve actor traffic on.
 	///
 	/// This will usually be the same as `actor_lan_ip` unless the node is accessed within the
@@ -137,7 +139,7 @@ pub struct ActorNetwork {
 	pub wan_port_range_max: Option<u16>,
 }
 
-impl ActorNetwork {
+impl Network {
 	pub fn lan_port_range_min(&self) -> u16 {
 		self.lan_port_range_min.unwrap_or(20000)
 	}

packages/infra/client/manager/src/ctx.rs

@@ -168,9 +168,9 @@ impl Ctx {
 		// Start runner socket
 		let self2 = self.clone();
 		let runner_socket = tokio::spawn(async move {
-			tracing::warn!(port=%self2.config().actor.runner_port(), "listening for runner sockets");
+			tracing::warn!(port=%self2.config().runner.port(), "listening for runner sockets");
 
-			let listener = TcpListener::bind(("0.0.0.0", self2.config().actor.runner_port()))
+			let listener = TcpListener::bind(("0.0.0.0", self2.config().runner.port()))
 				.await
 				.map_err(RuntimeError::RunnerSocketListenFailed)?;
 
@@ -341,17 +341,24 @@ impl Ctx {
 		} else {
 			tracing::info!("spawning new isolate runner");
 
-			let env = vec![(
-				"ACTORS_PATH",
-				self.actors_path().to_str().context("bad path")?.to_string(),
-			), ("RUNNER_ADDR", format!("127.0.0.1:{}", self.config().actor.runner_port()))];
+			let env = vec![
+				(
+					"ACTORS_PATH",
+					self.actors_path().to_str().context("bad path")?.to_string(),
+				),
+				(
+					"RUNNER_ADDR",
+					format!("127.0.0.1:{}", self.config().runner.port()),
+				),
+			];
 			let working_path = self.isolate_runner_path();
 
 			let runner = runner::Handle::spawn_orphaned(
 				runner::Comms::socket(),
-				&self.config().runtime.isolate_runner_binary_path(),
+				&self.config().runner.isolate_runner_binary_path(),
 				working_path,
 				&env,
+				self.config().runner.use_cgroup(),
 			)?;
 			let pid = runner.pid();
 
@@ -585,15 +592,15 @@ impl Ctx {
 	}
 
 	pub fn actors_path(&self) -> PathBuf {
-		self.config().runtime.data_dir().join("actors")
+		self.config().data_dir().join("actors")
 	}
 
 	pub fn actor_path(&self, actor_id: Uuid) -> PathBuf {
 		self.actors_path().join(actor_id.to_string())
 	}
 
 	pub fn isolate_runner_path(&self) -> PathBuf {
-		self.config().runtime.data_dir().join("runner")
+		self.config().data_dir().join("runner")
 	}
 }