Merge pull request #83 from golemcloud/sharding_tests_sync

Sharding tests sync

.github/workflows/ci.yaml

@@ -44,13 +44,21 @@ jobs:
         run: cargo clippy -- -Dwarnings
       - name: Build
         run: cargo build
-      - name: Tests
+      - name: Integration tests
         env:
           GOLEM_DOCKER_SERVICES: true
           GOLEM_TEST_TEMPLATES: ./test-templates
           RUST_LOG: info
           CI: true
-        run: cargo test
+        run: cargo test --test integration
+      - name: Sharding tests
+        env:
+          GOLEM_DOCKER_SERVICES: true
+          GOLEM_TEST_TEMPLATES: ./test-templates
+          RUST_LOG: info
+          CI: true
+        run: cargo test --test sharding
+        timeout-minutes: 10
   publish:
     needs: [build]
     if: "startsWith(github.ref, 'refs/tags/v')"

CONTRIBUTING.md

@@ -9,13 +9,20 @@ You can run all tests with
 
 To run individual tests you should first build all executables with `./scripts/build-all.sh` and then run tests by name:
 ```shell
-GOLEM_DOCKER_SERVICES=true GOLEM_TEST_TEMPLATES="./test-templates" cargo test worker_new_instance
+GOLEM_DOCKER_SERVICES=true GOLEM_TEST_TEMPLATES="./test-templates" cargo test --test integration worker_new_instance
 ```
 
 With `QUIET=true` you can hide services output:
 ```shell
-QUIET=true GOLEM_DOCKER_SERVICES=true GOLEM_TEST_TEMPLATES="./test-templates"  cargo test
+QUIET=true GOLEM_DOCKER_SERVICES=true GOLEM_TEST_TEMPLATES="./test-templates"  cargo test --test integration
 ```
 
 This way tests will use configured versions of golem docker images.
 To run tests against the latest binaries without docker - see [`golem-services` CONTRIBUTING.md](https://github.com/golemcloud/golem-services/blob/main/CONTRIBUTING.md)
+
+## Running sharding tests
+
+Same as integration tests, but with `--test sharding` instead of `--test integration`:
+```shell
+QUIET=true GOLEM_DOCKER_SERVICES=true GOLEM_TEST_TEMPLATES="./test-templates"  cargo test --test sharding
+```

Cargo.toml

@@ -14,6 +14,11 @@ name = "integration"
 path = "tests/main.rs"
 harness = false
 
+[[test]]
+name = "sharding"
+path = "tests/sharding.rs"
+harness = false
+
 [dependencies]
 async-trait = "0.1.76"
 chrono = { version = "0.4.31", features = ["serde"] }
@@ -46,6 +51,7 @@ h2 = "0.3.24"
 hyper = "0.14.28"
 tower = "0.4.13"
 testcontainers-modules = { version = "0.3.2", features = ["postgres", "redis"] }
+rand = "0.8.5"
 
 [dev-dependencies]
 env_logger = "0.11.1"
@@ -59,6 +65,8 @@ testcontainers-modules = { version = "0.3.2", features = ["postgres", "redis"] }
 tokio-postgres = "0.7.10"
 tonic = "0.10.2"
 tonic-health = "0.10.2"
+rand = "0.8.5"
+async-recursion = "1.0.5"
 
 [features]
 default = ["stubgen"]

scripts/it.sh

@@ -6,4 +6,4 @@ script_full_path=$(dirname "$0")
 cd "${script_full_path}"/.. || exit
 
 ./scripts/build-all.sh
-GOLEM_DOCKER_SERVICES=true GOLEM_TEST_TEMPLATES="./test-templates" RUST_LOG=info cargo test
+GOLEM_DOCKER_SERVICES=true GOLEM_TEST_TEMPLATES="./test-templates" RUST_LOG=info cargo test --test integration

tests/context.rs

@@ -16,7 +16,7 @@ use std::path::PathBuf;
 use testcontainers::clients;
 
 const NETWORK: &str = "golem_test_network";
-const TAG: &str = "v0.0.63";
+const TAG: &str = "v0.0.67";
 
 #[derive(Debug, Clone)]
 pub struct EnvConfig {
@@ -27,6 +27,7 @@ pub struct EnvConfig {
     pub wasm_root: PathBuf,
     pub local_golem: bool,
     pub db_type: DbType,
+    pub n_worker_executors: usize,
 }
 
 #[derive(Debug, Clone)]
@@ -49,7 +50,7 @@ impl DbType {
 }
 
 impl EnvConfig {
-    pub fn from_env() -> EnvConfig {
+    pub fn from_env_with_shards(shards: usize) -> EnvConfig {
         EnvConfig {
             verbose: std::env::var("CI").is_err(),
             on_ci: std::env::var("CI").is_ok(),
@@ -60,24 +61,28 @@ impl EnvConfig {
             ),
             local_golem: std::env::var("GOLEM_DOCKER_SERVICES").is_err(),
             db_type: DbType::from_env(),
+            n_worker_executors: shards,
         }
     }
+
+    pub fn from_env() -> EnvConfig {
+        Self::from_env_with_shards(3)
+    }
 }
 
 pub struct Context<'docker_client> {
-    env: EnvConfig,
-    db: Db<'docker_client>,
-    redis: Redis<'docker_client>,
-    shard_manager: ShardManager<'docker_client>,
-    golem_template_service: GolemTemplateService<'docker_client>,
-    golem_worker_service: GolemWorkerService<'docker_client>,
-    worker_executors: WorkerExecutors<'docker_client>,
+    pub docker: &'docker_client clients::Cli,
+    pub env: EnvConfig,
+    pub db: Db<'docker_client>,
+    pub redis: Redis<'docker_client>,
+    pub shard_manager: Option<ShardManager<'docker_client>>,
+    pub golem_template_service: GolemTemplateService<'docker_client>,
+    pub golem_worker_service: GolemWorkerService<'docker_client>,
+    pub worker_executors: WorkerExecutors<'docker_client>,
 }
 
 impl Context<'_> {
-    pub fn start(docker: &clients::Cli) -> Result<Context, Failed> {
-        let env_config = EnvConfig::from_env();
-
+    pub fn start(docker: &clients::Cli, env_config: EnvConfig) -> Result<Context, Failed> {
         println!("Starting context with env config: {env_config:?}");
 
         let db = Db::start(docker, &env_config)?;
@@ -105,10 +110,11 @@ impl Context<'_> {
         )?;
 
         Ok(Context {
+            docker,
             env: env_config,
             db,
             redis,
-            shard_manager,
+            shard_manager: Some(shard_manager),
             golem_template_service,
             golem_worker_service,
             worker_executors,
@@ -120,7 +126,7 @@ impl Context<'_> {
             env: self.env.clone(),
             db: self.db.info(),
             redis: self.redis.info(),
-            shard_manager: self.shard_manager.info(),
+            shard_manager: self.shard_manager.as_ref().unwrap().info(),
             golem_template_service: self.golem_template_service.info(),
             golem_worker_service: self.golem_worker_service.info(),
             worker_executors: self.worker_executors.info(),

tests/context/worker.rs

@@ -17,6 +17,7 @@ use tonic_health::pb::health_client::HealthClient;
 use tonic_health::pb::HealthCheckRequest;
 
 pub struct WorkerExecutor<'docker_client> {
+    pub shard_id: u16,
     host: String,
     port: u16,
     inner: WorkerExecutorInner<'docker_client>,
@@ -135,6 +136,7 @@ impl<'docker_client> WorkerExecutor<'docker_client> {
         let node = docker.run(image);
 
         Ok(WorkerExecutor {
+            shard_id,
             host: name,
             port,
             inner: WorkerExecutorInner::Docker(node),
@@ -264,6 +266,7 @@ impl<'docker_client> WorkerExecutor<'docker_client> {
         println!("Worker Executor {shard_id} online");
 
         Ok(WorkerExecutor {
+            shard_id,
             host: "localhost".to_string(),
             port,
             inner: WorkerExecutorInner::Process(child),
@@ -365,7 +368,7 @@ impl Drop for WorkerExecutor<'_> {
 }
 
 pub struct WorkerExecutors<'docker_client> {
-    worker_executors: Vec<WorkerExecutor<'docker_client>>,
+    pub worker_executors: Vec<WorkerExecutor<'docker_client>>,
 }
 
 impl<'docker_client> WorkerExecutors<'docker_client> {
@@ -377,7 +380,7 @@ impl<'docker_client> WorkerExecutors<'docker_client> {
         template: &GolemTemplateServiceInfo,
         shard_manager: &ShardManagerInfo,
     ) -> Result<WorkerExecutors<'docker_client>, Failed> {
-        let shards = 3;
+        let shards = env_config.n_worker_executors;
 
         let mut worker_executors = Vec::with_capacity(shards);
 

tests/main.rs

@@ -1,4 +1,4 @@
-use crate::context::Context;
+use crate::context::{Context, EnvConfig};
 use libtest_mimic::{Arguments, Conclusion, Failed};
 use std::sync::Arc;
 use testcontainers::clients;
@@ -25,7 +25,7 @@ fn main() -> Result<(), Failed> {
     env_logger::init();
 
     let docker = clients::Cli::default();
-    let context = Context::start(&docker)?;
+    let context = Context::start(&docker, EnvConfig::from_env())?;
 
     let res = run(&context);