From 907452e70300cb4a64e041e496d46d19ddf631a6 Mon Sep 17 00:00:00 2001 From: d0pare <4062518+d0pare@users.noreply.github.com> Date: Thu, 25 Apr 2024 11:48:18 -0400 Subject: [PATCH] Add new test for multi tenancy support with multiple hosts --- ...ew_tenant_databases_with_multiple_hosts.cs | 136 ++++++++++++++++++ src/Wolverine/TestingExtensions.cs | 13 +- 2 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 src/Persistence/MartenTests/MultiTenancy/dynamically_spin_up_new_durability_agents_for_new_tenant_databases_with_multiple_hosts.cs diff --git a/src/Persistence/MartenTests/MultiTenancy/dynamically_spin_up_new_durability_agents_for_new_tenant_databases_with_multiple_hosts.cs b/src/Persistence/MartenTests/MultiTenancy/dynamically_spin_up_new_durability_agents_for_new_tenant_databases_with_multiple_hosts.cs new file mode 100644 index 000000000..1ceef3fe5 --- /dev/null +++ b/src/Persistence/MartenTests/MultiTenancy/dynamically_spin_up_new_durability_agents_for_new_tenant_databases_with_multiple_hosts.cs @@ -0,0 +1,136 @@ +using IntegrationTests; +using JasperFx.Core; +using Marten; +using Marten.Storage; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Npgsql; +using Weasel.Postgresql; +using Weasel.Postgresql.Migrations; +using Wolverine; +using Wolverine.Marten; +using Wolverine.RDBMS; +using Xunit.Abstractions; + +namespace MartenTests.MultiTenancy; + +public class dynamically_spin_up_new_durability_agents_for_new_tenant_databases_with_multiple_hosts : IAsyncLifetime +{ + private readonly ITestOutputHelper _testOutputHelper; + private IHost _host1; + private IHost _host2; + private IDocumentStore theStore; + private string tenant1ConnectionString; + private string tenant2ConnectionString; + private string tenant3ConnectionString; + private string tenant4ConnectionString; + + public dynamically_spin_up_new_durability_agents_for_new_tenant_databases_with_multiple_hosts(ITestOutputHelper testOutputHelper) + { + _testOutputHelper = testOutputHelper; + } + + private async Task CreateDatabaseIfNotExists(NpgsqlConnection conn, string databaseName) + { + var builder = new NpgsqlConnectionStringBuilder(Servers.PostgresConnectionString); + + var exists = await conn.DatabaseExists(databaseName); + if (!exists) + { + await new DatabaseSpecification().BuildDatabase(conn, databaseName); + } + + builder.Database = databaseName; + + return builder.ConnectionString; + } + + public async Task InitializeAsync() + { + await using var conn = new NpgsqlConnection(Servers.PostgresConnectionString); + await conn.OpenAsync(); + + await conn.DropSchemaAsync("tenants"); + + + tenant1ConnectionString = await CreateDatabaseIfNotExists(conn, "tenant1"); + tenant2ConnectionString = await CreateDatabaseIfNotExists(conn, "tenant2"); + tenant3ConnectionString = await CreateDatabaseIfNotExists(conn, "tenant3"); + tenant4ConnectionString = await CreateDatabaseIfNotExists(conn, "tenant4"); + + await conn.CloseAsync(); + + // Setting up Hosts with Multi-tenancy + _host1 = await CreateHostAsync(); + _host2 = await CreateHostAsync(); + + theStore = _host1.Services.GetRequiredService(); + + var tenancy = (MasterTableTenancy)theStore.Options.Tenancy; + await tenancy.ClearAllDatabaseRecordsAsync(); + + async Task CreateHostAsync() + { + return await Host.CreateDefaultBuilder() + .UseWolverine(opts => + { + // This is too extreme for real usage, but helps tests to run faster + opts.Durability.NodeReassignmentPollingTime = 1.Seconds(); + opts.Durability.HealthCheckPollingTime = 1.Seconds(); + + opts.Services.AddMarten(o => + { + // This is a new strategy for configuring tenant databases with Marten + // In this usage, Marten is tracking the tenant databases in a single table in the "master" + // database by tenant + o.MultiTenantedDatabasesWithMasterDatabaseTable(Servers.PostgresConnectionString, "tenants"); + }) + .IntegrateWithWolverine("mt", masterDatabaseConnectionString: Servers.PostgresConnectionString) + + // All detected changes will be applied to all + // the configured tenant databases on startup + .ApplyAllDatabaseChangesOnStartup(); + }) + .StartAsync(); + } + } + + public async Task DisposeAsync() + { + await _host1.StopAsync(); + _host1.Dispose(); + await _host2.StopAsync(); + _host2.Dispose(); + } + + [Fact] + public async Task add_databases_and_see_durability_agents_start() + { + await _host1.WaitUntilAssignmentsChangeTo(w => + { + w.AgentScheme = DurabilityAgent.AgentScheme; + w.CountTotal = true; + + // 1 for the master + w.ExpectRunningAgents(_host1, 1); + w.ExpectRunningAgents(_host2, 0); + }, 10.Seconds()); + + + var tenancy = (MasterTableTenancy)theStore.Options.Tenancy; + await tenancy.AddDatabaseRecordAsync("tenant1", tenant1ConnectionString); + await tenancy.AddDatabaseRecordAsync("tenant2", tenant2ConnectionString); + await tenancy.AddDatabaseRecordAsync("tenant3", tenant3ConnectionString); + + await _host1.WaitUntilAssignmentsChangeTo(w => + { + w.AgentScheme = DurabilityAgent.AgentScheme; + w.CountTotal = true; + + // 1 for the master, 3 for the tenant databases + w.ExpectRunningAgents(_host1, 4); + w.ExpectRunningAgents(_host2, 0); + }, 1.Minutes()); + + } +} \ No newline at end of file diff --git a/src/Wolverine/TestingExtensions.cs b/src/Wolverine/TestingExtensions.cs index 7c187a6ef..bb1322a61 100644 --- a/src/Wolverine/TestingExtensions.cs +++ b/src/Wolverine/TestingExtensions.cs @@ -286,6 +286,9 @@ public class AssignmentWaiter private readonly WolverineRuntime _leaderRuntime; public string AgentScheme { get; set; } + + // For multiple host agent assignments can vary, so we may need to rely on total count + public bool CountTotal { get; set; } public AssignmentWaiter(IHost leader) { @@ -374,16 +377,20 @@ public bool HasReached() Func filter = AgentScheme.IsEmpty() ? x => !x.Scheme.StartsWith("wolverine") : x => x.Scheme.EqualsIgnoreCase(AgentScheme); + + var difference = 0; foreach (var pair in AgentCountByHost) { var runtime = _runtimes[pair.Key]; var runningCount = runtime.AllRunningAgentUris().Count(x => filter(x)); - if (pair.Value != runningCount) return false; + if (!CountTotal && pair.Value != runningCount) return false; + + difference += pair.Value - runningCount; } - - return true; + + return !CountTotal || difference == 0; } }