Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pool/snow-902610 min pool size #880

Merged
merged 7 commits into from
Apr 3, 2024
Merged

Pool/snow-902610 min pool size #880

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8fa2771
SNOW-902610 Min pool size
sfc-gh-knozderko Mar 1, 2024
8b85405
create sessions to reach min pool size on connection close
sfc-gh-knozderko Mar 27, 2024
95c030e
remove immutable lists, add awaiter tests
sfc-gh-knozderko Mar 28, 2024
35ac1f8
limit expiration timeout
sfc-gh-knozderko Mar 28, 2024
7a5ffae
prevent from recurrent min pool size satisfying attempts
sfc-gh-knozderko Apr 2, 2024
f953c93
revert limiting of expiration timeout
sfc-gh-knozderko Apr 2, 2024
7bf841b
little renaming
sfc-gh-knozderko Apr 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions Snowflake.Data.Tests/IntegrationTests/ConnectionMultiplePoolsAsyncIT.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
using System.Threading;
using System.Threading.Tasks;
using NUnit.Framework;
using Snowflake.Data.Client;
using Snowflake.Data.Core.Session;
using Snowflake.Data.Tests.Util;

namespace Snowflake.Data.Tests.IntegrationTests
{
[TestFixture]
[NonParallelizable]
public class ConnectionMultiplePoolsAsyncIT: SFBaseTestAsync
{
private readonly PoolConfig _previousPoolConfig = new PoolConfig();

[SetUp]
public new void BeforeTest()
{
SnowflakeDbConnectionPool.SetConnectionPoolVersion(ConnectionPoolType.MultipleConnectionPool);
SnowflakeDbConnectionPool.ClearAllPools();
}

[TearDown]
public new void AfterTest()
{
_previousPoolConfig.Reset();
}

[Test]
public async Task TestMinPoolSizeAsync()
{
// arrange
var connection = new SnowflakeDbConnection();
connection.ConnectionString = ConnectionString + "application=TestMinPoolSizeAsync;minPoolSize=3";

// act
await connection.OpenAsync().ConfigureAwait(false);
Thread.Sleep(3000);

// assert
var pool = SnowflakeDbConnectionPool.GetPool(connection.ConnectionString);
Assert.AreEqual(3, pool.GetCurrentPoolSize());

// cleanup
await connection.CloseAsync(CancellationToken.None).ConfigureAwait(false);
}
}
}
127 changes: 87 additions & 40 deletions Snowflake.Data.Tests/IntegrationTests/ConnectionMultiplePoolsIT.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Data;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using NUnit.Framework;
using Snowflake.Data.Client;
Expand Down Expand Up @@ -51,37 +52,38 @@ public void TestBasicConnectionPool()
[Test]
public void TestReuseSessionInConnectionPool() // old name: TestConnectionPool
{
var conn1 = new SnowflakeDbConnection(ConnectionString);
var connectionString = ConnectionString + "minPoolSize=1";
var conn1 = new SnowflakeDbConnection(connectionString);
conn1.Open();
Assert.AreEqual(ConnectionState.Open, conn1.State);
conn1.Close();
Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize());
Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(connectionString).GetCurrentPoolSize());

var conn2 = new SnowflakeDbConnection();
conn2.ConnectionString = ConnectionString;
conn2.ConnectionString = connectionString;
conn2.Open();
Assert.AreEqual(ConnectionState.Open, conn2.State);
Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize());
Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(connectionString).GetCurrentPoolSize());

conn2.Close();
Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize());
Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(connectionString).GetCurrentPoolSize());
Assert.AreEqual(ConnectionState.Closed, conn1.State);
Assert.AreEqual(ConnectionState.Closed, conn2.State);
}

[Test]
public void TestReuseSessionInConnectionPoolReachingMaxConnections() // old name: TestConnectionPoolFull
{
var pool = SnowflakeDbConnectionPool.GetPool(ConnectionString);
pool.SetMaxPoolSize(2);
var connectionString = ConnectionString + "maxPoolSize=2;minPoolSize=1";
var pool = SnowflakeDbConnectionPool.GetPool(connectionString);

var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = ConnectionString;
conn1.ConnectionString = connectionString;
conn1.Open();
Assert.AreEqual(ConnectionState.Open, conn1.State);

var conn2 = new SnowflakeDbConnection();
conn2.ConnectionString = ConnectionString;
conn2.ConnectionString = connectionString;
conn2.Open();
Assert.AreEqual(ConnectionState.Open, conn2.State);

Expand All @@ -91,12 +93,12 @@ public void TestReuseSessionInConnectionPoolReachingMaxConnections() // old name
Assert.AreEqual(2, pool.GetCurrentPoolSize());

var conn3 = new SnowflakeDbConnection();
conn3.ConnectionString = ConnectionString;
conn3.ConnectionString = connectionString;
conn3.Open();
Assert.AreEqual(ConnectionState.Open, conn3.State);

var conn4 = new SnowflakeDbConnection();
conn4.ConnectionString = ConnectionString;
conn4.ConnectionString = connectionString;
conn4.Open();
Assert.AreEqual(ConnectionState.Open, conn4.State);

Expand All @@ -115,16 +117,16 @@ public void TestReuseSessionInConnectionPoolReachingMaxConnections() // old name
public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimit()
{
// arrange
var connectionString = ConnectionString + "application=TestWaitForMaxSize1;waitingForIdleSessionTimeout=1s;maxPoolSize=2";
var connectionString = ConnectionString + "application=TestWaitForMaxSize1;waitingForIdleSessionTimeout=1s;maxPoolSize=2;minPoolSize=1";
sfc-gh-mhofman marked this conversation as resolved.
Show resolved Hide resolved
var pool = SnowflakeDbConnectionPool.GetPool(connectionString);
Assert.AreEqual(0, pool.GetCurrentPoolSize(), "expecting pool to be empty");
var conn1 = OpenedConnection(connectionString);
var conn2 = OpenedConnection(connectionString);
var conn1 = OpenConnection(connectionString);
var conn2 = OpenConnection(connectionString);
var watch = new StopWatch();

// act
watch.Start();
var thrown = Assert.Throws<SnowflakeDbException>(() => OpenedConnection(connectionString));
var thrown = Assert.Throws<SnowflakeDbException>(() => OpenConnection(connectionString));
watch.Stop();

// assert
Expand All @@ -141,16 +143,16 @@ public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimit()
public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimitAsync()
{
// arrange
var connectionString = ConnectionString + "application=TestWaitForMaxSize2;waitingForIdleSessionTimeout=1s;maxPoolSize=2";
var connectionString = ConnectionString + "application=TestWaitForMaxSize2;waitingForIdleSessionTimeout=1s;maxPoolSize=2;minPoolSize=1";
sfc-gh-mhofman marked this conversation as resolved.
Show resolved Hide resolved
var pool = SnowflakeDbConnectionPool.GetPool(connectionString);
Assert.AreEqual(0, pool.GetCurrentPoolSize(), "expecting pool to be empty");
var conn1 = OpenedConnection(connectionString);
var conn2 = OpenedConnection(connectionString);
var conn1 = OpenConnection(connectionString);
var conn2 = OpenConnection(connectionString);
var watch = new StopWatch();

// act
watch.Start();
var thrown = Assert.ThrowsAsync<SnowflakeDbException>(() => OpenedConnectionAsync(connectionString));
var thrown = Assert.ThrowsAsync<SnowflakeDbException>(() => OpenConnectionAsync(connectionString));
watch.Stop();

// assert
Expand All @@ -170,7 +172,7 @@ public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimitAsync()
public void TestWaitInAQueueForAnIdleSession()
{
// arrange
var connectionString = ConnectionString + "application=TestWaitForMaxSize3;waitingForIdleSessionTimeout=3s;maxPoolSize=2";
var connectionString = ConnectionString + "application=TestWaitForMaxSize3;waitingForIdleSessionTimeout=3s;maxPoolSize=2;minPoolSize=0";
var pool = SnowflakeDbConnectionPool.GetPool(connectionString);
Assert.AreEqual(0, pool.GetCurrentPoolSize(), "the pool is expected to be empty");
const long ADelay = 0;
Expand Down Expand Up @@ -201,7 +203,7 @@ public void TestWaitInAQueueForAnIdleSession()

// assert
var events = threads.Events().ToList();
Assert.AreEqual(6, events.Count);
Assert.AreEqual(6, events.Count); // A,B - connected; C,D - waiting, connected
var waitingEvents = events.Where(e => e.IsWaitingEvent()).ToList();
Assert.AreEqual(2, waitingEvents.Count);
CollectionAssert.AreEquivalent(new[] { "C", "D" }, waitingEvents.Select(e => e.ThreadName)); // equivalent = in any order
Expand All @@ -224,10 +226,10 @@ public void TestWaitInAQueueForAnIdleSession()
public void TestBusyAndIdleConnectionsCountedInPoolSize()
{
// arrange
var pool = SnowflakeDbConnectionPool.GetPool(ConnectionString);
pool.SetMaxPoolSize(2);
var connectionString = ConnectionString + "maxPoolSize=2;minPoolSize=1";
sfc-gh-mhofman marked this conversation as resolved.
Show resolved Hide resolved
var pool = SnowflakeDbConnectionPool.GetPool(connectionString);
var connection = new SnowflakeDbConnection();
connection.ConnectionString = ConnectionString;
connection.ConnectionString = connectionString;

// act
connection.Open();
Expand Down Expand Up @@ -279,7 +281,7 @@ public void TestConnectionPoolDisable()
[Test]
public void TestNewConnectionPoolClean()
{
var connectionString = ConnectionString + "maxPoolSize=2;";
var connectionString = ConnectionString + "maxPoolSize=2;minPoolSize=1;";
sfc-gh-mhofman marked this conversation as resolved.
Show resolved Hide resolved
var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = connectionString;
conn1.Open();
Expand Down Expand Up @@ -313,35 +315,80 @@ public void TestNewConnectionPoolClean()
[Test]
public void TestConnectionPoolExpirationWorks()
{
var connectionString = ConnectionString + "expirationTimeout=0;maxPoolSize=2";
var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = connectionString;
conn1.Open();
conn1.Close();
// arrange
const int ExpirationTimeoutInSeconds = 10;
var connectionString = ConnectionString + $"expirationTimeout={ExpirationTimeoutInSeconds};maxPoolSize=4;minPoolSize=2";
var pool = SnowflakeDbConnectionPool.GetPool(connectionString);
Assert.AreEqual(0, pool.GetCurrentPoolSize());

var conn2 = new SnowflakeDbConnection();
conn2.ConnectionString = connectionString;
conn2.Open();
// act
var conn1 = OpenConnection(connectionString);
var conn2 = OpenConnection(connectionString);
var conn3 = OpenConnection(connectionString);
var conn4 = OpenConnection(connectionString);

// assert
Assert.AreEqual(4, pool.GetCurrentPoolSize());

// act
WaitUntilAllSessionsCreatedOrTimeout(pool);
var beforeSleepMillis = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
Thread.Sleep(TimeSpan.FromSeconds(ExpirationTimeoutInSeconds));
conn1.Close();
conn2.Close();

var conn3 = new SnowflakeDbConnection();
conn3.ConnectionString = connectionString;
conn3.Open();
conn3.Close();
conn4.Close();

// assert
Assert.AreEqual(2, pool.GetCurrentPoolSize()); // 2 idle sessions, but expired because close doesn't remove expired sessions

// act
WaitUntilAllSessionsCreatedOrTimeout(pool);
var conn5 = OpenConnection(connectionString);
WaitUntilAllSessionsCreatedOrTimeout(pool);

// assert
Assert.AreEqual(2, pool.GetCurrentPoolSize()); // 1 idle session and 1 busy
var sessionStartTimes = pool.GetIdleSessionsStartTimes();
Assert.AreEqual(1, sessionStartTimes.Count);
Assert.That(sessionStartTimes.First(), Is.GreaterThan(beforeSleepMillis));
Assert.That(conn5.SfSession.GetStartTime(), Is.GreaterThan(beforeSleepMillis));
}

[Test]
public void TestMinPoolSize()
{
// arrange
var connection = new SnowflakeDbConnection();
connection.ConnectionString = ConnectionString + "application=TestMinPoolSize;minPoolSize=3";

// act
connection.Open();
Thread.Sleep(3000);

// assert
Assert.AreEqual(0, SnowflakeDbConnectionPool.GetPool(connectionString).GetCurrentPoolSize());
var pool = SnowflakeDbConnectionPool.GetPool(connection.ConnectionString);
Assert.AreEqual(3, pool.GetCurrentPoolSize());

// cleanup
connection.Close();
}

private void WaitUntilAllSessionsCreatedOrTimeout(SessionPool pool)
{
var expectingToWaitAtMostForSessionCreations = TimeSpan.FromSeconds(15);
Awaiter.WaitUntilConditionOrTimeout(() => pool.OngoingSessionCreationsCount() == 0, expectingToWaitAtMostForSessionCreations);
}

private SnowflakeDbConnection OpenedConnection(string connectionString)
private SnowflakeDbConnection OpenConnection(string connectionString)
{
var connection = new SnowflakeDbConnection();
connection.ConnectionString = connectionString;
connection.Open();
return connection;
}

private async Task<SnowflakeDbConnection> OpenedConnectionAsync(string connectionString)
private async Task<SnowflakeDbConnection> OpenConnectionAsync(string connectionString)
{
var connection = new SnowflakeDbConnection();
connection.ConnectionString = connectionString;
Expand Down
1 change: 1 addition & 0 deletions Snowflake.Data.Tests/IntegrationTests/ConnectionPoolCommonIT.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ public void TestConnectionPoolWithDispose()
conn1.ConnectionString = "bad connection string";
Assert.Throws<SnowflakeDbException>(() => conn1.Open());
conn1.Close();
Thread.Sleep(3000); // minPoolSize = 2 causes that another thread has been started. We sleep to make that thread finish.

Assert.AreEqual(ConnectionState.Closed, conn1.State);
Assert.AreEqual(0, SnowflakeDbConnectionPool.GetPool(conn1.ConnectionString).GetCurrentPoolSize());
Expand Down
1 change: 1 addition & 0 deletions Snowflake.Data.Tests/IntegrationTests/ConnectionSinglePoolCacheIT.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Data;
using System.Data.Common;
using System.Threading;
using System.Threading.Tasks;
using NUnit.Framework;
using Snowflake.Data.Client;
Expand Down
4 changes: 2 additions & 2 deletions Snowflake.Data.Tests/UnitTests/ConnectionPoolManagerTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ namespace Snowflake.Data.Tests.UnitTests
class ConnectionPoolManagerTest
{
private readonly ConnectionPoolManager _connectionPoolManager = new ConnectionPoolManager();
private const string ConnectionString1 = "database=D1;warehouse=W1;account=A1;user=U1;password=P1;role=R1;";
private const string ConnectionString2 = "database=D2;warehouse=W2;account=A2;user=U2;password=P2;role=R2;";
private const string ConnectionString1 = "database=D1;warehouse=W1;account=A1;user=U1;password=P1;role=R1;minPoolSize=1;";
sfc-gh-mhofman marked this conversation as resolved.
Show resolved Hide resolved
private const string ConnectionString2 = "database=D2;warehouse=W2;account=A2;user=U2;password=P2;role=R2;minPoolSize=1;";
private readonly SecureString _password = new SecureString();
private static PoolConfig s_poolConfig;

Expand Down
3 changes: 1 addition & 2 deletions Snowflake.Data.Tests/UnitTests/Session/ConnectionPoolConfigExtractorTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,6 @@ public void TestExtractConnectionTimeoutFailsForWrongValue(string propertyValue)
[TestCase("TRUE", true)]
[TestCase("false", false)]
[TestCase("FALSE", false)]
// [TestCase("0", false)]
public void TestExtractPoolingEnabled(string propertyValue, bool poolingEnabled)
{
// arrange
Expand Down Expand Up @@ -256,7 +255,7 @@ public TimeoutTestCase(string propertyValue, TimeSpan expectedTimeout)

public static IEnumerable<TimeoutTestCase> CorrectTimeoutsWithZeroUnchanged() =>
CorrectTimeoutsWithoutZero().Concat(ZeroUnchangedTimeouts());

public static IEnumerable<TimeoutTestCase> CorrectTimeoutsWithZeroAsInfinite() =>
CorrectTimeoutsWithoutZero().Concat(ZeroAsInfiniteTimeouts());

Expand Down
Loading
Loading