Skip to content

Commit

Permalink
SNOW-902632 connection string driven pool config (#873)
Browse files Browse the repository at this point in the history 
### Description
SNOW-902632 connection string driven pool config

### Checklist
- [x] Code compiles correctly
- [x] Code is formatted according to [Coding
Conventions](../CodingConventions.md)
- [x] Created tests which fail without the change (if possible)
- [x] All tests passing (`dotnet test`)
- [x] Extended the README / documentation, if necessary
- [x] Provide JIRA issue id (if possible) or GitHub issue id in PR name
  • Loading branch information
@sfc-gh-knozderko
sfc-gh-knozderko authored Mar 25, 2024
1 parent 85204fc commit 052d895
Show file tree
Hide file tree
Showing 34 changed files with 1,539 additions and 366 deletions.
67 changes: 37 additions & 30 deletions README.md
View file

Large diffs are not rendered by default.

61 changes: 44 additions & 17 deletions Snowflake.Data.Tests/IntegrationTests/ConnectionMultiplePoolsIT.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ public class ConnectionMultiplePoolsIT: SFBaseTest
{
SnowflakeDbConnectionPool.SetConnectionPoolVersion(ConnectionPoolType.MultipleConnectionPool);
SnowflakeDbConnectionPool.ClearAllPools();
SnowflakeDbConnectionPool.SetPooling(true);
}

[TearDown]
Expand All @@ -35,6 +34,20 @@ public static void AfterAllTests()
SnowflakeDbConnectionPool.ClearAllPools();
}

[Test]
public void TestBasicConnectionPool()
{
var connectionString = ConnectionString + "minPoolSize=0;maxPoolSize=1";
var conn1 = new SnowflakeDbConnection(connectionString);
conn1.Open();
Assert.AreEqual(ConnectionState.Open, conn1.State);
conn1.Close();

// assert
Assert.AreEqual(ConnectionState.Closed, conn1.State);
Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(connectionString).GetCurrentPoolSize());
}

[Test]
public void TestReuseSessionInConnectionPool() // old name: TestConnectionPool
{
Expand Down Expand Up @@ -102,20 +115,16 @@ public void TestReuseSessionInConnectionPoolReachingMaxConnections() // old name
public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimit()
{
// arrange
var connectionString = ConnectionString + "application=TestWaitForMaxSize1";
var connectionString = ConnectionString + "application=TestWaitForMaxSize1;waitingForIdleSessionTimeout=1s;maxPoolSize=2";
var pool = SnowflakeDbConnectionPool.GetPool(connectionString);
Assert.AreEqual(0, pool.GetCurrentPoolSize(), "expecting pool to be empty");
pool.SetMaxPoolSize(2);
pool.SetWaitingForSessionToReuseTimeout(1000);
var conn1 = OpenedConnection(connectionString);
var conn2 = OpenedConnection(connectionString);
var watch = new StopWatch();

// act
watch.Start();
var start = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
var thrown = Assert.Throws<SnowflakeDbException>(() => OpenedConnection(connectionString));
var stop = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
watch.Stop();

// assert
Expand All @@ -132,11 +141,9 @@ public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimit()
public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimitAsync()
{
// arrange
var connectionString = ConnectionString + "application=TestWaitForMaxSize2";
var connectionString = ConnectionString + "application=TestWaitForMaxSize2;waitingForIdleSessionTimeout=1s;maxPoolSize=2";
var pool = SnowflakeDbConnectionPool.GetPool(connectionString);
Assert.AreEqual(0, pool.GetCurrentPoolSize(), "expecting pool to be empty");
pool.SetMaxPoolSize(2);
pool.SetWaitingForSessionToReuseTimeout(1000);
var conn1 = OpenedConnection(connectionString);
var conn2 = OpenedConnection(connectionString);
var watch = new StopWatch();
Expand All @@ -163,11 +170,9 @@ public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimitAsync()
public void TestWaitInAQueueForAnIdleSession()
{
// arrange
var connectionString = ConnectionString + "application=TestWaitForMaxSize3";
var connectionString = ConnectionString + "application=TestWaitForMaxSize3;waitingForIdleSessionTimeout=3s;maxPoolSize=2";
var pool = SnowflakeDbConnectionPool.GetPool(connectionString);
Assert.AreEqual(0, pool.GetCurrentPoolSize(), "the pool is expected to be empty");
pool.SetMaxPoolSize(2);
pool.SetWaitingForSessionToReuseTimeout(3000);
const long ADelay = 0;
const long BDelay = 400;
const long CDelay = 2 * BDelay;
Expand Down Expand Up @@ -238,7 +243,6 @@ public void TestBusyAndIdleConnectionsCountedInPoolSize()
}

[Test]
[Ignore("Enable when disabling pooling in connection string enabled - SNOW-902632")]
public void TestConnectionPoolNotPossibleToDisableForAllPools()
{
// act
Expand Down Expand Up @@ -275,19 +279,19 @@ public void TestConnectionPoolDisable()
[Test]
public void TestNewConnectionPoolClean()
{
SnowflakeDbConnectionPool.SetMaxPoolSize(2);
var connectionString = ConnectionString + "maxPoolSize=2;";
var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = ConnectionString;
conn1.ConnectionString = connectionString;
conn1.Open();
Assert.AreEqual(ConnectionState.Open, conn1.State);

var conn2 = new SnowflakeDbConnection();
conn2.ConnectionString = ConnectionString + " retryCount=1";
conn2.ConnectionString = connectionString + "retryCount=1";
conn2.Open();
Assert.AreEqual(ConnectionState.Open, conn2.State);

var conn3 = new SnowflakeDbConnection();
conn3.ConnectionString = ConnectionString + " retryCount=2";
conn3.ConnectionString = connectionString + "retryCount=2";
conn3.Open();
Assert.AreEqual(ConnectionState.Open, conn3.State);

Expand All @@ -305,6 +309,29 @@ public void TestNewConnectionPoolClean()
Assert.AreEqual(ConnectionState.Closed, conn2.State);
Assert.AreEqual(ConnectionState.Closed, conn3.State);
}

[Test]
public void TestConnectionPoolExpirationWorks()
{
var connectionString = ConnectionString + "expirationTimeout=0;maxPoolSize=2";
var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = connectionString;
conn1.Open();
conn1.Close();

var conn2 = new SnowflakeDbConnection();
conn2.ConnectionString = connectionString;
conn2.Open();
conn2.Close();

var conn3 = new SnowflakeDbConnection();
conn3.ConnectionString = connectionString;
conn3.Open();
conn3.Close();

// assert
Assert.AreEqual(0, SnowflakeDbConnectionPool.GetPool(connectionString).GetCurrentPoolSize());
}

private SnowflakeDbConnection OpenedConnection(string connectionString)
{
Expand Down
56 changes: 8 additions & 48 deletions Snowflake.Data.Tests/IntegrationTests/ConnectionPoolCommonIT.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@ public ConnectionPoolCommonIT(ConnectionPoolType connectionPoolTypeUnderTest)
{
SnowflakeDbConnectionPool.SetConnectionPoolVersion(_connectionPoolTypeUnderTest);
SnowflakeDbConnectionPool.ClearAllPools();
SnowflakeDbConnectionPool.SetPooling(true);
if (_connectionPoolTypeUnderTest == ConnectionPoolType.SingleConnectionCache)
{
SnowflakeDbConnectionPool.SetPooling(true);
}
s_logger.Debug($"---------------- BeforeTest ---------------------");
s_logger.Debug($"Testing Pool Type: {SnowflakeDbConnectionPool.GetConnectionPoolVersion()}");
}
Expand All @@ -49,48 +52,6 @@ public static void AfterAllTests()
{
SnowflakeDbConnectionPool.ClearAllPools();
}

[Test]
public void TestBasicConnectionPool()
{
SnowflakeDbConnectionPool.SetMaxPoolSize(1);

var conn1 = new SnowflakeDbConnection(ConnectionString);
conn1.Open();
Assert.AreEqual(ConnectionState.Open, conn1.State);
conn1.Close();

Assert.AreEqual(ConnectionState.Closed, conn1.State);
Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize());
}

[Test]
public void TestConnectionPoolExpirationWorks()
{
SnowflakeDbConnectionPool.SetMaxPoolSize(2);
SnowflakeDbConnectionPool.SetTimeout(10);

var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = ConnectionString;

conn1.Open();
conn1.Close();
SnowflakeDbConnectionPool.SetTimeout(-1);

var conn2 = new SnowflakeDbConnection();
conn2.ConnectionString = ConnectionString;
conn2.Open();
conn2.Close();
var conn3 = new SnowflakeDbConnection();
conn3.ConnectionString = ConnectionString;
conn3.Open();
conn3.Close();

// The pooling timeout should apply to all connections being pooled,
// not just the connections created after the new setting,
// so expected result should be 0
Assert.AreEqual(0, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize());
}

[Test]
public void TestConnectionPoolMultiThreading()
Expand Down Expand Up @@ -128,16 +89,15 @@ void ThreadProcess2(string connstr)
Assert.AreEqual(true, resultSet.Next());
Assert.AreEqual("1", resultSet.GetString(0));
conn1.Close();
SnowflakeDbConnectionPool.ClearAllPools();
SnowflakeDbConnectionPool.SetMaxPoolSize(0);
SnowflakeDbConnectionPool.SetPooling(true);
}

[Test]
public void TestConnectionPoolWithDispose()
{
SnowflakeDbConnectionPool.SetMaxPoolSize(1);

if (_connectionPoolTypeUnderTest == ConnectionPoolType.SingleConnectionCache)
{
SnowflakeDbConnectionPool.SetMaxPoolSize(1);
}
var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = "bad connection string";
Assert.Throws<SnowflakeDbException>(() => conn1.Open());
Expand Down
43 changes: 43 additions & 0 deletions Snowflake.Data.Tests/IntegrationTests/ConnectionSinglePoolCacheIT.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,20 @@ public static void AfterAllTests()
SnowflakeDbConnectionPool.ClearAllPools();
}

[Test]
public void TestBasicConnectionPool()
{
SnowflakeDbConnectionPool.SetMaxPoolSize(1);

var conn1 = new SnowflakeDbConnection(ConnectionString);
conn1.Open();
Assert.AreEqual(ConnectionState.Open, conn1.State);
conn1.Close();

Assert.AreEqual(ConnectionState.Closed, conn1.State);
Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize());
}

[Test]
public void TestConcurrentConnectionPooling()
{
Expand Down Expand Up @@ -256,5 +270,34 @@ public void TestConnectionPoolClean()
Assert.AreEqual(ConnectionState.Closed, conn2.State);
Assert.AreEqual(ConnectionState.Closed, conn3.State);
}

[Test]
public void TestConnectionPoolExpirationWorks()
{
SnowflakeDbConnectionPool.SetMaxPoolSize(2);
SnowflakeDbConnectionPool.SetTimeout(10);

var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = ConnectionString;

conn1.Open();
conn1.Close();
SnowflakeDbConnectionPool.SetTimeout(0);

var conn2 = new SnowflakeDbConnection();
conn2.ConnectionString = ConnectionString;
conn2.Open();
conn2.Close();

var conn3 = new SnowflakeDbConnection();
conn3.ConnectionString = ConnectionString;
conn3.Open();
conn3.Close();

// The pooling timeout should apply to all connections being pooled,
// not just the connections created after the new setting,
// so expected result should be 0
Assert.AreEqual(0, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize());
}
}
}
19 changes: 10 additions & 9 deletions Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ namespace Snowflake.Data.Tests.IntegrationTests
using System.Collections.Generic;
using System.Globalization;
using Snowflake.Data.Tests.Mock;
using Snowflake.Data.Core;

[TestFixture]
class SFDbCommandITAsync : SFBaseTestAsync
Expand Down Expand Up @@ -102,7 +101,7 @@ public void TestExecuteAsyncWithMaxRetryReached()

using (DbConnection conn = new MockSnowflakeDbConnection(mockRestRequester))
{
string maxRetryConnStr = ConnectionString + "maxHttpRetries=5";
string maxRetryConnStr = ConnectionString + "maxHttpRetries=8";

conn.ConnectionString = maxRetryConnStr;
conn.Open();
Expand All @@ -124,10 +123,11 @@ public void TestExecuteAsyncWithMaxRetryReached()
}
stopwatch.Stop();

// retry 5 times with backoff 1, 2, 4, 8, 16 seconds
var totalDelaySeconds = 1 + 2 + 4 + 8 + 16 + 16 + 16 + 16;
// retry 8 times with backoff 1, 2, 4, 8, 16, 16, 16, 16 seconds
// but should not delay more than another 16 seconds
Assert.Less(stopwatch.ElapsedMilliseconds, 51 * 1000);
Assert.GreaterOrEqual(stopwatch.ElapsedMilliseconds, 30 * 1000);
Assert.Less(stopwatch.ElapsedMilliseconds, (totalDelaySeconds + 20) * 1000);
Assert.GreaterOrEqual(stopwatch.ElapsedMilliseconds, totalDelaySeconds * 1000);
}
}
}
Expand Down Expand Up @@ -594,7 +594,7 @@ public void TestExecuteWithMaxRetryReached()

using (IDbConnection conn = new MockSnowflakeDbConnection(mockRestRequester))
{
string maxRetryConnStr = ConnectionString + "maxHttpRetries=5";
string maxRetryConnStr = ConnectionString + "maxHttpRetries=8";

conn.ConnectionString = maxRetryConnStr;
conn.Open();
Expand All @@ -615,10 +615,11 @@ public void TestExecuteWithMaxRetryReached()
}
stopwatch.Stop();

// retry 5 times with backoff 1, 2, 4, 8, 16 seconds
var totalDelaySeconds = 1 + 2 + 4 + 8 + 16 + 16 + 16 + 16;
// retry 8 times with backoff 1, 2, 4, 8, 16, 16, 16, 16 seconds
// but should not delay more than another 16 seconds
Assert.Less(stopwatch.ElapsedMilliseconds, 51 * 1000);
Assert.GreaterOrEqual(stopwatch.ElapsedMilliseconds, 30 * 1000);
Assert.Less(stopwatch.ElapsedMilliseconds, (totalDelaySeconds + 20) * 1000);
Assert.GreaterOrEqual(stopwatch.ElapsedMilliseconds, totalDelaySeconds * 1000);
}
}

Expand Down
Loading

0 comments on commit 052d895

Please sign in to comment.