Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SNOW-902632 connection string driven pool config #873

Merged
merged 13 commits into from
Mar 25, 2024
+1,539 −366
Merged

SNOW-902632 connection string driven pool config #873

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
558b5fb
SNOW-902632 Connection string driven pool configuration
sfc-gh-knozderko Jan 11, 2024
6e7c2d2
temporary ignore TestExecuteAsyncWithMaxRetryReached
sfc-gh-knozderko Feb 28, 2024
0c0e110
temporary ignore TestExecuteWithMaxRetryReached
sfc-gh-knozderko Feb 28, 2024
7f8f0b0
fix getting enum values on .net framework
sfc-gh-knozderko Feb 28, 2024
c0e377c
enable tests for max retries for http client
sfc-gh-knozderko Feb 29, 2024
42e27b6
remove sleeping from expiration timeout tests
sfc-gh-knozderko Mar 1, 2024
0e7e029
add documentation
sfc-gh-knozderko Mar 6, 2024
3b8b009
changes after review
sfc-gh-knozderko Mar 11, 2024
53c1f7c
disable setters in new connection pool manager
sfc-gh-knozderko Mar 13, 2024
d102619
warn about overriden config
sfc-gh-knozderko Mar 14, 2024
d26cb75
More logs
sfc-gh-knozderko Mar 14, 2024
c4015b9
changes after review
sfc-gh-knozderko Mar 14, 2024
db8130a
changes after review
sfc-gh-knozderko Mar 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 37 additions & 30 deletions README.md
View file
Open in desktop

Large diffs are not rendered by default.

61 changes: 44 additions & 17 deletions Snowflake.Data.Tests/IntegrationTests/ConnectionMultiplePoolsIT.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ public class ConnectionMultiplePoolsIT: SFBaseTest
{
SnowflakeDbConnectionPool.SetConnectionPoolVersion(ConnectionPoolType.MultipleConnectionPool);
SnowflakeDbConnectionPool.ClearAllPools();
SnowflakeDbConnectionPool.SetPooling(true);
}

[TearDown]
Expand All @@ -35,6 +34,20 @@ public static void AfterAllTests()
SnowflakeDbConnectionPool.ClearAllPools();
}

[Test]
public void TestBasicConnectionPool()
{
var connectionString = ConnectionString + "minPoolSize=0;maxPoolSize=1";
var conn1 = new SnowflakeDbConnection(connectionString);
conn1.Open();
Assert.AreEqual(ConnectionState.Open, conn1.State);
conn1.Close();

// assert
Assert.AreEqual(ConnectionState.Closed, conn1.State);
Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(connectionString).GetCurrentPoolSize());
}

[Test]
public void TestReuseSessionInConnectionPool() // old name: TestConnectionPool
{
Expand Down Expand Up @@ -102,20 +115,16 @@ public void TestReuseSessionInConnectionPoolReachingMaxConnections() // old name
public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimit()
{
// arrange
var connectionString = ConnectionString + "application=TestWaitForMaxSize1";
var connectionString = ConnectionString + "application=TestWaitForMaxSize1;waitingForIdleSessionTimeout=1s;maxPoolSize=2";
sfc-gh-mhofman marked this conversation as resolved.
Show resolved Hide resolved
var pool = SnowflakeDbConnectionPool.GetPool(connectionString);
Assert.AreEqual(0, pool.GetCurrentPoolSize(), "expecting pool to be empty");
sfc-gh-knozderko marked this conversation as resolved.
Show resolved Hide resolved
pool.SetMaxPoolSize(2);
pool.SetWaitingForSessionToReuseTimeout(1000);
var conn1 = OpenedConnection(connectionString);
var conn2 = OpenedConnection(connectionString);
var watch = new StopWatch();

// act
watch.Start();
var start = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
var thrown = Assert.Throws<SnowflakeDbException>(() => OpenedConnection(connectionString));
var stop = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
watch.Stop();

// assert
Expand All @@ -132,11 +141,9 @@ public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimit()
public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimitAsync()
{
// arrange
var connectionString = ConnectionString + "application=TestWaitForMaxSize2";
var connectionString = ConnectionString + "application=TestWaitForMaxSize2;waitingForIdleSessionTimeout=1s;maxPoolSize=2";
var pool = SnowflakeDbConnectionPool.GetPool(connectionString);
Assert.AreEqual(0, pool.GetCurrentPoolSize(), "expecting pool to be empty");
pool.SetMaxPoolSize(2);
pool.SetWaitingForSessionToReuseTimeout(1000);
var conn1 = OpenedConnection(connectionString);
var conn2 = OpenedConnection(connectionString);
var watch = new StopWatch();
Expand All @@ -163,11 +170,9 @@ public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimitAsync()
public void TestWaitInAQueueForAnIdleSession()
{
// arrange
var connectionString = ConnectionString + "application=TestWaitForMaxSize3";
var connectionString = ConnectionString + "application=TestWaitForMaxSize3;waitingForIdleSessionTimeout=3s;maxPoolSize=2";
var pool = SnowflakeDbConnectionPool.GetPool(connectionString);
Assert.AreEqual(0, pool.GetCurrentPoolSize(), "the pool is expected to be empty");
pool.SetMaxPoolSize(2);
pool.SetWaitingForSessionToReuseTimeout(3000);
const long ADelay = 0;
const long BDelay = 400;
const long CDelay = 2 * BDelay;
Expand Down Expand Up @@ -238,7 +243,6 @@ public void TestBusyAndIdleConnectionsCountedInPoolSize()
}

[Test]
[Ignore("Enable when disabling pooling in connection string enabled - SNOW-902632")]
public void TestConnectionPoolNotPossibleToDisableForAllPools()
{
// act
Expand Down Expand Up @@ -275,19 +279,19 @@ public void TestConnectionPoolDisable()
[Test]
public void TestNewConnectionPoolClean()
{
SnowflakeDbConnectionPool.SetMaxPoolSize(2);
var connectionString = ConnectionString + "maxPoolSize=2;";
var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = ConnectionString;
conn1.ConnectionString = connectionString;
conn1.Open();
Assert.AreEqual(ConnectionState.Open, conn1.State);

var conn2 = new SnowflakeDbConnection();
conn2.ConnectionString = ConnectionString + " retryCount=1";
conn2.ConnectionString = connectionString + "retryCount=1";
conn2.Open();
Assert.AreEqual(ConnectionState.Open, conn2.State);

var conn3 = new SnowflakeDbConnection();
conn3.ConnectionString = ConnectionString + " retryCount=2";
conn3.ConnectionString = connectionString + "retryCount=2";
conn3.Open();
Assert.AreEqual(ConnectionState.Open, conn3.State);

Expand All @@ -305,6 +309,29 @@ public void TestNewConnectionPoolClean()
Assert.AreEqual(ConnectionState.Closed, conn2.State);
Assert.AreEqual(ConnectionState.Closed, conn3.State);
}

[Test]
public void TestConnectionPoolExpirationWorks()
{
var connectionString = ConnectionString + "expirationTimeout=0;maxPoolSize=2";
var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = connectionString;
conn1.Open();
conn1.Close();
sfc-gh-mhofman marked this conversation as resolved.
Show resolved Hide resolved

var conn2 = new SnowflakeDbConnection();
conn2.ConnectionString = connectionString;
conn2.Open();
conn2.Close();

var conn3 = new SnowflakeDbConnection();
conn3.ConnectionString = connectionString;
conn3.Open();
conn3.Close();

// assert
Assert.AreEqual(0, SnowflakeDbConnectionPool.GetPool(connectionString).GetCurrentPoolSize());
}

private SnowflakeDbConnection OpenedConnection(string connectionString)
{
Expand Down
56 changes: 8 additions & 48 deletions Snowflake.Data.Tests/IntegrationTests/ConnectionPoolCommonIT.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@ public ConnectionPoolCommonIT(ConnectionPoolType connectionPoolTypeUnderTest)
{
SnowflakeDbConnectionPool.SetConnectionPoolVersion(_connectionPoolTypeUnderTest);
SnowflakeDbConnectionPool.ClearAllPools();
SnowflakeDbConnectionPool.SetPooling(true);
if (_connectionPoolTypeUnderTest == ConnectionPoolType.SingleConnectionCache)
{
SnowflakeDbConnectionPool.SetPooling(true);
}
s_logger.Debug($"---------------- BeforeTest ---------------------");
s_logger.Debug($"Testing Pool Type: {SnowflakeDbConnectionPool.GetConnectionPoolVersion()}");
}
Expand All @@ -49,48 +52,6 @@ public static void AfterAllTests()
{
SnowflakeDbConnectionPool.ClearAllPools();
}

[Test]
public void TestBasicConnectionPool()
{
SnowflakeDbConnectionPool.SetMaxPoolSize(1);

var conn1 = new SnowflakeDbConnection(ConnectionString);
conn1.Open();
Assert.AreEqual(ConnectionState.Open, conn1.State);
conn1.Close();

Assert.AreEqual(ConnectionState.Closed, conn1.State);
Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize());
}

[Test]
sfc-gh-mhofman marked this conversation as resolved.
Show resolved Hide resolved
public void TestConnectionPoolExpirationWorks()
{
SnowflakeDbConnectionPool.SetMaxPoolSize(2);
SnowflakeDbConnectionPool.SetTimeout(10);

var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = ConnectionString;

conn1.Open();
conn1.Close();
SnowflakeDbConnectionPool.SetTimeout(-1);

var conn2 = new SnowflakeDbConnection();
conn2.ConnectionString = ConnectionString;
conn2.Open();
conn2.Close();
var conn3 = new SnowflakeDbConnection();
conn3.ConnectionString = ConnectionString;
conn3.Open();
conn3.Close();

// The pooling timeout should apply to all connections being pooled,
// not just the connections created after the new setting,
// so expected result should be 0
Assert.AreEqual(0, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize());
}

[Test]
public void TestConnectionPoolMultiThreading()
Expand Down Expand Up @@ -128,16 +89,15 @@ void ThreadProcess2(string connstr)
Assert.AreEqual(true, resultSet.Next());
Assert.AreEqual("1", resultSet.GetString(0));
conn1.Close();
SnowflakeDbConnectionPool.ClearAllPools();
SnowflakeDbConnectionPool.SetMaxPoolSize(0);
SnowflakeDbConnectionPool.SetPooling(true);
}

[Test]
public void TestConnectionPoolWithDispose()
{
SnowflakeDbConnectionPool.SetMaxPoolSize(1);

if (_connectionPoolTypeUnderTest == ConnectionPoolType.SingleConnectionCache)
{
SnowflakeDbConnectionPool.SetMaxPoolSize(1);
}
var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = "bad connection string";
Assert.Throws<SnowflakeDbException>(() => conn1.Open());
Expand Down
43 changes: 43 additions & 0 deletions Snowflake.Data.Tests/IntegrationTests/ConnectionSinglePoolCacheIT.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,20 @@ public static void AfterAllTests()
SnowflakeDbConnectionPool.ClearAllPools();
}

[Test]
public void TestBasicConnectionPool()
{
SnowflakeDbConnectionPool.SetMaxPoolSize(1);

var conn1 = new SnowflakeDbConnection(ConnectionString);
conn1.Open();
Assert.AreEqual(ConnectionState.Open, conn1.State);
conn1.Close();

Assert.AreEqual(ConnectionState.Closed, conn1.State);
Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize());
}

[Test]
public void TestConcurrentConnectionPooling()
{
Expand Down Expand Up @@ -256,5 +270,34 @@ public void TestConnectionPoolClean()
Assert.AreEqual(ConnectionState.Closed, conn2.State);
Assert.AreEqual(ConnectionState.Closed, conn3.State);
}

[Test]
public void TestConnectionPoolExpirationWorks()
{
SnowflakeDbConnectionPool.SetMaxPoolSize(2);
SnowflakeDbConnectionPool.SetTimeout(10);

var conn1 = new SnowflakeDbConnection();
conn1.ConnectionString = ConnectionString;

conn1.Open();
conn1.Close();
SnowflakeDbConnectionPool.SetTimeout(0);
sfc-gh-mhofman marked this conversation as resolved.
Show resolved Hide resolved

var conn2 = new SnowflakeDbConnection();
conn2.ConnectionString = ConnectionString;
conn2.Open();
conn2.Close();

var conn3 = new SnowflakeDbConnection();
conn3.ConnectionString = ConnectionString;
conn3.Open();
conn3.Close();

// The pooling timeout should apply to all connections being pooled,
// not just the connections created after the new setting,
// so expected result should be 0
Assert.AreEqual(0, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize());
}
}
}
19 changes: 10 additions & 9 deletions Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ namespace Snowflake.Data.Tests.IntegrationTests
using System.Collections.Generic;
using System.Globalization;
using Snowflake.Data.Tests.Mock;
using Snowflake.Data.Core;

[TestFixture]
class SFDbCommandITAsync : SFBaseTestAsync
Expand Down Expand Up @@ -102,7 +101,7 @@ public void TestExecuteAsyncWithMaxRetryReached()

using (DbConnection conn = new MockSnowflakeDbConnection(mockRestRequester))
{
string maxRetryConnStr = ConnectionString + "maxHttpRetries=5";
string maxRetryConnStr = ConnectionString + "maxHttpRetries=8";

conn.ConnectionString = maxRetryConnStr;
conn.Open();
Expand All @@ -124,10 +123,11 @@ public void TestExecuteAsyncWithMaxRetryReached()
}
stopwatch.Stop();

// retry 5 times with backoff 1, 2, 4, 8, 16 seconds
var totalDelaySeconds = 1 + 2 + 4 + 8 + 16 + 16 + 16 + 16;
// retry 8 times with backoff 1, 2, 4, 8, 16, 16, 16, 16 seconds
// but should not delay more than another 16 seconds
Assert.Less(stopwatch.ElapsedMilliseconds, 51 * 1000);
Assert.GreaterOrEqual(stopwatch.ElapsedMilliseconds, 30 * 1000);
Assert.Less(stopwatch.ElapsedMilliseconds, (totalDelaySeconds + 20) * 1000);
Assert.GreaterOrEqual(stopwatch.ElapsedMilliseconds, totalDelaySeconds * 1000);
}
}
}
Expand Down Expand Up @@ -594,7 +594,7 @@ public void TestExecuteWithMaxRetryReached()

using (IDbConnection conn = new MockSnowflakeDbConnection(mockRestRequester))
{
string maxRetryConnStr = ConnectionString + "maxHttpRetries=5";
string maxRetryConnStr = ConnectionString + "maxHttpRetries=8";

conn.ConnectionString = maxRetryConnStr;
conn.Open();
Expand All @@ -615,10 +615,11 @@ public void TestExecuteWithMaxRetryReached()
}
stopwatch.Stop();

// retry 5 times with backoff 1, 2, 4, 8, 16 seconds
var totalDelaySeconds = 1 + 2 + 4 + 8 + 16 + 16 + 16 + 16;
// retry 8 times with backoff 1, 2, 4, 8, 16, 16, 16, 16 seconds
// but should not delay more than another 16 seconds
Assert.Less(stopwatch.ElapsedMilliseconds, 51 * 1000);
Assert.GreaterOrEqual(stopwatch.ElapsedMilliseconds, 30 * 1000);
Assert.Less(stopwatch.ElapsedMilliseconds, (totalDelaySeconds + 20) * 1000);
Assert.GreaterOrEqual(stopwatch.ElapsedMilliseconds, totalDelaySeconds * 1000);
}
}

Expand Down
Loading
Loading