From 052d89523be1fa5faee12b9c52280ad4bfab6d6f Mon Sep 17 00:00:00 2001 From: Krzysztof Nozderko Date: Mon, 25 Mar 2024 09:02:15 +0100 Subject: [PATCH] SNOW-902632 connection string driven pool config (#873) ### Description SNOW-902632 connection string driven pool config ### Checklist - [x] Code compiles correctly - [x] Code is formatted according to [Coding Conventions](../CodingConventions.md) - [x] Created tests which fail without the change (if possible) - [x] All tests passing (`dotnet test`) - [x] Extended the README / documentation, if necessary - [x] Provide JIRA issue id (if possible) or GitHub issue id in PR name --- README.md | 67 ++-- .../ConnectionMultiplePoolsIT.cs | 61 +++- .../ConnectionPoolCommonIT.cs | 56 +--- .../ConnectionSinglePoolCacheIT.cs | 43 +++ .../IntegrationTests/SFDbCommandIT.cs | 19 +- .../UnitTests/ConnectionPoolManagerTest.cs | 50 ++- .../UnitTests/SFSessionPropertyTest.cs | 61 +++- .../ConnectionPoolConfigExtractorTest.cs | 304 ++++++++++++++++++ ...CountingSessionCreationTokenCounterTest.cs | 3 +- .../Session/SFHttpClientPropertiesTest.cs | 99 +++--- .../SessionCreationTokenCounterTest.cs | 19 +- .../Session/SessionCreationTokenTest.cs | 9 +- .../UnitTests/Session/SessionPoolTest.cs | 65 ++++ ...ropertiesWithDefaultValuesExtractorTest.cs | 213 ++++++++++++ .../UnitTests/Tools/TimeoutHelperTest.cs | 141 ++++++++ .../Core/Session/ChangedSessionBehavior.cs | 33 ++ .../Core/Session/ConnectionCacheManager.cs | 2 +- .../Core/Session/ConnectionPoolConfig.cs | 15 + .../Core/Session/ConnectionPoolManager.cs | 23 +- .../Core/Session/FixedZeroCounter.cs | 4 + Snowflake.Data/Core/Session/ICounter.cs | 2 + Snowflake.Data/Core/Session/IWaitingQueue.cs | 4 - .../NonCountingSessionCreationTokenCounter.cs | 6 +- .../Core/Session/NonNegativeCounter.cs | 2 + .../Core/Session/NonWaitingQueue.cs | 7 - Snowflake.Data/Core/Session/SFSession.cs | 23 +- .../Session/SFSessionHttpClientProperties.cs | 174 ++++++++-- .../Core/Session/SFSessionProperty.cs | 14 +- .../Core/Session/SessionCreationToken.cs | 10 +- .../Session/SessionCreationTokenCounter.cs | 8 +- Snowflake.Data/Core/Session/SessionPool.cs | 181 +++++++---- ...ionPropertiesWithDefaultValuesExtractor.cs | 143 ++++++++ Snowflake.Data/Core/Session/WaitingQueue.cs | 7 +- Snowflake.Data/Core/Tools/TimeoutHelper.cs | 37 +++ 34 files changed, 1539 insertions(+), 366 deletions(-) create mode 100644 Snowflake.Data.Tests/UnitTests/Session/ConnectionPoolConfigExtractorTest.cs create mode 100644 Snowflake.Data.Tests/UnitTests/Session/SessionPoolTest.cs create mode 100644 Snowflake.Data.Tests/UnitTests/Session/SessionPropertiesWithDefaultValuesExtractorTest.cs create mode 100644 Snowflake.Data.Tests/UnitTests/Tools/TimeoutHelperTest.cs create mode 100644 Snowflake.Data/Core/Session/ChangedSessionBehavior.cs create mode 100644 Snowflake.Data/Core/Session/ConnectionPoolConfig.cs create mode 100644 Snowflake.Data/Core/Session/SessionPropertiesWithDefaultValuesExtractor.cs create mode 100644 Snowflake.Data/Core/Tools/TimeoutHelper.cs diff --git a/README.md b/README.md index 13f2ca41d..43845773a 100644 --- a/README.md +++ b/README.md @@ -133,37 +133,44 @@ i.e "\=\;\=\...". The following table lists all valid connection properties:
-| Connection Property | Required | Comment | -|--------------------------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| ACCOUNT | Yes | Your full account name might include additional segments that identify the region and cloud platform where your account is hosted | -| APPLICATION | No | **_Snowflake partner use only_**: Specifies the name of a partner application to connect through .NET. The name must match the following pattern: ^\[A-Za-z](\[A-Za-z0-9.-]){1,50}$ (one letter followed by 1 to 50 letter, digit, .,- or, \_ characters). | -| DB | No | | -| HOST | No | Specifies the hostname for your account in the following format: \.snowflakecomputing.com.
If no value is specified, the driver uses \.snowflakecomputing.com. | -| PASSWORD | Depends | Required if AUTHENTICATOR is set to `snowflake` (the default value) or the URL for native SSO through Okta. Ignored for all the other authentication types. | -| ROLE | No | | -| SCHEMA | No | | -| USER | Depends | If AUTHENTICATOR is set to `externalbrowser` this is optional. For native SSO through Okta, set this to the login name for your identity provider (IdP). | -| WAREHOUSE | No | | -| CONNECTION_TIMEOUT | No | Total timeout in seconds when connecting to Snowflake. The default is 300 seconds | -| RETRY_TIMEOUT | No | Total timeout in seconds for supported endpoints of retry policy. The default is 300 seconds. The value can only be increased from the default value or set to 0 for infinite timeout | -| MAXHTTPRETRIES | No | Maximum number of times to retry failed HTTP requests (default: 7). You can set `MAXHTTPRETRIES=0` to remove the retry limit, but doing so runs the risk of the .NET driver infinitely retrying failed HTTP calls. | -| CLIENT_SESSION_KEEP_ALIVE | No | Whether to keep the current session active after a period of inactivity, or to force the user to login again. If the value is `true`, Snowflake keeps the session active indefinitely, even if there is no activity from the user. If the value is `false`, the user must log in again after four hours of inactivity. The default is `false`. Setting this value overrides the server session property for the current session. | -| DISABLERETRY | No | Set this property to `true` to prevent the driver from reconnecting automatically when the connection fails or drops. The default value is `false`. | +| Connection Property | Required | Comment | +|--------------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ACCOUNT | Yes | Your full account name might include additional segments that identify the region and cloud platform where your account is hosted | +| APPLICATION | No | **_Snowflake partner use only_**: Specifies the name of a partner application to connect through .NET. The name must match the following pattern: ^\[A-Za-z](\[A-Za-z0-9.-]){1,50}$ (one letter followed by 1 to 50 letter, digit, .,- or, \_ characters). | +| DB | No | | +| HOST | No | Specifies the hostname for your account in the following format: \.snowflakecomputing.com.
If no value is specified, the driver uses \.snowflakecomputing.com. | +| PASSWORD | Depends | Required if AUTHENTICATOR is set to `snowflake` (the default value) or the URL for native SSO through Okta. Ignored for all the other authentication types. | +| ROLE | No | | +| SCHEMA | No | | +| USER | Depends | If AUTHENTICATOR is set to `externalbrowser` this is optional. For native SSO through Okta, set this to the login name for your identity provider (IdP). | +| WAREHOUSE | No | | +| CONNECTION_TIMEOUT | No | Total timeout in seconds when connecting to Snowflake. The default is 300 seconds | +| RETRY_TIMEOUT | No | Total timeout in seconds for supported endpoints of retry policy. The default is 300 seconds. The value can only be increased from the default value or set to 0 for infinite timeout | +| MAXHTTPRETRIES | No | Maximum number of times to retry failed HTTP requests (default: 7). You can set `MAXHTTPRETRIES=0` to remove the retry limit, but doing so runs the risk of the .NET driver infinitely retrying failed HTTP calls. | +| CLIENT_SESSION_KEEP_ALIVE | No | Whether to keep the current session active after a period of inactivity, or to force the user to login again. If the value is `true`, Snowflake keeps the session active indefinitely, even if there is no activity from the user. If the value is `false`, the user must log in again after four hours of inactivity. The default is `false`. Setting this value overrides the server session property for the current session. | +| DISABLERETRY | No | Set this property to `true` to prevent the driver from reconnecting automatically when the connection fails or drops. The default value is `false`. | | AUTHENTICATOR | No | The method of authentication. Currently supports the following values:
- snowflake (default): You must also set USER and PASSWORD.
- [the URL for native SSO through Okta](https://docs.snowflake.com/en/user-guide/admin-security-fed-auth-use.html#native-sso-okta-only): You must also set USER and PASSWORD.
- [externalbrowser](https://docs.snowflake.com/en/user-guide/admin-security-fed-auth-use.html#browser-based-sso): You must also set USER.
- [snowflake_jwt](https://docs.snowflake.com/en/user-guide/key-pair-auth.html): You must also set PRIVATE_KEY_FILE or PRIVATE_KEY.
- [oauth](https://docs.snowflake.com/en/user-guide/oauth.html): You must also set TOKEN. | BROWSER_RESPONSE_TIMEOUT | No | Number to seconds to wait for authentication in an external browser (default: 120). | -| VALIDATE_DEFAULT_PARAMETERS | No | Whether DB, SCHEMA and WAREHOUSE should be verified when making connection. Default to be true. | -| PRIVATE_KEY_FILE | Depends | The path to the private key file to use for key-pair authentication. Must be used in combination with AUTHENTICATOR=snowflake_jwt | -| PRIVATE_KEY_PWD | No | The passphrase to use for decrypting the private key, if the key is encrypted. | -| PRIVATE_KEY | Depends | The private key to use for key-pair authentication. Must be used in combination with AUTHENTICATOR=snowflake_jwt.
If the private key value includes any equal signs (=), make sure to replace each equal sign with two signs (==) to ensure that the connection string is parsed correctly. | -| TOKEN | Depends | The OAuth token to use for OAuth authentication. Must be used in combination with AUTHENTICATOR=oauth. | -| INSECUREMODE | No | Set to true to disable the certificate revocation list check. Default is false. | -| USEPROXY | No | Set to true if you need to use a proxy server. The default value is false.

This parameter was introduced in v2.0.4. | -| PROXYHOST | Depends | The hostname of the proxy server.

If USEPROXY is set to `true`, you must set this parameter.

This parameter was introduced in v2.0.4. | -| PROXYPORT | Depends | The port number of the proxy server.

If USEPROXY is set to `true`, you must set this parameter.

This parameter was introduced in v2.0.4. | -| PROXYUSER | No | The username for authenticating to the proxy server.

This parameter was introduced in v2.0.4. | -| PROXYPASSWORD | Depends | The password for authenticating to the proxy server.

If USEPROXY is `true` and PROXYUSER is set, you must set this parameter.

This parameter was introduced in v2.0.4. | -| NONPROXYHOSTS | No | The list of hosts that the driver should connect to directly, bypassing the proxy server. Separate the hostnames with a pipe symbol (\|). You can also use an asterisk (`*`) as a wildcard.

This parameter was introduced in v2.0.4. | -| FILE_TRANSFER_MEMORY_THRESHOLD | No | The maximum number of bytes to store in memory used in order to provide a file encryption. If encrypting/decrypting file size exeeds provided value a temporary file will be created and the work will be continued in the temporary file instead of memory.
If no value provided 1MB will be used as a default value (that is 1048576 bytes).
It is possible to configure any integer value bigger than zero representing maximal number of bytes to reside in memory. | -| CLIENT_CONFIG_FILE | No | The location of the client configuration json file. In this file you can configure easy logging feature. | +| VALIDATE_DEFAULT_PARAMETERS | No | Whether DB, SCHEMA and WAREHOUSE should be verified when making connection. Default to be true. | +| PRIVATE_KEY_FILE | Depends | The path to the private key file to use for key-pair authentication. Must be used in combination with AUTHENTICATOR=snowflake_jwt | +| PRIVATE_KEY_PWD | No | The passphrase to use for decrypting the private key, if the key is encrypted. | +| PRIVATE_KEY | Depends | The private key to use for key-pair authentication. Must be used in combination with AUTHENTICATOR=snowflake_jwt.
If the private key value includes any equal signs (=), make sure to replace each equal sign with two signs (==) to ensure that the connection string is parsed correctly. | +| TOKEN | Depends | The OAuth token to use for OAuth authentication. Must be used in combination with AUTHENTICATOR=oauth. | +| INSECUREMODE | No | Set to true to disable the certificate revocation list check. Default is false. | +| USEPROXY | No | Set to true if you need to use a proxy server. The default value is false.

This parameter was introduced in v2.0.4. | +| PROXYHOST | Depends | The hostname of the proxy server.

If USEPROXY is set to `true`, you must set this parameter.

This parameter was introduced in v2.0.4. | +| PROXYPORT | Depends | The port number of the proxy server.

If USEPROXY is set to `true`, you must set this parameter.

This parameter was introduced in v2.0.4. | +| PROXYUSER | No | The username for authenticating to the proxy server.

This parameter was introduced in v2.0.4. | +| PROXYPASSWORD | Depends | The password for authenticating to the proxy server.

If USEPROXY is `true` and PROXYUSER is set, you must set this parameter.

This parameter was introduced in v2.0.4. | +| NONPROXYHOSTS | No | The list of hosts that the driver should connect to directly, bypassing the proxy server. Separate the hostnames with a pipe symbol (\|). You can also use an asterisk (`*`) as a wildcard.

This parameter was introduced in v2.0.4. | +| FILE_TRANSFER_MEMORY_THRESHOLD | No | The maximum number of bytes to store in memory used in order to provide a file encryption. If encrypting/decrypting file size exeeds provided value a temporary file will be created and the work will be continued in the temporary file instead of memory.
If no value provided 1MB will be used as a default value (that is 1048576 bytes).
It is possible to configure any integer value bigger than zero representing maximal number of bytes to reside in memory. | +| CLIENT_CONFIG_FILE | No | The location of the client configuration json file. In this file you can configure easy logging feature. | +| MAXPOOLSIZE | No | Maximum number of connections in a pool. Default value is 10. `maxPoolSize` value cannot be lower than `minPoolSize` value. | +| MINPOOLSIZE | No | Expected minimum number of connections in pool. When you get a connection from the pool, more connections might be initialised in background to increase the pool size to `minPoolSize`. If you specify 0 or 1 there will be no attempts to create extra initialisations in background. The default value is 2. `maxPoolSize` value cannot be lower than `minPoolSize` value. The parameter is used only in a new version of connection pool. | +| CHANGEDSESSION | No | Specifies what should happen with a closed connection when some of its session variables are altered (e. g. you used `ALTER SESSION SET SCHEMA` to change the databese schema). The default behaviour is `OriginalPool` which means the session stays in the original pool. Currently no other option is possible. Parameter used only in a new version of connection pool. | +| WAITINGFORIDLESESSIONTIMEOUT | No | Timeout for waiting for an idle session when pool is full. It happens when there is no idle session and we cannot create a new one because of reaching `maxPoolSize`. The default value is 30 seconds. Usage of units possible and allowed are: e. g. `1000ms` (milliseconds), `15s` (seconds), `2m` (minutes) where seconds are default for a skipped postfix. Special values: `0` - immediate fail for new connection to open when session is full. You cannot specify infinite value. | +| EXPIRATIONTIMEOUT | No | Timeout for using each connection. Connections which last more than specified timeout are considered to be expired and are being removed from the pool. The default is 1 hour. Usage of units possible and allowed are: e. g. `360000ms` (milliseconds), `3600s` (seconds), `60m` (minutes) where seconds are default for a skipped postfix. Special values: `0` - immediate expiration of the connection just after its creation. Expiration timeout cannot be set to infinity. | +| POOLINGENABLED | No | Boolean flag indicating if the connection should be a part of a pool. The default value is `true`. | +
diff --git a/Snowflake.Data.Tests/IntegrationTests/ConnectionMultiplePoolsIT.cs b/Snowflake.Data.Tests/IntegrationTests/ConnectionMultiplePoolsIT.cs index 599b99859..51355b323 100644 --- a/Snowflake.Data.Tests/IntegrationTests/ConnectionMultiplePoolsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/ConnectionMultiplePoolsIT.cs @@ -20,7 +20,6 @@ public class ConnectionMultiplePoolsIT: SFBaseTest { SnowflakeDbConnectionPool.SetConnectionPoolVersion(ConnectionPoolType.MultipleConnectionPool); SnowflakeDbConnectionPool.ClearAllPools(); - SnowflakeDbConnectionPool.SetPooling(true); } [TearDown] @@ -35,6 +34,20 @@ public static void AfterAllTests() SnowflakeDbConnectionPool.ClearAllPools(); } + [Test] + public void TestBasicConnectionPool() + { + var connectionString = ConnectionString + "minPoolSize=0;maxPoolSize=1"; + var conn1 = new SnowflakeDbConnection(connectionString); + conn1.Open(); + Assert.AreEqual(ConnectionState.Open, conn1.State); + conn1.Close(); + + // assert + Assert.AreEqual(ConnectionState.Closed, conn1.State); + Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(connectionString).GetCurrentPoolSize()); + } + [Test] public void TestReuseSessionInConnectionPool() // old name: TestConnectionPool { @@ -102,20 +115,16 @@ public void TestReuseSessionInConnectionPoolReachingMaxConnections() // old name public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimit() { // arrange - var connectionString = ConnectionString + "application=TestWaitForMaxSize1"; + var connectionString = ConnectionString + "application=TestWaitForMaxSize1;waitingForIdleSessionTimeout=1s;maxPoolSize=2"; var pool = SnowflakeDbConnectionPool.GetPool(connectionString); Assert.AreEqual(0, pool.GetCurrentPoolSize(), "expecting pool to be empty"); - pool.SetMaxPoolSize(2); - pool.SetWaitingForSessionToReuseTimeout(1000); var conn1 = OpenedConnection(connectionString); var conn2 = OpenedConnection(connectionString); var watch = new StopWatch(); // act watch.Start(); - var start = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); var thrown = Assert.Throws(() => OpenedConnection(connectionString)); - var stop = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); watch.Stop(); // assert @@ -132,11 +141,9 @@ public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimit() public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimitAsync() { // arrange - var connectionString = ConnectionString + "application=TestWaitForMaxSize2"; + var connectionString = ConnectionString + "application=TestWaitForMaxSize2;waitingForIdleSessionTimeout=1s;maxPoolSize=2"; var pool = SnowflakeDbConnectionPool.GetPool(connectionString); Assert.AreEqual(0, pool.GetCurrentPoolSize(), "expecting pool to be empty"); - pool.SetMaxPoolSize(2); - pool.SetWaitingForSessionToReuseTimeout(1000); var conn1 = OpenedConnection(connectionString); var conn2 = OpenedConnection(connectionString); var watch = new StopWatch(); @@ -163,11 +170,9 @@ public void TestWaitForTheIdleConnectionWhenExceedingMaxConnectionsLimitAsync() public void TestWaitInAQueueForAnIdleSession() { // arrange - var connectionString = ConnectionString + "application=TestWaitForMaxSize3"; + var connectionString = ConnectionString + "application=TestWaitForMaxSize3;waitingForIdleSessionTimeout=3s;maxPoolSize=2"; var pool = SnowflakeDbConnectionPool.GetPool(connectionString); Assert.AreEqual(0, pool.GetCurrentPoolSize(), "the pool is expected to be empty"); - pool.SetMaxPoolSize(2); - pool.SetWaitingForSessionToReuseTimeout(3000); const long ADelay = 0; const long BDelay = 400; const long CDelay = 2 * BDelay; @@ -238,7 +243,6 @@ public void TestBusyAndIdleConnectionsCountedInPoolSize() } [Test] - [Ignore("Enable when disabling pooling in connection string enabled - SNOW-902632")] public void TestConnectionPoolNotPossibleToDisableForAllPools() { // act @@ -275,19 +279,19 @@ public void TestConnectionPoolDisable() [Test] public void TestNewConnectionPoolClean() { - SnowflakeDbConnectionPool.SetMaxPoolSize(2); + var connectionString = ConnectionString + "maxPoolSize=2;"; var conn1 = new SnowflakeDbConnection(); - conn1.ConnectionString = ConnectionString; + conn1.ConnectionString = connectionString; conn1.Open(); Assert.AreEqual(ConnectionState.Open, conn1.State); var conn2 = new SnowflakeDbConnection(); - conn2.ConnectionString = ConnectionString + " retryCount=1"; + conn2.ConnectionString = connectionString + "retryCount=1"; conn2.Open(); Assert.AreEqual(ConnectionState.Open, conn2.State); var conn3 = new SnowflakeDbConnection(); - conn3.ConnectionString = ConnectionString + " retryCount=2"; + conn3.ConnectionString = connectionString + "retryCount=2"; conn3.Open(); Assert.AreEqual(ConnectionState.Open, conn3.State); @@ -305,6 +309,29 @@ public void TestNewConnectionPoolClean() Assert.AreEqual(ConnectionState.Closed, conn2.State); Assert.AreEqual(ConnectionState.Closed, conn3.State); } + + [Test] + public void TestConnectionPoolExpirationWorks() + { + var connectionString = ConnectionString + "expirationTimeout=0;maxPoolSize=2"; + var conn1 = new SnowflakeDbConnection(); + conn1.ConnectionString = connectionString; + conn1.Open(); + conn1.Close(); + + var conn2 = new SnowflakeDbConnection(); + conn2.ConnectionString = connectionString; + conn2.Open(); + conn2.Close(); + + var conn3 = new SnowflakeDbConnection(); + conn3.ConnectionString = connectionString; + conn3.Open(); + conn3.Close(); + + // assert + Assert.AreEqual(0, SnowflakeDbConnectionPool.GetPool(connectionString).GetCurrentPoolSize()); + } private SnowflakeDbConnection OpenedConnection(string connectionString) { diff --git a/Snowflake.Data.Tests/IntegrationTests/ConnectionPoolCommonIT.cs b/Snowflake.Data.Tests/IntegrationTests/ConnectionPoolCommonIT.cs index a75c19790..e848e43e1 100644 --- a/Snowflake.Data.Tests/IntegrationTests/ConnectionPoolCommonIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/ConnectionPoolCommonIT.cs @@ -33,7 +33,10 @@ public ConnectionPoolCommonIT(ConnectionPoolType connectionPoolTypeUnderTest) { SnowflakeDbConnectionPool.SetConnectionPoolVersion(_connectionPoolTypeUnderTest); SnowflakeDbConnectionPool.ClearAllPools(); - SnowflakeDbConnectionPool.SetPooling(true); + if (_connectionPoolTypeUnderTest == ConnectionPoolType.SingleConnectionCache) + { + SnowflakeDbConnectionPool.SetPooling(true); + } s_logger.Debug($"---------------- BeforeTest ---------------------"); s_logger.Debug($"Testing Pool Type: {SnowflakeDbConnectionPool.GetConnectionPoolVersion()}"); } @@ -49,48 +52,6 @@ public static void AfterAllTests() { SnowflakeDbConnectionPool.ClearAllPools(); } - - [Test] - public void TestBasicConnectionPool() - { - SnowflakeDbConnectionPool.SetMaxPoolSize(1); - - var conn1 = new SnowflakeDbConnection(ConnectionString); - conn1.Open(); - Assert.AreEqual(ConnectionState.Open, conn1.State); - conn1.Close(); - - Assert.AreEqual(ConnectionState.Closed, conn1.State); - Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize()); - } - - [Test] - public void TestConnectionPoolExpirationWorks() - { - SnowflakeDbConnectionPool.SetMaxPoolSize(2); - SnowflakeDbConnectionPool.SetTimeout(10); - - var conn1 = new SnowflakeDbConnection(); - conn1.ConnectionString = ConnectionString; - - conn1.Open(); - conn1.Close(); - SnowflakeDbConnectionPool.SetTimeout(-1); - - var conn2 = new SnowflakeDbConnection(); - conn2.ConnectionString = ConnectionString; - conn2.Open(); - conn2.Close(); - var conn3 = new SnowflakeDbConnection(); - conn3.ConnectionString = ConnectionString; - conn3.Open(); - conn3.Close(); - - // The pooling timeout should apply to all connections being pooled, - // not just the connections created after the new setting, - // so expected result should be 0 - Assert.AreEqual(0, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize()); - } [Test] public void TestConnectionPoolMultiThreading() @@ -128,16 +89,15 @@ void ThreadProcess2(string connstr) Assert.AreEqual(true, resultSet.Next()); Assert.AreEqual("1", resultSet.GetString(0)); conn1.Close(); - SnowflakeDbConnectionPool.ClearAllPools(); - SnowflakeDbConnectionPool.SetMaxPoolSize(0); - SnowflakeDbConnectionPool.SetPooling(true); } [Test] public void TestConnectionPoolWithDispose() { - SnowflakeDbConnectionPool.SetMaxPoolSize(1); - + if (_connectionPoolTypeUnderTest == ConnectionPoolType.SingleConnectionCache) + { + SnowflakeDbConnectionPool.SetMaxPoolSize(1); + } var conn1 = new SnowflakeDbConnection(); conn1.ConnectionString = "bad connection string"; Assert.Throws(() => conn1.Open()); diff --git a/Snowflake.Data.Tests/IntegrationTests/ConnectionSinglePoolCacheIT.cs b/Snowflake.Data.Tests/IntegrationTests/ConnectionSinglePoolCacheIT.cs index 42ae22b74..dcb7d6099 100644 --- a/Snowflake.Data.Tests/IntegrationTests/ConnectionSinglePoolCacheIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/ConnectionSinglePoolCacheIT.cs @@ -35,6 +35,20 @@ public static void AfterAllTests() SnowflakeDbConnectionPool.ClearAllPools(); } + [Test] + public void TestBasicConnectionPool() + { + SnowflakeDbConnectionPool.SetMaxPoolSize(1); + + var conn1 = new SnowflakeDbConnection(ConnectionString); + conn1.Open(); + Assert.AreEqual(ConnectionState.Open, conn1.State); + conn1.Close(); + + Assert.AreEqual(ConnectionState.Closed, conn1.State); + Assert.AreEqual(1, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize()); + } + [Test] public void TestConcurrentConnectionPooling() { @@ -256,5 +270,34 @@ public void TestConnectionPoolClean() Assert.AreEqual(ConnectionState.Closed, conn2.State); Assert.AreEqual(ConnectionState.Closed, conn3.State); } + + [Test] + public void TestConnectionPoolExpirationWorks() + { + SnowflakeDbConnectionPool.SetMaxPoolSize(2); + SnowflakeDbConnectionPool.SetTimeout(10); + + var conn1 = new SnowflakeDbConnection(); + conn1.ConnectionString = ConnectionString; + + conn1.Open(); + conn1.Close(); + SnowflakeDbConnectionPool.SetTimeout(0); + + var conn2 = new SnowflakeDbConnection(); + conn2.ConnectionString = ConnectionString; + conn2.Open(); + conn2.Close(); + + var conn3 = new SnowflakeDbConnection(); + conn3.ConnectionString = ConnectionString; + conn3.Open(); + conn3.Close(); + + // The pooling timeout should apply to all connections being pooled, + // not just the connections created after the new setting, + // so expected result should be 0 + Assert.AreEqual(0, SnowflakeDbConnectionPool.GetPool(ConnectionString).GetCurrentPoolSize()); + } } } diff --git a/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs index 58bf90b46..0f0924e12 100755 --- a/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs @@ -17,7 +17,6 @@ namespace Snowflake.Data.Tests.IntegrationTests using System.Collections.Generic; using System.Globalization; using Snowflake.Data.Tests.Mock; - using Snowflake.Data.Core; [TestFixture] class SFDbCommandITAsync : SFBaseTestAsync @@ -102,7 +101,7 @@ public void TestExecuteAsyncWithMaxRetryReached() using (DbConnection conn = new MockSnowflakeDbConnection(mockRestRequester)) { - string maxRetryConnStr = ConnectionString + "maxHttpRetries=5"; + string maxRetryConnStr = ConnectionString + "maxHttpRetries=8"; conn.ConnectionString = maxRetryConnStr; conn.Open(); @@ -124,10 +123,11 @@ public void TestExecuteAsyncWithMaxRetryReached() } stopwatch.Stop(); - // retry 5 times with backoff 1, 2, 4, 8, 16 seconds + var totalDelaySeconds = 1 + 2 + 4 + 8 + 16 + 16 + 16 + 16; + // retry 8 times with backoff 1, 2, 4, 8, 16, 16, 16, 16 seconds // but should not delay more than another 16 seconds - Assert.Less(stopwatch.ElapsedMilliseconds, 51 * 1000); - Assert.GreaterOrEqual(stopwatch.ElapsedMilliseconds, 30 * 1000); + Assert.Less(stopwatch.ElapsedMilliseconds, (totalDelaySeconds + 20) * 1000); + Assert.GreaterOrEqual(stopwatch.ElapsedMilliseconds, totalDelaySeconds * 1000); } } } @@ -594,7 +594,7 @@ public void TestExecuteWithMaxRetryReached() using (IDbConnection conn = new MockSnowflakeDbConnection(mockRestRequester)) { - string maxRetryConnStr = ConnectionString + "maxHttpRetries=5"; + string maxRetryConnStr = ConnectionString + "maxHttpRetries=8"; conn.ConnectionString = maxRetryConnStr; conn.Open(); @@ -615,10 +615,11 @@ public void TestExecuteWithMaxRetryReached() } stopwatch.Stop(); - // retry 5 times with backoff 1, 2, 4, 8, 16 seconds + var totalDelaySeconds = 1 + 2 + 4 + 8 + 16 + 16 + 16 + 16; + // retry 8 times with backoff 1, 2, 4, 8, 16, 16, 16, 16 seconds // but should not delay more than another 16 seconds - Assert.Less(stopwatch.ElapsedMilliseconds, 51 * 1000); - Assert.GreaterOrEqual(stopwatch.ElapsedMilliseconds, 30 * 1000); + Assert.Less(stopwatch.ElapsedMilliseconds, (totalDelaySeconds + 20) * 1000); + Assert.GreaterOrEqual(stopwatch.ElapsedMilliseconds, totalDelaySeconds * 1000); } } diff --git a/Snowflake.Data.Tests/UnitTests/ConnectionPoolManagerTest.cs b/Snowflake.Data.Tests/UnitTests/ConnectionPoolManagerTest.cs index ebcb85c5c..c0ddb6c02 100644 --- a/Snowflake.Data.Tests/UnitTests/ConnectionPoolManagerTest.cs +++ b/Snowflake.Data.Tests/UnitTests/ConnectionPoolManagerTest.cs @@ -3,7 +3,6 @@ */ using System; -using System.Collections.Generic; using System.Security; using System.Threading; using System.Threading.Tasks; @@ -34,12 +33,18 @@ public static void BeforeAllTests() } [OneTimeTearDown] - public void AfterAllTests() + public static void AfterAllTests() { s_poolConfig.Reset(); SessionPool.SessionFactory = new SessionFactory(); } + [SetUp] + public void BeforeEach() + { + _connectionPoolManager.ClearAllPools(); + } + [Test] public void TestPoolManagerReturnsSessionPoolForGivenConnectionString() { @@ -105,7 +110,6 @@ public async Task TestGetSessionAsyncWorksForSpecifiedConnectionString() } [Test] - [Ignore("Enable after completion of SNOW-937189")] // TODO: public void TestCountingOfSessionProvidedByPool() { // Act @@ -131,38 +135,33 @@ public void TestCountingOfSessionReturnedBackToPool() } [Test] - public void TestSetMaxPoolSizeForAllPools() + public void TestSetMaxPoolSizeForAllPoolsDisabled() { // Arrange - var sessionPool1 = _connectionPoolManager.GetPool(ConnectionString1, _password); - var sessionPool2 = _connectionPoolManager.GetPool(ConnectionString2, _password); + _connectionPoolManager.GetPool(ConnectionString1, _password); // Act - _connectionPoolManager.SetMaxPoolSize(3); + var thrown = Assert.Throws(() => _connectionPoolManager.SetMaxPoolSize(3)); // Assert - Assert.AreEqual(3, sessionPool1.GetMaxPoolSize()); - Assert.AreEqual(3, sessionPool2.GetMaxPoolSize()); + Assert.That(thrown.Message, Does.Contain("You cannot change connection pool parameters for all the pools. Instead you can change it on a particular pool")); } [Test] - public void TestSetTimeoutForAllPools() + public void TestSetTimeoutForAllPoolsDisabled() { // Arrange - var sessionPool1 = _connectionPoolManager.GetPool(ConnectionString1, _password); - var sessionPool2 = _connectionPoolManager.GetPool(ConnectionString2, _password); - + _connectionPoolManager.GetPool(ConnectionString1, _password); + // Act - _connectionPoolManager.SetTimeout(3000); + var thrown = Assert.Throws(() => _connectionPoolManager.SetTimeout(3000)); // Assert - Assert.AreEqual(3000, sessionPool1.GetTimeout()); - Assert.AreEqual(3000, sessionPool2.GetTimeout()); + Assert.That(thrown.Message, Does.Contain("You cannot change connection pool parameters for all the pools. Instead you can change it on a particular pool")); } [Test] - [Ignore("Enable when disabling pooling in connection string enabled - SNOW-902632")] - public void TestSetPoolingDisabledForAllPoolsNotPossible() + public void TestSetPoolingForAllPoolsDisabled() { // Arrange _connectionPoolManager.GetPool(ConnectionString1, _password); @@ -171,7 +170,7 @@ public void TestSetPoolingDisabledForAllPoolsNotPossible() var thrown = Assert.Throws(() => _connectionPoolManager.SetPooling(false)); // Assert - Assert.IsNotNull(thrown); + Assert.That(thrown.Message, Does.Contain("You cannot change connection pool parameters for all the pools. Instead you can change it on a particular pool")); } [Test] @@ -279,19 +278,10 @@ private void EnsurePoolSize(string connectionString, int requiredCurrentSize) { var sessionPool = _connectionPoolManager.GetPool(connectionString, _password); sessionPool.SetMaxPoolSize(requiredCurrentSize); - var busySessions = new List(); for (var i = 0; i < requiredCurrentSize; i++) { - var sfSession = _connectionPoolManager.GetSession(connectionString, _password); - busySessions.Add(sfSession); + _connectionPoolManager.GetSession(connectionString, _password); } - - foreach (var session in busySessions) // TODO: remove after SNOW-937189 since sessions will be already counted by GetCurrentPool size - { - session.close(); - _connectionPoolManager.AddSession(session); - } - Assert.AreEqual(requiredCurrentSize, sessionPool.GetCurrentPoolSize()); } } @@ -304,7 +294,7 @@ public SFSession NewSession(string connectionString, SecureString password) mockSfSession.Setup(x => x.Open()).Verifiable(); mockSfSession.Setup(x => x.OpenAsync(default)).Returns(Task.FromResult(this)); mockSfSession.Setup(x => x.IsNotOpen()).Returns(false); - mockSfSession.Setup(x => x.IsExpired(It.IsAny(), It.IsAny())).Returns(false); + mockSfSession.Setup(x => x.IsExpired(It.IsAny(), It.IsAny())).Returns(false); return mockSfSession.Object; } } diff --git a/Snowflake.Data.Tests/UnitTests/SFSessionPropertyTest.cs b/Snowflake.Data.Tests/UnitTests/SFSessionPropertyTest.cs index dd31f1c16..3ca2dec73 100644 --- a/Snowflake.Data.Tests/UnitTests/SFSessionPropertyTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFSessionPropertyTest.cs @@ -14,7 +14,7 @@ namespace Snowflake.Data.Tests.UnitTests class SFSessionPropertyTest { - [Test, TestCaseSource("ConnectionStringTestCases")] + [Test, TestCaseSource(nameof(ConnectionStringTestCases))] public void TestThatPropertiesAreParsed(TestCase testcase) { // act @@ -86,7 +86,13 @@ public static IEnumerable ConnectionStringTestCases() { SFSessionProperty.RETRY_TIMEOUT, defRetryTimeout }, { SFSessionProperty.MAXHTTPRETRIES, defMaxHttpRetries }, { SFSessionProperty.INCLUDERETRYREASON, defIncludeRetryReason }, - { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, defDisableQueryContextCache } + { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, defDisableQueryContextCache }, + { SFSessionProperty.MAXPOOLSIZE, DefaultValue(SFSessionProperty.MAXPOOLSIZE) }, + { SFSessionProperty.MINPOOLSIZE, DefaultValue(SFSessionProperty.MINPOOLSIZE) }, + { SFSessionProperty.CHANGEDSESSION, DefaultValue(SFSessionProperty.CHANGEDSESSION) }, + { SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT, DefaultValue(SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT) }, + { SFSessionProperty.EXPIRATIONTIMEOUT, DefaultValue(SFSessionProperty.EXPIRATIONTIMEOUT) }, + { SFSessionProperty.POOLINGENABLED, DefaultValue(SFSessionProperty.POOLINGENABLED) } } }; var testCaseWithBrowserResponseTimeout = new TestCase() @@ -112,7 +118,13 @@ public static IEnumerable ConnectionStringTestCases() { SFSessionProperty.RETRY_TIMEOUT, defRetryTimeout }, { SFSessionProperty.MAXHTTPRETRIES, defMaxHttpRetries }, { SFSessionProperty.INCLUDERETRYREASON, defIncludeRetryReason }, - { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, defDisableQueryContextCache } + { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, defDisableQueryContextCache }, + { SFSessionProperty.MAXPOOLSIZE, DefaultValue(SFSessionProperty.MAXPOOLSIZE) }, + { SFSessionProperty.MINPOOLSIZE, DefaultValue(SFSessionProperty.MINPOOLSIZE) }, + { SFSessionProperty.CHANGEDSESSION, DefaultValue(SFSessionProperty.CHANGEDSESSION) }, + { SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT, DefaultValue(SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT) }, + { SFSessionProperty.EXPIRATIONTIMEOUT, DefaultValue(SFSessionProperty.EXPIRATIONTIMEOUT) }, + { SFSessionProperty.POOLINGENABLED, DefaultValue(SFSessionProperty.POOLINGENABLED) } } }; var testCaseWithProxySettings = new TestCase() @@ -141,7 +153,13 @@ public static IEnumerable ConnectionStringTestCases() { SFSessionProperty.RETRY_TIMEOUT, defRetryTimeout }, { SFSessionProperty.MAXHTTPRETRIES, defMaxHttpRetries }, { SFSessionProperty.INCLUDERETRYREASON, defIncludeRetryReason }, - { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, defDisableQueryContextCache } + { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, defDisableQueryContextCache }, + { SFSessionProperty.MAXPOOLSIZE, DefaultValue(SFSessionProperty.MAXPOOLSIZE) }, + { SFSessionProperty.MINPOOLSIZE, DefaultValue(SFSessionProperty.MINPOOLSIZE) }, + { SFSessionProperty.CHANGEDSESSION, DefaultValue(SFSessionProperty.CHANGEDSESSION) }, + { SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT, DefaultValue(SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT) }, + { SFSessionProperty.EXPIRATIONTIMEOUT, DefaultValue(SFSessionProperty.EXPIRATIONTIMEOUT) }, + { SFSessionProperty.POOLINGENABLED, DefaultValue(SFSessionProperty.POOLINGENABLED) } }, ConnectionString = $"ACCOUNT={defAccount};USER={defUser};PASSWORD={defPassword};useProxy=true;proxyHost=proxy.com;proxyPort=1234;nonProxyHosts=localhost" @@ -172,7 +190,13 @@ public static IEnumerable ConnectionStringTestCases() { SFSessionProperty.RETRY_TIMEOUT, defRetryTimeout }, { SFSessionProperty.MAXHTTPRETRIES, defMaxHttpRetries }, { SFSessionProperty.INCLUDERETRYREASON, defIncludeRetryReason }, - { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, defDisableQueryContextCache } + { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, defDisableQueryContextCache }, + { SFSessionProperty.MAXPOOLSIZE, DefaultValue(SFSessionProperty.MAXPOOLSIZE) }, + { SFSessionProperty.MINPOOLSIZE, DefaultValue(SFSessionProperty.MINPOOLSIZE) }, + { SFSessionProperty.CHANGEDSESSION, DefaultValue(SFSessionProperty.CHANGEDSESSION) }, + { SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT, DefaultValue(SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT) }, + { SFSessionProperty.EXPIRATIONTIMEOUT, DefaultValue(SFSessionProperty.EXPIRATIONTIMEOUT) }, + { SFSessionProperty.POOLINGENABLED, DefaultValue(SFSessionProperty.POOLINGENABLED) } }, ConnectionString = $"ACCOUNT={defAccount};USER={defUser};PASSWORD={defPassword};proxyHost=proxy.com;proxyPort=1234;nonProxyHosts=localhost" @@ -202,7 +226,13 @@ public static IEnumerable ConnectionStringTestCases() { SFSessionProperty.MAXHTTPRETRIES, defMaxHttpRetries }, { SFSessionProperty.FILE_TRANSFER_MEMORY_THRESHOLD, "25" }, { SFSessionProperty.INCLUDERETRYREASON, defIncludeRetryReason }, - { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, defDisableQueryContextCache } + { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, defDisableQueryContextCache }, + { SFSessionProperty.MAXPOOLSIZE, DefaultValue(SFSessionProperty.MAXPOOLSIZE) }, + { SFSessionProperty.MINPOOLSIZE, DefaultValue(SFSessionProperty.MINPOOLSIZE) }, + { SFSessionProperty.CHANGEDSESSION, DefaultValue(SFSessionProperty.CHANGEDSESSION) }, + { SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT, DefaultValue(SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT) }, + { SFSessionProperty.EXPIRATIONTIMEOUT, DefaultValue(SFSessionProperty.EXPIRATIONTIMEOUT) }, + { SFSessionProperty.POOLINGENABLED, DefaultValue(SFSessionProperty.POOLINGENABLED) } } }; var testCaseWithIncludeRetryReason = new TestCase() @@ -229,7 +259,13 @@ public static IEnumerable ConnectionStringTestCases() { SFSessionProperty.RETRY_TIMEOUT, defRetryTimeout }, { SFSessionProperty.MAXHTTPRETRIES, defMaxHttpRetries }, { SFSessionProperty.INCLUDERETRYREASON, "false" }, - { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, defDisableQueryContextCache } + { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, defDisableQueryContextCache }, + { SFSessionProperty.MAXPOOLSIZE, DefaultValue(SFSessionProperty.MAXPOOLSIZE) }, + { SFSessionProperty.MINPOOLSIZE, DefaultValue(SFSessionProperty.MINPOOLSIZE) }, + { SFSessionProperty.CHANGEDSESSION, DefaultValue(SFSessionProperty.CHANGEDSESSION) }, + { SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT, DefaultValue(SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT) }, + { SFSessionProperty.EXPIRATIONTIMEOUT, DefaultValue(SFSessionProperty.EXPIRATIONTIMEOUT) }, + { SFSessionProperty.POOLINGENABLED, DefaultValue(SFSessionProperty.POOLINGENABLED) } } }; var testCaseWithDisableQueryContextCache = new TestCase() @@ -255,7 +291,13 @@ public static IEnumerable ConnectionStringTestCases() { SFSessionProperty.RETRY_TIMEOUT, defRetryTimeout }, { SFSessionProperty.MAXHTTPRETRIES, defMaxHttpRetries }, { SFSessionProperty.INCLUDERETRYREASON, defIncludeRetryReason }, - { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, "true" } + { SFSessionProperty.DISABLEQUERYCONTEXTCACHE, "true" }, + { SFSessionProperty.MAXPOOLSIZE, DefaultValue(SFSessionProperty.MAXPOOLSIZE) }, + { SFSessionProperty.MINPOOLSIZE, DefaultValue(SFSessionProperty.MINPOOLSIZE) }, + { SFSessionProperty.CHANGEDSESSION, DefaultValue(SFSessionProperty.CHANGEDSESSION) }, + { SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT, DefaultValue(SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT) }, + { SFSessionProperty.EXPIRATIONTIMEOUT, DefaultValue(SFSessionProperty.EXPIRATIONTIMEOUT) }, + { SFSessionProperty.POOLINGENABLED, DefaultValue(SFSessionProperty.POOLINGENABLED) } }, ConnectionString = $"ACCOUNT={defAccount};USER={defUser};PASSWORD={defPassword};DISABLEQUERYCONTEXTCACHE=true" @@ -271,6 +313,9 @@ public static IEnumerable ConnectionStringTestCases() testCaseWithDisableQueryContextCache }; } + + private static string DefaultValue(SFSessionProperty property) => + property.GetAttribute().defaultValue; internal class TestCase { diff --git a/Snowflake.Data.Tests/UnitTests/Session/ConnectionPoolConfigExtractorTest.cs b/Snowflake.Data.Tests/UnitTests/Session/ConnectionPoolConfigExtractorTest.cs new file mode 100644 index 000000000..60dbc2084 --- /dev/null +++ b/Snowflake.Data.Tests/UnitTests/Session/ConnectionPoolConfigExtractorTest.cs @@ -0,0 +1,304 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using Snowflake.Data.Client; +using Snowflake.Data.Core; +using Snowflake.Data.Core.Session; +using Snowflake.Data.Core.Tools; + +namespace Snowflake.Data.Tests.UnitTests.Session +{ + [TestFixture] + public class ConnectionPoolConfigExtractorTest + { + [Test] + public void TestExtractDefaultValues() + { + // arrange + var connectionString = "account=test;user=test;password=test;"; + + // act + var result = ExtractConnectionPoolConfig(connectionString); + + // assert + Assert.AreEqual(SFSessionHttpClientProperties.DefaultMaxPoolSize, result.MaxPoolSize, "max pool size"); + Assert.AreEqual(SFSessionHttpClientProperties.DefaultMinPoolSize, result.MinPoolSize, "min pool size"); + Assert.AreEqual(SFSessionHttpClientProperties.DefaultChangedSession, result.ChangedSession, "changed session"); + Assert.AreEqual(SFSessionHttpClientProperties.DefaultExpirationTimeout, result.ExpirationTimeout, "expiration timeout"); + Assert.AreEqual(SFSessionHttpClientProperties.DefaultWaitingForIdleSessionTimeout, result.WaitingForIdleSessionTimeout, "waiting for idle session timeout"); + Assert.AreEqual(SFSessionHttpClientProperties.DefaultConnectionTimeout, result.ConnectionTimeout, "connection timeout"); + Assert.AreEqual(SFSessionHttpClientProperties.DefaultPoolingEnabled, result.PoolingEnabled, "pooling enabled"); + } + + [Test] + public void TestExtractMaxPoolSize() + { + // arrange + var maxPoolSize = 15; + var connectionString = $"account=test;user=test;password=test;maxPoolSize={maxPoolSize}"; + + // act + var result = ExtractConnectionPoolConfig(connectionString); + + // assert + Assert.AreEqual(maxPoolSize, result.MaxPoolSize); + } + + [Test] + [TestCase("wrong_value")] + [TestCase("0")] + [TestCase("-1")] + public void TestExtractFailsForWrongValueOfMaxPoolSize(string maxPoolSize) + { + // arrange + var connectionString = $"account=test;user=test;password=test;maxPoolSize={maxPoolSize}"; + + // act + var thrown = Assert.Throws(() => ExtractConnectionPoolConfig(connectionString)); + + // assert + Assert.That(thrown.Message, Does.Contain($"Invalid value of parameter {SFSessionProperty.MAXPOOLSIZE.ToString()}")); + } + + [Test] + [TestCase("0", 0)] + [TestCase("7", 7)] + [TestCase("10", 10)] + public void TestExtractMinPoolSize(string propertyValue, int expectedMinPoolSize) + { + // arrange + var connectionString = $"account=test;user=test;password=test;minPoolSize={propertyValue}"; + + // act + var result = ExtractConnectionPoolConfig(connectionString); + + // assert + Assert.AreEqual(expectedMinPoolSize, result.MinPoolSize); + } + + [Test] + [TestCase("wrong_value")] + [TestCase("-1")] + public void TestExtractFailsForWrongValueOfMinPoolSize(string minPoolSize) + { + // arrange + var connectionString = $"account=test;user=test;password=test;minPoolSize={minPoolSize}"; + + // act + var thrown = Assert.Throws(() => ExtractConnectionPoolConfig(connectionString)); + + // assert + Assert.That(thrown.Message, Does.Contain($"Invalid value of parameter {SFSessionProperty.MINPOOLSIZE.ToString()}")); + } + + [Test] + public void TestExtractFailsWhenMinPoolSizeGreaterThanMaxPoolSize() + { + // arrange + var connectionString = $"account=test;user=test;password=test;minPoolSize=10;maxPoolSize=9"; + + // act + var thrown = Assert.Throws(() => ExtractConnectionPoolConfig(connectionString)); + + // assert + Assert.That(thrown.Message, Does.Contain("MinPoolSize cannot be greater than MaxPoolSize")); + } + + [Test] + [TestCaseSource(nameof(CorrectTimeoutsWithZeroUnchanged))] + public void TestExtractExpirationTimeout(TimeoutTestCase testCase) + { + // arrange + var connectionString = $"account=test;user=test;password=test;expirationTimeout={testCase.PropertyValue}"; + + // act + var result = ExtractConnectionPoolConfig(connectionString); + + // assert + Assert.AreEqual(testCase.ExpectedTimeout, result.ExpirationTimeout); + } + + [Test] + [TestCaseSource(nameof(IncorrectTimeouts))] + public void TestExtractExpirationTimeoutFailsWhenWrongValue(string propertyValue) + { + // arrange + var connectionString = $"account=test;user=test;password=test;expirationTimeout={propertyValue}"; + + // act + var thrown = Assert.Throws(() => ExtractConnectionPoolConfig(connectionString)); + + // assert + Assert.That(thrown.Message, Does.Contain($"Invalid value of parameter {SFSessionProperty.EXPIRATIONTIMEOUT.ToString()}")); + } + + [Test] + [TestCaseSource(nameof(PositiveTimeoutsAndZeroUnchanged))] + public void TestExtractWaitingForIdleSessionTimeout(TimeoutTestCase testCase) + { + // arrange + var connectionString = $"account=test;user=test;password=test;waitingForIdleSessionTimeout={testCase.PropertyValue}"; + + // act + var result = ExtractConnectionPoolConfig(connectionString); + + // assert + Assert.AreEqual(testCase.ExpectedTimeout, result.WaitingForIdleSessionTimeout); + } + + [Test] + public void TestExtractWaitingForIdleSessionTimeoutFailsForInfiniteTimeout() + { + // arrange + var connectionString = $"account=test;user=test;password=test;waitingForIdleSessionTimeout=-1"; + + // act + var thrown = Assert.Throws(() => ExtractConnectionPoolConfig(connectionString)); + + // assert + Assert.That(thrown.Message, Does.Contain("Waiting for idle session timeout cannot be infinite")); + } + + [Test] + [TestCaseSource(nameof(IncorrectTimeouts))] + public void TestExtractWaitingForIdleSessionTimeoutFailsWhenWrongValue(string propertyValue) + { + // arrange + var connectionString = $"account=test;user=test;password=test;waitingForIdleSessionTimeout={propertyValue}"; + + // act + var thrown = Assert.Throws(() => ExtractConnectionPoolConfig(connectionString)); + + // assert + Assert.That(thrown.Message, Does.Contain($"Invalid value of parameter {SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT.ToString()}")); + } + + [Test] + [TestCaseSource(nameof(CorrectTimeoutsWithZeroAsInfinite))] + public void TestExtractConnectionTimeout(TimeoutTestCase testCase) + { + // arrange + var connectionString = $"account=test;user=test;password=test;CONNECTION_TIMEOUT={testCase.PropertyValue};RETRY_TIMEOUT=60m"; + + // act + var result = ExtractConnectionPoolConfig(connectionString); + + // assert + Assert.AreEqual(testCase.ExpectedTimeout, result.ConnectionTimeout); + } + + [Test] + [TestCaseSource(nameof(IncorrectTimeouts))] + public void TestExtractConnectionTimeoutFailsForWrongValue(string propertyValue) + { + // arrange + var connectionString = $"account=test;user=test;password=test;CONNECTION_TIMEOUT={propertyValue}"; + + // act + var thrown = Assert.Throws(() => ExtractConnectionPoolConfig(connectionString)); + + // assert + Assert.That(thrown.Message, Does.Contain($"Invalid value of parameter {SFSessionProperty.CONNECTION_TIMEOUT.ToString()}")); + } + + [Test] + [TestCase("true", true)] + [TestCase("TRUE", true)] + [TestCase("false", false)] + [TestCase("FALSE", false)] + // [TestCase("0", false)] + public void TestExtractPoolingEnabled(string propertyValue, bool poolingEnabled) + { + // arrange + var connectionString = $"account=test;user=test;password=test;poolingEnabled={propertyValue}"; + + // act + var result = ExtractConnectionPoolConfig(connectionString); + + // assert + Assert.AreEqual(poolingEnabled, result.PoolingEnabled); + } + + [Test] + [TestCase("wrong_value")] + [TestCase("15")] + public void TestExtractFailsForWrongValueOfPoolingEnabled(string propertyValue) + { + // arrange + var connectionString = $"account=test;user=test;password=test;poolingEnabled={propertyValue}"; + + // act + var thrown = Assert.Throws(() => ExtractConnectionPoolConfig(connectionString)); + + // assert + Assert.That(thrown.Message, Does.Contain($"Invalid value of parameter {SFSessionProperty.POOLINGENABLED.ToString()}")); + } + + private ConnectionPoolConfig ExtractConnectionPoolConfig(string connectionString) + { + var properties = SFSessionProperties.parseConnectionString(connectionString, null); + var extractedProperties = SFSessionHttpClientProperties.ExtractAndValidate(properties); + return extractedProperties.BuildConnectionPoolConfig(); + } + + public class TimeoutTestCase + { + public string PropertyValue { get; } + public TimeSpan ExpectedTimeout { get; } + + public TimeoutTestCase(string propertyValue, TimeSpan expectedTimeout) + { + PropertyValue = propertyValue; + ExpectedTimeout = expectedTimeout; + } + } + + public static IEnumerable CorrectTimeoutsWithZeroUnchanged() => + CorrectTimeoutsWithoutZero().Concat(ZeroUnchangedTimeouts()); + + public static IEnumerable CorrectTimeoutsWithZeroAsInfinite() => + CorrectTimeoutsWithoutZero().Concat(ZeroAsInfiniteTimeouts()); + + public static IEnumerable PositiveTimeoutsAndZeroUnchanged() => + PositiveTimeouts().Concat(ZeroUnchangedTimeouts()); + + private static IEnumerable CorrectTimeoutsWithoutZero() => + NegativeAsInfinityTimeouts().Concat(PositiveTimeouts()); + + private static IEnumerable NegativeAsInfinityTimeouts() + { + yield return new TimeoutTestCase("-1", TimeoutHelper.Infinity()); + } + + private static IEnumerable PositiveTimeouts() + { + yield return new TimeoutTestCase("5", TimeSpan.FromSeconds(5)); + yield return new TimeoutTestCase("6s", TimeSpan.FromSeconds(6)); + yield return new TimeoutTestCase("7S", TimeSpan.FromSeconds(7)); + yield return new TimeoutTestCase("8m", TimeSpan.FromMinutes(8)); + yield return new TimeoutTestCase("9M", TimeSpan.FromMinutes(9)); + yield return new TimeoutTestCase("10ms", TimeSpan.FromMilliseconds(10)); + yield return new TimeoutTestCase("11ms", TimeSpan.FromMilliseconds(11)); + } + + private static IEnumerable ZeroAsInfiniteTimeouts() + { + yield return new TimeoutTestCase("0", TimeoutHelper.Infinity()); + yield return new TimeoutTestCase("0ms", TimeoutHelper.Infinity()); + } + + private static IEnumerable ZeroUnchangedTimeouts() + { + yield return new TimeoutTestCase("0", TimeSpan.Zero); + yield return new TimeoutTestCase("0ms", TimeSpan.Zero); + } + + public static IEnumerable IncorrectTimeouts() + { + yield return "wrong value"; + yield return "1h"; + yield return "1s1s"; + } + } +} diff --git a/Snowflake.Data.Tests/UnitTests/Session/NonCountingSessionCreationTokenCounterTest.cs b/Snowflake.Data.Tests/UnitTests/Session/NonCountingSessionCreationTokenCounterTest.cs index 7c4a3ae99..9f73c7c7a 100644 --- a/Snowflake.Data.Tests/UnitTests/Session/NonCountingSessionCreationTokenCounterTest.cs +++ b/Snowflake.Data.Tests/UnitTests/Session/NonCountingSessionCreationTokenCounterTest.cs @@ -1,4 +1,5 @@ using NUnit.Framework; +using Snowflake.Data.Core; using Snowflake.Data.Core.Session; namespace Snowflake.Data.Tests.UnitTests.Session @@ -39,7 +40,7 @@ public void TestCompleteUnknownTokenDoesNotThrowExceptions() // arrange var tokens = new NonCountingSessionCreationTokenCounter(); tokens.NewToken(); - var unknownToken = new SessionCreationToken(0); + var unknownToken = new SessionCreationToken(SFSessionHttpClientProperties.DefaultConnectionTimeout); // act tokens.RemoveToken(unknownToken); diff --git a/Snowflake.Data.Tests/UnitTests/Session/SFHttpClientPropertiesTest.cs b/Snowflake.Data.Tests/UnitTests/Session/SFHttpClientPropertiesTest.cs index ea6deb5ca..fadb9dca3 100644 --- a/Snowflake.Data.Tests/UnitTests/Session/SFHttpClientPropertiesTest.cs +++ b/Snowflake.Data.Tests/UnitTests/Session/SFHttpClientPropertiesTest.cs @@ -2,15 +2,15 @@ * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved. */ +using System; using System.Collections.Generic; -using Moq; using NUnit.Framework; using Snowflake.Data.Core; +using Snowflake.Data.Core.Tools; using Snowflake.Data.Tests.Util; namespace Snowflake.Data.Tests.UnitTests.Session { - [TestFixture] public class SFHttpClientPropertiesTest { @@ -32,11 +32,11 @@ public void ShouldConvertToMapOnly2Properties( { validateDefaultParameters = validateDefaultParameters, clientSessionKeepAlive = clientSessionKeepAlive, - timeoutInSec = SFSessionHttpClientProperties.s_retryTimeoutDefault, + connectionTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, insecureMode = false, disableRetry = false, forceRetryOn404 = false, - retryTimeout = SFSessionHttpClientProperties.s_retryTimeoutDefault, + retryTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, maxHttpRetries = 7, proxyProperties = proxyProperties }; @@ -66,11 +66,11 @@ public void ShouldBuildHttpClientConfig() { validateDefaultParameters = TestDataGenarator.NextBool(), clientSessionKeepAlive = TestDataGenarator.NextBool(), - timeoutInSec = TestDataGenarator.NextInt(30, 151), + connectionTimeout = TimeSpan.FromSeconds(TestDataGenarator.NextInt(30, 151)), insecureMode = TestDataGenarator.NextBool(), disableRetry = TestDataGenarator.NextBool(), forceRetryOn404 = TestDataGenarator.NextBool(), - retryTimeout = TestDataGenarator.NextInt(300, 600), + retryTimeout = TimeSpan.FromSeconds(TestDataGenarator.NextInt(300, 600)), maxHttpRetries = TestDataGenarator.NextInt(0, 15), proxyProperties = proxyProperties }; @@ -94,29 +94,22 @@ public void ShouldBuildHttpClientConfig() public void ShouldExtractProperties(PropertiesTestCase testCase) { // given - var proxyExtractorMock = new Moq.Mock(); - var extractor = new SFSessionHttpClientProperties.Extractor(proxyExtractorMock.Object); var properties = SFSessionProperties.parseConnectionString(testCase.conectionString, null); var proxyProperties = new SFSessionHttpClientProxyProperties(); - proxyExtractorMock - .Setup(e => e.ExtractProperties(properties)) - .Returns(proxyProperties); // when - var extractedProperties = extractor.ExtractProperties(properties); - extractedProperties.CheckPropertiesAreValid(); + var extractedProperties = SFSessionHttpClientProperties.ExtractAndValidate(properties); // then Assert.AreEqual(testCase.expectedProperties.validateDefaultParameters, extractedProperties.validateDefaultParameters); Assert.AreEqual(testCase.expectedProperties.clientSessionKeepAlive, extractedProperties.clientSessionKeepAlive); - Assert.AreEqual(testCase.expectedProperties.timeoutInSec, extractedProperties.timeoutInSec); + Assert.AreEqual(testCase.expectedProperties.connectionTimeout, extractedProperties.connectionTimeout); Assert.AreEqual(testCase.expectedProperties.insecureMode, extractedProperties.insecureMode); Assert.AreEqual(testCase.expectedProperties.disableRetry, extractedProperties.disableRetry); Assert.AreEqual(testCase.expectedProperties.forceRetryOn404, extractedProperties.forceRetryOn404); Assert.AreEqual(testCase.expectedProperties.retryTimeout, extractedProperties.retryTimeout); Assert.AreEqual(testCase.expectedProperties.maxHttpRetries, extractedProperties.maxHttpRetries); - Assert.AreEqual(proxyProperties, extractedProperties.proxyProperties); - proxyExtractorMock.Verify(e => e.ExtractProperties(properties), Times.Once); + Assert.NotNull(proxyProperties); } public static IEnumerable PropertiesProvider() @@ -128,12 +121,12 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = true, clientSessionKeepAlive = false, - timeoutInSec = SFSessionHttpClientProperties.s_retryTimeoutDefault, + connectionTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, insecureMode = false, disableRetry = false, forceRetryOn404 = false, - retryTimeout = SFSessionHttpClientProperties.s_retryTimeoutDefault, - maxHttpRetries = SFSessionHttpClientProperties.s_maxHttpRetriesDefault + retryTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, + maxHttpRetries = SFSessionHttpClientProperties.DefaultMaxHttpRetries } }; var propertiesWithValidateDefaultParametersChanged = new PropertiesTestCase() @@ -143,12 +136,12 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = false, clientSessionKeepAlive = false, - timeoutInSec = SFSessionHttpClientProperties.s_retryTimeoutDefault, + connectionTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, insecureMode = false, disableRetry = false, forceRetryOn404 = false, - retryTimeout = SFSessionHttpClientProperties.s_retryTimeoutDefault, - maxHttpRetries = SFSessionHttpClientProperties.s_maxHttpRetriesDefault + retryTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, + maxHttpRetries = SFSessionHttpClientProperties.DefaultMaxHttpRetries } }; var propertiesWithClientSessionKeepAliveChanged = new PropertiesTestCase() @@ -158,12 +151,12 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = true, clientSessionKeepAlive = true, - timeoutInSec = SFSessionHttpClientProperties.s_retryTimeoutDefault, + connectionTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, insecureMode = false, disableRetry = false, forceRetryOn404 = false, - retryTimeout = SFSessionHttpClientProperties.s_retryTimeoutDefault, - maxHttpRetries = SFSessionHttpClientProperties.s_maxHttpRetriesDefault + retryTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, + maxHttpRetries = SFSessionHttpClientProperties.DefaultMaxHttpRetries } }; var propertiesWithTimeoutChanged = new PropertiesTestCase() @@ -173,12 +166,12 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = true, clientSessionKeepAlive = false, - timeoutInSec = 15, + connectionTimeout = TimeSpan.FromSeconds(15), insecureMode = false, disableRetry = false, forceRetryOn404 = false, - retryTimeout = SFSessionHttpClientProperties.s_retryTimeoutDefault, - maxHttpRetries = SFSessionHttpClientProperties.s_maxHttpRetriesDefault + retryTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, + maxHttpRetries = SFSessionHttpClientProperties.DefaultMaxHttpRetries } }; var propertiesWithInsecureModeChanged = new PropertiesTestCase() @@ -188,12 +181,12 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = true, clientSessionKeepAlive = false, - timeoutInSec = SFSessionHttpClientProperties.s_retryTimeoutDefault, + connectionTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, insecureMode = true, disableRetry = false, forceRetryOn404 = false, - retryTimeout = SFSessionHttpClientProperties.s_retryTimeoutDefault, - maxHttpRetries = SFSessionHttpClientProperties.s_maxHttpRetriesDefault + retryTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, + maxHttpRetries = SFSessionHttpClientProperties.DefaultMaxHttpRetries } }; var propertiesWithDisableRetryChanged = new PropertiesTestCase() @@ -203,12 +196,12 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = true, clientSessionKeepAlive = false, - timeoutInSec = SFSessionHttpClientProperties.s_retryTimeoutDefault, + connectionTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, insecureMode = false, disableRetry = true, forceRetryOn404 = false, - retryTimeout = SFSessionHttpClientProperties.s_retryTimeoutDefault, - maxHttpRetries = SFSessionHttpClientProperties.s_maxHttpRetriesDefault + retryTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, + maxHttpRetries = SFSessionHttpClientProperties.DefaultMaxHttpRetries } }; var propertiesWithForceRetryOn404Changed = new PropertiesTestCase() @@ -218,12 +211,12 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = true, clientSessionKeepAlive = false, - timeoutInSec = SFSessionHttpClientProperties.s_retryTimeoutDefault, + connectionTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, insecureMode = false, disableRetry = false, forceRetryOn404 = true, - retryTimeout = SFSessionHttpClientProperties.s_retryTimeoutDefault, - maxHttpRetries = SFSessionHttpClientProperties.s_maxHttpRetriesDefault + retryTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, + maxHttpRetries = SFSessionHttpClientProperties.DefaultMaxHttpRetries } }; var propertiesWithRetryTimeoutChangedToAValueAbove300 = new PropertiesTestCase() @@ -233,12 +226,12 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = true, clientSessionKeepAlive = false, - timeoutInSec = SFSessionHttpClientProperties.s_retryTimeoutDefault, + connectionTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, insecureMode = false, disableRetry = false, forceRetryOn404 = false, - retryTimeout = 600, - maxHttpRetries = SFSessionHttpClientProperties.s_maxHttpRetriesDefault + retryTimeout = TimeSpan.FromSeconds(600), + maxHttpRetries = SFSessionHttpClientProperties.DefaultMaxHttpRetries } }; var propertiesWithRetryTimeoutChangedToAValueBelow300 = new PropertiesTestCase() @@ -248,12 +241,12 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = true, clientSessionKeepAlive = false, - timeoutInSec = SFSessionHttpClientProperties.s_retryTimeoutDefault, + connectionTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, insecureMode = false, disableRetry = false, forceRetryOn404 = false, - retryTimeout = SFSessionHttpClientProperties.s_retryTimeoutDefault, - maxHttpRetries = SFSessionHttpClientProperties.s_maxHttpRetriesDefault + retryTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, + maxHttpRetries = SFSessionHttpClientProperties.DefaultMaxHttpRetries } }; var propertiesWithRetryTimeoutChangedToZero = new PropertiesTestCase() @@ -263,12 +256,12 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = true, clientSessionKeepAlive = false, - timeoutInSec = 0, + connectionTimeout = SFSessionHttpClientProperties.DefaultConnectionTimeout, insecureMode = false, disableRetry = false, forceRetryOn404 = false, - retryTimeout = 0, - maxHttpRetries = SFSessionHttpClientProperties.s_maxHttpRetriesDefault + retryTimeout = TimeoutHelper.Infinity(), + maxHttpRetries = SFSessionHttpClientProperties.DefaultMaxHttpRetries } }; var propertiesWithMaxHttpRetriesChangedToAValueAbove7 = new PropertiesTestCase() @@ -278,11 +271,11 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = true, clientSessionKeepAlive = false, - timeoutInSec = SFSessionHttpClientProperties.s_retryTimeoutDefault, + connectionTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, insecureMode = false, disableRetry = false, forceRetryOn404 = false, - retryTimeout = SFSessionHttpClientProperties.s_retryTimeoutDefault, + retryTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, maxHttpRetries = 10 } }; @@ -293,12 +286,12 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = true, clientSessionKeepAlive = false, - timeoutInSec = SFSessionHttpClientProperties.s_retryTimeoutDefault, + connectionTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, insecureMode = false, disableRetry = false, forceRetryOn404 = false, - retryTimeout = SFSessionHttpClientProperties.s_retryTimeoutDefault, - maxHttpRetries = SFSessionHttpClientProperties.s_maxHttpRetriesDefault + retryTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, + maxHttpRetries = SFSessionHttpClientProperties.DefaultMaxHttpRetries } }; var propertiesWithMaxHttpRetriesChangedToZero = new PropertiesTestCase() @@ -308,11 +301,11 @@ public static IEnumerable PropertiesProvider() { validateDefaultParameters = true, clientSessionKeepAlive = false, - timeoutInSec = SFSessionHttpClientProperties.s_retryTimeoutDefault, + connectionTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, insecureMode = false, disableRetry = false, forceRetryOn404 = false, - retryTimeout = SFSessionHttpClientProperties.s_retryTimeoutDefault, + retryTimeout = SFSessionHttpClientProperties.DefaultRetryTimeout, maxHttpRetries = 0 } }; diff --git a/Snowflake.Data.Tests/UnitTests/Session/SessionCreationTokenCounterTest.cs b/Snowflake.Data.Tests/UnitTests/Session/SessionCreationTokenCounterTest.cs index 8c00e7533..4cbb7dcb4 100644 --- a/Snowflake.Data.Tests/UnitTests/Session/SessionCreationTokenCounterTest.cs +++ b/Snowflake.Data.Tests/UnitTests/Session/SessionCreationTokenCounterTest.cs @@ -1,5 +1,7 @@ +using System; using System.Threading; using NUnit.Framework; +using Snowflake.Data.Core; using Snowflake.Data.Core.Session; namespace Snowflake.Data.Tests.UnitTests.Session @@ -7,14 +9,14 @@ namespace Snowflake.Data.Tests.UnitTests.Session [TestFixture] public class SessionCreationTokenCounterTest { - private const long LongTimeAsMillis = 30000; - private const long ShortTimeAsMillis = 50; + private static readonly TimeSpan s_longTime = TimeSpan.FromSeconds(30); + private static readonly TimeSpan s_shortTime = TimeSpan.FromMilliseconds(50); [Test] public void TestGrantSessionCreation() { // arrange - var tokens = new SessionCreationTokenCounter(LongTimeAsMillis); + var tokens = new SessionCreationTokenCounter(s_longTime); // act tokens.NewToken(); @@ -33,7 +35,7 @@ public void TestGrantSessionCreation() public void TestCompleteSessionCreation() { // arrange - var tokens = new SessionCreationTokenCounter(LongTimeAsMillis); + var tokens = new SessionCreationTokenCounter(s_longTime); var token1 = tokens.NewToken(); var token2 = tokens.NewToken(); @@ -54,9 +56,9 @@ public void TestCompleteSessionCreation() public void TestCompleteUnknownTokenDoesNotThrowExceptions() { // arrange - var tokens = new SessionCreationTokenCounter(LongTimeAsMillis); + var tokens = new SessionCreationTokenCounter(s_longTime); tokens.NewToken(); - var unknownToken = new SessionCreationToken(0); + var unknownToken = new SessionCreationToken(SFSessionHttpClientProperties.DefaultConnectionTimeout); // act tokens.RemoveToken(unknownToken); @@ -69,11 +71,12 @@ public void TestCompleteUnknownTokenDoesNotThrowExceptions() public void TestCompleteCleansExpiredTokens() { // arrange - var tokens = new SessionCreationTokenCounter(ShortTimeAsMillis); + var tokens = new SessionCreationTokenCounter(s_shortTime); var token = tokens.NewToken(); tokens.NewToken(); // this token will be cleaned because of expiration Assert.AreEqual(2, tokens.Count()); - Thread.Sleep((int) ShortTimeAsMillis + 5); + const int EpsilonMillis = 5; + Thread.Sleep((int) s_shortTime.TotalMilliseconds + EpsilonMillis); // act tokens.RemoveToken(token); diff --git a/Snowflake.Data.Tests/UnitTests/Session/SessionCreationTokenTest.cs b/Snowflake.Data.Tests/UnitTests/Session/SessionCreationTokenTest.cs index 4a43c044c..13b45b9b1 100644 --- a/Snowflake.Data.Tests/UnitTests/Session/SessionCreationTokenTest.cs +++ b/Snowflake.Data.Tests/UnitTests/Session/SessionCreationTokenTest.cs @@ -7,13 +7,13 @@ namespace Snowflake.Data.Tests.UnitTests.Session [TestFixture] public class SessionCreationTokenTest { - private const long Timeout30SecondsAsMillis = 30000; + private static readonly TimeSpan s_timeout30Seconds = TimeSpan.FromSeconds(30); [Test] public void TestTokenIsNotExpired() { // arrange - var token = new SessionCreationToken(Timeout30SecondsAsMillis); + var token = new SessionCreationToken(s_timeout30Seconds); // act var isExpired = token.IsExpired(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); @@ -26,10 +26,11 @@ public void TestTokenIsNotExpired() public void TestTokenIsExpired() { // arrange - var token = new SessionCreationToken(Timeout30SecondsAsMillis); + var token = new SessionCreationToken(s_timeout30Seconds); + var timeout30SecondsAsMillis = (long) s_timeout30Seconds.TotalMilliseconds; // act - var isExpired = token.IsExpired(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + Timeout30SecondsAsMillis + 1); + var isExpired = token.IsExpired(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + timeout30SecondsAsMillis + 1); // assert Assert.IsTrue(isExpired); diff --git a/Snowflake.Data.Tests/UnitTests/Session/SessionPoolTest.cs b/Snowflake.Data.Tests/UnitTests/Session/SessionPoolTest.cs new file mode 100644 index 000000000..a4b54d6b9 --- /dev/null +++ b/Snowflake.Data.Tests/UnitTests/Session/SessionPoolTest.cs @@ -0,0 +1,65 @@ +using NUnit.Framework; +using Snowflake.Data.Core.Session; + +namespace Snowflake.Data.Tests.UnitTests.Session +{ + [TestFixture] + public class SessionPoolTest + { + private const string ConnectionString = "ACCOUNT=testaccount;USER=testuser;PASSWORD=testpassword;"; + + [Test] + public void TestPoolParametersAreNotOverriden() + { + // act + var pool = SessionPool.CreateSessionPool(ConnectionString, null); + + // assert + Assert.IsFalse(pool.IsConfigOverridden()); + } + + [Test] + public void TestOverrideMaxPoolSize() + { + // arrange + var pool = SessionPool.CreateSessionPool(ConnectionString, null); + var newMaxPoolSize = 15; + + // act + pool.SetMaxPoolSize(newMaxPoolSize); + + // assert + Assert.AreEqual(newMaxPoolSize, pool.GetMaxPoolSize()); + Assert.IsTrue(pool.IsConfigOverridden()); + } + + [Test] + public void TestOverrideExpirationTimeout() + { + // arrange + var pool = SessionPool.CreateSessionPool(ConnectionString, null); + var newExpirationTimeoutSeconds = 15; + + // act + pool.SetTimeout(newExpirationTimeoutSeconds); + + // assert + Assert.AreEqual(newExpirationTimeoutSeconds, pool.GetTimeout()); + Assert.IsTrue(pool.IsConfigOverridden()); + } + + [Test] + public void TestOverrideSetPooling() + { + // arrange + var pool = SessionPool.CreateSessionPool(ConnectionString, null); + + // act + pool.SetPooling(false); + + // assert + Assert.IsFalse(pool.GetPooling()); + Assert.IsTrue(pool.IsConfigOverridden()); + } + } +} diff --git a/Snowflake.Data.Tests/UnitTests/Session/SessionPropertiesWithDefaultValuesExtractorTest.cs b/Snowflake.Data.Tests/UnitTests/Session/SessionPropertiesWithDefaultValuesExtractorTest.cs new file mode 100644 index 000000000..9612ba82f --- /dev/null +++ b/Snowflake.Data.Tests/UnitTests/Session/SessionPropertiesWithDefaultValuesExtractorTest.cs @@ -0,0 +1,213 @@ +using System; +using NUnit.Framework; +using Snowflake.Data.Core; +using Snowflake.Data.Core.Session; + +namespace Snowflake.Data.Tests.UnitTests.Session +{ + [TestFixture] + public class SessionPropertiesWithDefaultValuesExtractorTest + { + [Test] + public void TestReturnExtractedValue() + { + // arrange + var properties = SFSessionProperties.parseConnectionString("account=test;user=test;password=test;connection_timeout=15", null); + var extractor = new SessionPropertiesWithDefaultValuesExtractor(properties, false); + + // act + var value = extractor.ExtractPropertyWithDefaultValue( + SFSessionProperty.CONNECTION_TIMEOUT, + int.Parse, + s => true, + i => true + ); + + // assert + Assert.AreEqual(15, value); + } + + [Test] + public void TestReturnDefaultValueWhenValueIsMissing( + [Values] bool failOnWrongValue) + { + // arrange + var properties = SFSessionProperties.parseConnectionString($"account=test;user=test;password=test", null); + var extractor = new SessionPropertiesWithDefaultValuesExtractor(properties, false); + var defaultValue = GetDefaultIntSessionProperty(SFSessionProperty.CONNECTION_TIMEOUT); + + // act + var value = extractor.ExtractPropertyWithDefaultValue( + SFSessionProperty.CONNECTION_TIMEOUT, + int.Parse, + s => true, + i => true + ); + + // assert + Assert.AreEqual(defaultValue, value); + } + + [Test] + public void TestReturnDefaultValueWhenPreValidationFails() + { + // arrange + var properties = SFSessionProperties.parseConnectionString("account=test;user=test;password=test;connection_timeout=15", null); + var extractor = new SessionPropertiesWithDefaultValuesExtractor(properties, false); + var defaultValue = GetDefaultIntSessionProperty(SFSessionProperty.CONNECTION_TIMEOUT); + + // act + var value = extractor.ExtractPropertyWithDefaultValue( + SFSessionProperty.CONNECTION_TIMEOUT, + int.Parse, + s => false, + i => true + ); + + // assert + Assert.AreEqual(defaultValue, value); + } + + [Test] + public void TestFailForPropertyWithInvalidDefaultValue() + { + // arrange + var properties = SFSessionProperties.parseConnectionString("account=test;user=test;password=test;", null); + var extractor = new SessionPropertiesWithDefaultValuesExtractor(properties, false); + + // act + var thrown = Assert.Throws(() => extractor.ExtractPropertyWithDefaultValue( + SFSessionProperty.CONNECTION_TIMEOUT, + s => s, + s => true, + s => false)); + + // assert + Assert.That(thrown.Message, Does.Contain("Invalid default value of CONNECTION_TIMEOUT")); + } + + [Test] + public void TestReturnDefaultValueForNullProperty() + { + // arrange + var properties = SFSessionProperties.parseConnectionString("account=test;user=test;password=test;", null); + properties[SFSessionProperty.CONNECTION_TIMEOUT] = null; + var extractor = new SessionPropertiesWithDefaultValuesExtractor(properties, false); + var defaultValue = GetDefaultIntSessionProperty(SFSessionProperty.CONNECTION_TIMEOUT); + + // act + var value = extractor.ExtractPropertyWithDefaultValue( + SFSessionProperty.CONNECTION_TIMEOUT, + int.Parse, + s => true, + i => true); + + // assert + Assert.AreEqual(defaultValue, value); + } + + [Test] + public void TestReturnDefaultValueWhenPostValidationFails() + { + // arrange + var properties = SFSessionProperties.parseConnectionString("account=test;user=test;password=test;connection_timeout=15", null); + var extractor = new SessionPropertiesWithDefaultValuesExtractor(properties, false); + var defaultValue = GetDefaultIntSessionProperty(SFSessionProperty.CONNECTION_TIMEOUT); + + // act + var value = extractor.ExtractPropertyWithDefaultValue( + SFSessionProperty.CONNECTION_TIMEOUT, + int.Parse, + s => true, + i => i == defaultValue + ); + + // assert + Assert.AreEqual(defaultValue, value); + } + + [Test] + public void TestReturnDefaultValueWhenExtractFails() + { + // arrange + var properties = SFSessionProperties.parseConnectionString("account=test;user=test;password=test;connection_timeout=15X", null); + var extractor = new SessionPropertiesWithDefaultValuesExtractor(properties, false); + var defaultValue = GetDefaultIntSessionProperty(SFSessionProperty.CONNECTION_TIMEOUT); + + // act + var value = extractor.ExtractPropertyWithDefaultValue( + SFSessionProperty.CONNECTION_TIMEOUT, + int.Parse, + s => true, + i => true + ); + + // assert + Assert.AreEqual(defaultValue, value); + } + + [Test] + public void TestFailWhenPreValidationFails() + { + // arrange + var properties = SFSessionProperties.parseConnectionString("account=test;user=test;password=test;connection_timeout=15", null); + var extractor = new SessionPropertiesWithDefaultValuesExtractor(properties, true); + + // act + var thrown = Assert.Throws(() => + extractor.ExtractPropertyWithDefaultValue( + SFSessionProperty.CONNECTION_TIMEOUT, + int.Parse, + s => false, + i => true + )); + + // assert + Assert.That(thrown.Message, Does.Contain("Invalid value of parameter CONNECTION_TIMEOUT")); + } + + [Test] + public void TestFailWhenPostValidationFails() + { + // arrange + var properties = SFSessionProperties.parseConnectionString("account=test;user=test;password=test;connection_timeout=15", null); + var extractor = new SessionPropertiesWithDefaultValuesExtractor(properties, true); + var defaultValue = GetDefaultIntSessionProperty(SFSessionProperty.CONNECTION_TIMEOUT); + + // act + var thrown = Assert.Throws(() => + extractor.ExtractPropertyWithDefaultValue( + SFSessionProperty.CONNECTION_TIMEOUT, + int.Parse, + s => true, + i => i == defaultValue + )); + + // assert + Assert.That(thrown.Message, Does.Contain("Invalid value of parameter CONNECTION_TIMEOUT")); + } + + [Test] + public void TestFailWhenExtractFails() + { + // arrange + var properties = SFSessionProperties.parseConnectionString("account=test;user=test;password=test;connection_timeout=15X", null); + var extractor = new SessionPropertiesWithDefaultValuesExtractor(properties, true); + + // act + var thrown = Assert.Throws(() => + extractor.ExtractPropertyWithDefaultValue( + SFSessionProperty.CONNECTION_TIMEOUT, + int.Parse, + s => true, + i => true + )); + + // assert + Assert.That(thrown.Message, Does.Contain("Invalid value of parameter CONNECTION_TIMEOUT")); + } + + private int GetDefaultIntSessionProperty(SFSessionProperty property) => + int.Parse(SFSessionProperty.CONNECTION_TIMEOUT.GetAttribute().defaultValue); + } +} diff --git a/Snowflake.Data.Tests/UnitTests/Tools/TimeoutHelperTest.cs b/Snowflake.Data.Tests/UnitTests/Tools/TimeoutHelperTest.cs new file mode 100644 index 000000000..743eab111 --- /dev/null +++ b/Snowflake.Data.Tests/UnitTests/Tools/TimeoutHelperTest.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using NUnit.Framework; +using Snowflake.Data.Core.Tools; + +namespace Snowflake.Data.Tests.UnitTests.Tools +{ + [TestFixture] + public class TimeoutHelperTest + { + [Test] + [TestCaseSource(nameof(InfiniteTimeouts))] + public void TestInfinity(TimeSpan infiniteTimeout) + { + // act + var isInfinite = TimeoutHelper.IsInfinite(infiniteTimeout); + + // assert + Assert.IsTrue(isInfinite); + } + + [Test] + [TestCaseSource(nameof(FiniteTimeouts))] + public void TestFiniteValue(TimeSpan finiteTimeout) + { + // act + var isInfinite = TimeoutHelper.IsInfinite(finiteTimeout); + + // assert + Assert.IsFalse(isInfinite); + } + + [Test] + [TestCaseSource(nameof(ZeroLengthTimeouts))] + public void TestZeroLength(TimeSpan zeroTimeout) + { + // act + var isZeroLength = TimeoutHelper.IsZeroLength(zeroTimeout); + + // assert + Assert.IsTrue(isZeroLength); + } + + [Test] + [TestCaseSource(nameof(NonZeroLengthTimeouts))] + public void TestNonZeroLength(TimeSpan nonZeroTimeout) + { + // act + var isZeroLength = TimeoutHelper.IsZeroLength(nonZeroTimeout); + + // assert + Assert.IsFalse(isZeroLength); + } + + [Test] + [TestCase(1000, 1000)] + [TestCase(1000, 2000)] + public void TestInfiniteTimeoutDoesNotExpire(long startedAtMillis, long nowMillis) + { + // act + var isExpired = TimeoutHelper.IsExpired(startedAtMillis, nowMillis, TimeoutHelper.Infinity()); + + // assert + Assert.IsFalse(isExpired); + } + + [Test] + [TestCase(1000, 1000, 0, true)] + [TestCase(1000, 2000, 0, true)] + [TestCase(1000, 1100, 100, true)] + [TestCase(1000, 1099, 100, false)] + [TestCase(1000, 2000, 100, true)] + public void TestExpiredTimeout(long startedAtMillis, long nowMillis, long timeoutMillis, bool expectedIsExpired) + { + // arrange + var timeout = TimeSpan.FromMilliseconds(timeoutMillis); + + // act + var isExpired = TimeoutHelper.IsExpired(startedAtMillis, nowMillis, timeout); + + // assert + Assert.AreEqual(expectedIsExpired, isExpired); + } + + [Test] + public void TestFiniteTimeoutLeftFailsForInfiniteTimeout() + { + // act + var thrown = Assert.Throws(() => + TimeoutHelper.FiniteTimeoutLeftMillis(1000, 2000, TimeoutHelper.Infinity())); + + // assert + Assert.That(thrown.Message, Does.Contain("Infinite timeout cannot be used to determine milliseconds left")); + } + + + [Test] + [TestCase(1000, 1000, 0, 0)] + [TestCase(1000, 2000, 0, 0)] + [TestCase(1000, 1100, 100, 0)] + [TestCase(1000, 1095, 100, 5)] + public void TestFiniteTimeoutLeft(long startedAtMillis, long nowMillis, long timeoutMillis, long expectedMillisLeft) + { + // arrange + var timeout = TimeSpan.FromMilliseconds(timeoutMillis); + + // act + var millisLeft = TimeoutHelper.FiniteTimeoutLeftMillis(startedAtMillis, nowMillis, timeout); + + // assert + Assert.AreEqual(expectedMillisLeft, millisLeft); + } + + public static IEnumerable InfiniteTimeouts() + { + yield return TimeoutHelper.Infinity(); + yield return TimeSpan.FromMilliseconds(-1); + } + + public static IEnumerable FiniteTimeouts() + { + yield return TimeSpan.Zero; + yield return TimeSpan.FromMilliseconds(1); + yield return TimeSpan.FromSeconds(2); + } + + public static IEnumerable ZeroLengthTimeouts() + { + yield return TimeSpan.Zero; + yield return TimeSpan.FromMilliseconds(0); + yield return TimeSpan.FromSeconds(0); + } + + public static IEnumerable NonZeroLengthTimeouts() + { + yield return TimeoutHelper.Infinity(); + yield return TimeSpan.FromMilliseconds(3); + yield return TimeSpan.FromSeconds(5); + } + } +} diff --git a/Snowflake.Data/Core/Session/ChangedSessionBehavior.cs b/Snowflake.Data/Core/Session/ChangedSessionBehavior.cs new file mode 100644 index 000000000..caf7ded2a --- /dev/null +++ b/Snowflake.Data/Core/Session/ChangedSessionBehavior.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Snowflake.Data.Core.Session +{ + /** + * It describes what should happen to a session with a changed state (e. g. schema/role/etc.) when it is being returned to the pool. + */ + internal enum ChangedSessionBehavior + { + OriginalPool, + ChangePool, + Destroy + } + + internal static class ChangedSessionBehaviorExtensions + { + public static List StringValues() + { + return Enum.GetValues(typeof(ChangedSessionBehavior)) + .Cast() + .Where(e => e == ChangedSessionBehavior.OriginalPool) // currently we support only OriginalPool case; TODO: SNOW-937188 + .Select(b => b.ToString()) + .ToList(); + } + + public static ChangedSessionBehavior From(string changedSession) + { + return (ChangedSessionBehavior) Enum.Parse(typeof(ChangedSessionBehavior), changedSession, true); + } + } +} diff --git a/Snowflake.Data/Core/Session/ConnectionCacheManager.cs b/Snowflake.Data/Core/Session/ConnectionCacheManager.cs index 4ab1cc3cb..393605ff8 100644 --- a/Snowflake.Data/Core/Session/ConnectionCacheManager.cs +++ b/Snowflake.Data/Core/Session/ConnectionCacheManager.cs @@ -15,7 +15,7 @@ internal sealed class ConnectionCacheManager : IConnectionManager public Task GetSessionAsync(string connectionString, SecureString password, CancellationToken cancellationToken) => _sessionPool.GetSessionAsync(connectionString, password, cancellationToken); public bool AddSession(SFSession session) => _sessionPool.AddSession(session); - public void ClearAllPools() => _sessionPool.ClearIdleSessions(); + public void ClearAllPools() => _sessionPool.ClearSessions(); public void SetMaxPoolSize(int maxPoolSize) => _sessionPool.SetMaxPoolSize(maxPoolSize); public int GetMaxPoolSize() => _sessionPool.GetMaxPoolSize(); public void SetTimeout(long connectionTimeout) => _sessionPool.SetTimeout(connectionTimeout); diff --git a/Snowflake.Data/Core/Session/ConnectionPoolConfig.cs b/Snowflake.Data/Core/Session/ConnectionPoolConfig.cs new file mode 100644 index 000000000..25f1fcd46 --- /dev/null +++ b/Snowflake.Data/Core/Session/ConnectionPoolConfig.cs @@ -0,0 +1,15 @@ +using System; + +namespace Snowflake.Data.Core.Session +{ + internal class ConnectionPoolConfig + { + public int MaxPoolSize { get; set; } = SFSessionHttpClientProperties.DefaultMaxPoolSize; + public int MinPoolSize { get; set; } = SFSessionHttpClientProperties.DefaultMinPoolSize; + public ChangedSessionBehavior ChangedSession { get; set; } = SFSessionHttpClientProperties.DefaultChangedSession; + public TimeSpan WaitingForIdleSessionTimeout { get; set; } = SFSessionHttpClientProperties.DefaultWaitingForIdleSessionTimeout; + public TimeSpan ExpirationTimeout { get; set; } = SFSessionHttpClientProperties.DefaultExpirationTimeout; + public bool PoolingEnabled { get; set; } = SFSessionHttpClientProperties.DefaultPoolingEnabled; + public TimeSpan ConnectionTimeout { get; set; } = SFSessionHttpClientProperties.DefaultConnectionTimeout; + } +} diff --git a/Snowflake.Data/Core/Session/ConnectionPoolManager.cs b/Snowflake.Data/Core/Session/ConnectionPoolManager.cs index 1e5147b8a..e51101296 100644 --- a/Snowflake.Data/Core/Session/ConnectionPoolManager.cs +++ b/Snowflake.Data/Core/Session/ConnectionPoolManager.cs @@ -17,6 +17,7 @@ internal sealed class ConnectionPoolManager : IConnectionManager { private static readonly SFLogger s_logger = SFLoggerFactory.GetLogger(); private static readonly Object s_poolsLock = new Object(); + private static readonly Exception s_operationNotAvailable = new Exception("You cannot change connection pool parameters for all the pools. Instead you can change it on a particular pool"); private readonly Dictionary _pools; internal ConnectionPoolManager() @@ -50,18 +51,14 @@ public void ClearAllPools() s_logger.Debug("ConnectionPoolManager::ClearAllPools"); foreach (var sessionPool in _pools.Values) { - sessionPool.ClearIdleSessions(); + sessionPool.ClearSessions(); } _pools.Clear(); } public void SetMaxPoolSize(int maxPoolSize) { - s_logger.Debug("ConnectionPoolManager::SetMaxPoolSize for all pools"); - foreach (var pool in _pools.Values) - { - pool.SetMaxPoolSize(maxPoolSize); - } + throw s_operationNotAvailable; } public int GetMaxPoolSize() @@ -75,11 +72,7 @@ public int GetMaxPoolSize() public void SetTimeout(long connectionTimeout) { - s_logger.Debug("ConnectionPoolManager::SetTimeout for all pools"); - foreach (var pool in _pools.Values) - { - pool.SetTimeout(connectionTimeout); - } + throw s_operationNotAvailable; } public long GetTimeout() @@ -102,13 +95,7 @@ public int GetCurrentPoolSize() public bool SetPooling(bool poolingEnabled) { - // if (!poolingEnabled) // TODO: enable when disabling pooling in connection string completed SNOW-902632 - // throw new Exception( - // "Could not disable pooling for all connections. You could disable pooling by given connection string instead."); - s_logger.Debug("ConnectionPoolManager::SetPooling for all pools"); - return _pools.Values - .Select(pool => pool.SetPooling(poolingEnabled)) - .All(setPoolingResult => setPoolingResult); + throw s_operationNotAvailable; } public bool GetPooling() diff --git a/Snowflake.Data/Core/Session/FixedZeroCounter.cs b/Snowflake.Data/Core/Session/FixedZeroCounter.cs index 9f545f2f3..f1d8467be 100644 --- a/Snowflake.Data/Core/Session/FixedZeroCounter.cs +++ b/Snowflake.Data/Core/Session/FixedZeroCounter.cs @@ -11,5 +11,9 @@ public void Increase() public void Decrease() { } + + public void Reset() + { + } } } diff --git a/Snowflake.Data/Core/Session/ICounter.cs b/Snowflake.Data/Core/Session/ICounter.cs index 61065d73f..5a38878e7 100644 --- a/Snowflake.Data/Core/Session/ICounter.cs +++ b/Snowflake.Data/Core/Session/ICounter.cs @@ -7,5 +7,7 @@ internal interface ICounter void Increase(); void Decrease(); + + void Reset(); } } diff --git a/Snowflake.Data/Core/Session/IWaitingQueue.cs b/Snowflake.Data/Core/Session/IWaitingQueue.cs index a2cd53f39..6759e9a0d 100644 --- a/Snowflake.Data/Core/Session/IWaitingQueue.cs +++ b/Snowflake.Data/Core/Session/IWaitingQueue.cs @@ -11,9 +11,5 @@ internal interface IWaitingQueue bool IsAnyoneWaiting(); bool IsWaitingEnabled(); - - long GetWaitingTimeoutMillis(); - - void SetWaitingTimeout(long timeoutMillis); } } diff --git a/Snowflake.Data/Core/Session/NonCountingSessionCreationTokenCounter.cs b/Snowflake.Data/Core/Session/NonCountingSessionCreationTokenCounter.cs index 5ccac831e..cf6671d17 100644 --- a/Snowflake.Data/Core/Session/NonCountingSessionCreationTokenCounter.cs +++ b/Snowflake.Data/Core/Session/NonCountingSessionCreationTokenCounter.cs @@ -1,10 +1,12 @@ +using System; + namespace Snowflake.Data.Core.Session { internal class NonCountingSessionCreationTokenCounter: ISessionCreationTokenCounter { - private const int IrrelevantCreateSessionTimeout = 0; // in case of old caching pool or pooling disabled we do not remove expired ones nor even store them + private static readonly TimeSpan s_irrelevantCreateSessionTimeout = SFSessionHttpClientProperties.DefaultConnectionTimeout; // in case of old caching pool or pooling disabled we do not remove expired ones nor even store them - public SessionCreationToken NewToken() => new SessionCreationToken(IrrelevantCreateSessionTimeout); + public SessionCreationToken NewToken() => new SessionCreationToken(s_irrelevantCreateSessionTimeout); public void RemoveToken(SessionCreationToken creationToken) { diff --git a/Snowflake.Data/Core/Session/NonNegativeCounter.cs b/Snowflake.Data/Core/Session/NonNegativeCounter.cs index 3d755c6cb..5f1fa5959 100644 --- a/Snowflake.Data/Core/Session/NonNegativeCounter.cs +++ b/Snowflake.Data/Core/Session/NonNegativeCounter.cs @@ -14,5 +14,7 @@ public void Decrease() { _value = Math.Max(_value - 1, 0); } + + public void Reset() => _value = 0; } } diff --git a/Snowflake.Data/Core/Session/NonWaitingQueue.cs b/Snowflake.Data/Core/Session/NonWaitingQueue.cs index 4be286171..5604ea4a8 100644 --- a/Snowflake.Data/Core/Session/NonWaitingQueue.cs +++ b/Snowflake.Data/Core/Session/NonWaitingQueue.cs @@ -22,12 +22,5 @@ public bool IsWaitingEnabled() { return false; } - - public long GetWaitingTimeoutMillis() => 0; - - public void SetWaitingTimeout(long timeoutMillis) - { - throw new System.NotImplementedException(); - } } } diff --git a/Snowflake.Data/Core/Session/SFSession.cs b/Snowflake.Data/Core/Session/SFSession.cs index f554263d3..845c47785 100755 --- a/Snowflake.Data/Core/Session/SFSession.cs +++ b/Snowflake.Data/Core/Session/SFSession.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Security; using System.Web; @@ -15,6 +14,8 @@ using System.Threading.Tasks; using System.Net.Http; using System.Text.RegularExpressions; +using Snowflake.Data.Core.Session; +using Snowflake.Data.Core.Tools; namespace Snowflake.Data.Core { @@ -50,7 +51,9 @@ public class SFSession internal string serverVersion; - internal TimeSpan connectionTimeout; + private readonly ConnectionPoolConfig _poolConfig; + + internal TimeSpan connectionTimeout => _poolConfig.ConnectionTimeout; internal bool InsecureMode; @@ -61,9 +64,6 @@ public class SFSession private string arrayBindStage = null; private int arrayBindStageThreshold = 0; internal int masterValidityInSeconds = 0; - - internal static readonly SFSessionHttpClientProperties.Extractor propertiesExtractor = new SFSessionHttpClientProperties.Extractor( - new SFSessionHttpClientProxyProperties.Extractor()); private readonly EasyLoggingStarter _easyLoggingStarter = EasyLoggingStarter.Instance; @@ -94,7 +94,7 @@ internal void ProcessLoginResponse(LoginResponse authnResponse) logger.Debug("Query context cache disabled."); } logger.Debug($"Session opened: {sessionId}"); - _startTime = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + _startTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); } else { @@ -152,14 +152,13 @@ internal SFSession( ValidateApplicationName(properties); try { - var extractedProperties = propertiesExtractor.ExtractProperties(properties); + var extractedProperties = SFSessionHttpClientProperties.ExtractAndValidate(properties); var httpClientConfig = extractedProperties.BuildHttpClientConfig(); ParameterMap = extractedProperties.ToParameterMap(); InsecureMode = extractedProperties.insecureMode; _HttpClient = HttpUtil.Instance.GetHttpClient(httpClientConfig); restRequester = new RestRequester(_HttpClient); - extractedProperties.CheckPropertiesAreValid(); - connectionTimeout = extractedProperties.TimeoutDuration(); + _poolConfig = extractedProperties.BuildConnectionPoolConfig(); properties.TryGetValue(SFSessionProperty.CLIENT_CONFIG_FILE, out var easyLoggingConfigFile); _easyLoggingStarter.Init(easyLoggingConfigFile); } @@ -563,10 +562,8 @@ internal virtual bool IsNotOpen() return _startTime == 0; } - internal virtual bool IsExpired(long timeoutInSeconds, long utcTimeInSeconds) - { - return _startTime + timeoutInSeconds <= utcTimeInSeconds; - } + internal virtual bool IsExpired(TimeSpan timeout, long utcTimeInMillis) => + TimeoutHelper.IsExpired(_startTime, utcTimeInMillis, timeout); } } diff --git a/Snowflake.Data/Core/Session/SFSessionHttpClientProperties.cs b/Snowflake.Data/Core/Session/SFSessionHttpClientProperties.cs index b7466a362..71f74e180 100644 --- a/Snowflake.Data/Core/Session/SFSessionHttpClientProperties.cs +++ b/Snowflake.Data/Core/Session/SFSessionHttpClientProperties.cs @@ -1,6 +1,9 @@ using System; using System.Collections.Generic; -using System.Threading; +using System.Linq; +using Snowflake.Data.Client; +using Snowflake.Data.Core.Session; +using Snowflake.Data.Core.Tools; using Snowflake.Data.Log; namespace Snowflake.Data.Core @@ -8,71 +11,143 @@ namespace Snowflake.Data.Core internal class SFSessionHttpClientProperties { - internal static readonly int s_maxHttpRetriesDefault = 7; - internal static readonly int s_retryTimeoutDefault = 300; - private static readonly SFLogger logger = SFLoggerFactory.GetLogger(); - + private static readonly Extractor s_propertiesExtractor = new Extractor(new SFSessionHttpClientProxyProperties.Extractor()); + public const int DefaultMaxPoolSize = 10; + public const int DefaultMinPoolSize = 2; + public const ChangedSessionBehavior DefaultChangedSession = ChangedSessionBehavior.OriginalPool; + public static readonly TimeSpan DefaultWaitingForIdleSessionTimeout = TimeSpan.FromSeconds(30); + public static readonly TimeSpan DefaultConnectionTimeout = TimeSpan.FromMinutes(5); + public static readonly TimeSpan DefaultExpirationTimeout = TimeSpan.FromHours(1); + public const bool DefaultPoolingEnabled = true; + public const int DefaultMaxHttpRetries = 7; + public static readonly TimeSpan DefaultRetryTimeout = TimeSpan.FromSeconds(300); + private static readonly SFLogger s_logger = SFLoggerFactory.GetLogger(); + private static readonly List s_changedSessionValues = ChangedSessionBehaviorExtensions.StringValues(); + internal bool validateDefaultParameters; internal bool clientSessionKeepAlive; - internal int timeoutInSec; + internal TimeSpan connectionTimeout; internal bool insecureMode; internal bool disableRetry; internal bool forceRetryOn404; - internal int retryTimeout; + internal TimeSpan retryTimeout; internal int maxHttpRetries; internal bool includeRetryReason; internal SFSessionHttpClientProxyProperties proxyProperties; - - internal void CheckPropertiesAreValid() + private int _maxPoolSize; + private int _minPoolSize; + private ChangedSessionBehavior _changedSession; + private TimeSpan _waitingForSessionIdleTimeout; + private TimeSpan _expirationTimeout; + private bool _poolingEnabled; + + public static SFSessionHttpClientProperties ExtractAndValidate(SFSessionProperties properties) { - if (timeoutInSec < s_retryTimeoutDefault) + var extractedProperties = s_propertiesExtractor.ExtractProperties(properties); + extractedProperties.CheckPropertiesAreValid(); + return extractedProperties; + } + + private void CheckPropertiesAreValid() + { + try { - logger.Warn($"Connection timeout provided is less than recommended minimum value of" + - $" {s_retryTimeoutDefault}"); + ValidateConnectionTimeout(); + ValidateRetryTimeout(); + ShortenConnectionTimeoutByRetryTimeout(); + ValidateHttpRetries(); + ValidateMinMaxPoolSize(); + ValidateWaitingForSessionIdleTimeout(); } - - if (timeoutInSec < 0) + catch (SnowflakeDbException) { - logger.Warn($"Connection timeout provided is negative. Timeout will be infinite."); + throw; } + catch (Exception exception) + { + throw new SnowflakeDbException(SFError.INVALID_CONNECTION_STRING, exception); + } + } - if (retryTimeout > 0 && retryTimeout < s_retryTimeoutDefault) + private void ValidateConnectionTimeout() + { + if (TimeoutHelper.IsZeroLength(connectionTimeout)) { - logger.Warn($"Max retry timeout provided is less than the allowed minimum value of" + - $" {s_retryTimeoutDefault}"); + s_logger.Warn("Connection timeout provided is 0. Timeout will be infinite"); + connectionTimeout = TimeoutHelper.Infinity(); + } + else if (TimeoutHelper.IsInfinite(connectionTimeout)) + { + s_logger.Warn("Connection timeout provided is negative. Timeout will be infinite."); + } + if (!TimeoutHelper.IsInfinite(connectionTimeout) && connectionTimeout < DefaultRetryTimeout) + { + s_logger.Warn($"Connection timeout provided is less than recommended minimum value of {DefaultRetryTimeout}"); + } + } - retryTimeout = s_retryTimeoutDefault; + private void ValidateRetryTimeout() + { + if (retryTimeout.TotalMilliseconds > 0 && retryTimeout < DefaultRetryTimeout) + { + s_logger.Warn($"Max retry timeout provided is less than the allowed minimum value of {DefaultRetryTimeout}"); + retryTimeout = DefaultRetryTimeout; } - else if (retryTimeout == 0) + else if (TimeoutHelper.IsZeroLength(retryTimeout)) { - logger.Warn($"Max retry timeout provided is 0. Timeout will be infinite"); + s_logger.Warn($"Max retry timeout provided is 0. Timeout will be infinite"); + retryTimeout = TimeoutHelper.Infinity(); } + else if (TimeoutHelper.IsInfinite(retryTimeout)) + { + s_logger.Warn($"Max retry timeout provided is negative. Timeout will be infinite"); + } + } - // Use the shorter timeout between CONNECTION_TIMEOUT and RETRY_TIMEOUT - if (retryTimeout < timeoutInSec) + private void ShortenConnectionTimeoutByRetryTimeout() + { + if (!TimeoutHelper.IsInfinite(retryTimeout) && retryTimeout < connectionTimeout) { - timeoutInSec = retryTimeout; + s_logger.Warn($"Connection timeout greater than retry timeout. Setting connection time same as retry timeout"); + connectionTimeout = retryTimeout; } + } - if (maxHttpRetries > 0 && maxHttpRetries < s_maxHttpRetriesDefault) + private void ValidateHttpRetries() + { + if (maxHttpRetries > 0 && maxHttpRetries < DefaultMaxHttpRetries) { - logger.Warn($"Max retry count provided is less than the allowed minimum value of" + - $" {s_maxHttpRetriesDefault}"); + s_logger.Warn($"Max retry count provided is less than the allowed minimum value of {DefaultMaxHttpRetries}"); - maxHttpRetries = s_maxHttpRetriesDefault; + maxHttpRetries = DefaultMaxHttpRetries; } else if (maxHttpRetries == 0) { - logger.Warn($"Max retry count provided is 0. Retry count will be infinite"); + s_logger.Warn($"Max retry count provided is 0. Retry count will be infinite"); + } + } + + private void ValidateMinMaxPoolSize() + { + if (_minPoolSize > _maxPoolSize) + { + throw new Exception("MinPoolSize cannot be greater than MaxPoolSize"); } } - internal TimeSpan TimeoutDuration() + private void ValidateWaitingForSessionIdleTimeout() { - return timeoutInSec > 0 ? TimeSpan.FromSeconds(timeoutInSec) : Timeout.InfiniteTimeSpan; + if (TimeoutHelper.IsInfinite(_waitingForSessionIdleTimeout)) + { + throw new Exception("Waiting for idle session timeout cannot be infinite"); + } + if (TimeoutHelper.IsZeroLength(_waitingForSessionIdleTimeout)) + { + s_logger.Warn("Waiting for idle session timeout is 0. There will be no waiting for idle session"); + } } - internal HttpClientConfig BuildHttpClientConfig() + public HttpClientConfig BuildHttpClientConfig() { return new HttpClientConfig( insecureMode, @@ -87,6 +162,18 @@ internal HttpClientConfig BuildHttpClientConfig() includeRetryReason); } + public ConnectionPoolConfig BuildConnectionPoolConfig() => + new ConnectionPoolConfig() + { + MaxPoolSize = _maxPoolSize, + MinPoolSize = _minPoolSize, + ChangedSession = _changedSession, + WaitingForIdleSessionTimeout = _waitingForSessionIdleTimeout, + ExpirationTimeout = _expirationTimeout, + PoolingEnabled = _poolingEnabled, + ConnectionTimeout = connectionTimeout + }; + internal Dictionary ToParameterMap() { var parameterMap = new Dictionary(); @@ -111,20 +198,37 @@ public Extractor(SFSessionHttpClientProxyProperties.IExtractor proxyPropertiesEx public SFSessionHttpClientProperties ExtractProperties(SFSessionProperties propertiesDictionary) { + var extractor = new SessionPropertiesWithDefaultValuesExtractor(propertiesDictionary, true); return new SFSessionHttpClientProperties() { validateDefaultParameters = Boolean.Parse(propertiesDictionary[SFSessionProperty.VALIDATE_DEFAULT_PARAMETERS]), clientSessionKeepAlive = Boolean.Parse(propertiesDictionary[SFSessionProperty.CLIENT_SESSION_KEEP_ALIVE]), - timeoutInSec = int.Parse(propertiesDictionary[SFSessionProperty.CONNECTION_TIMEOUT]), + connectionTimeout = extractor.ExtractTimeout(SFSessionProperty.CONNECTION_TIMEOUT), insecureMode = Boolean.Parse(propertiesDictionary[SFSessionProperty.INSECUREMODE]), disableRetry = Boolean.Parse(propertiesDictionary[SFSessionProperty.DISABLERETRY]), forceRetryOn404 = Boolean.Parse(propertiesDictionary[SFSessionProperty.FORCERETRYON404]), - retryTimeout = int.Parse(propertiesDictionary[SFSessionProperty.RETRY_TIMEOUT]), + retryTimeout = extractor.ExtractTimeout(SFSessionProperty.RETRY_TIMEOUT), maxHttpRetries = int.Parse(propertiesDictionary[SFSessionProperty.MAXHTTPRETRIES]), includeRetryReason = Boolean.Parse(propertiesDictionary[SFSessionProperty.INCLUDERETRYREASON]), - proxyProperties = proxyPropertiesExtractor.ExtractProperties(propertiesDictionary) + proxyProperties = proxyPropertiesExtractor.ExtractProperties(propertiesDictionary), + _maxPoolSize = extractor.ExtractPositiveIntegerWithDefaultValue(SFSessionProperty.MAXPOOLSIZE), + _minPoolSize = extractor.ExtractNonNegativeIntegerWithDefaultValue(SFSessionProperty.MINPOOLSIZE), + _changedSession = ExtractChangedSession(extractor, SFSessionProperty.CHANGEDSESSION), + _waitingForSessionIdleTimeout = extractor.ExtractTimeout(SFSessionProperty.WAITINGFORIDLESESSIONTIMEOUT), + _expirationTimeout = extractor.ExtractTimeout(SFSessionProperty.EXPIRATIONTIMEOUT), + _poolingEnabled = extractor.ExtractBooleanWithDefaultValue(SFSessionProperty.POOLINGENABLED) }; } + + private ChangedSessionBehavior ExtractChangedSession( + SessionPropertiesWithDefaultValuesExtractor extractor, + SFSessionProperty property) => + extractor.ExtractPropertyWithDefaultValue( + property, + ChangedSessionBehaviorExtensions.From, + s => !string.IsNullOrEmpty(s) && s_changedSessionValues.Contains(s, StringComparer.OrdinalIgnoreCase), + b => true + ); } } } \ No newline at end of file diff --git a/Snowflake.Data/Core/Session/SFSessionProperty.cs b/Snowflake.Data/Core/Session/SFSessionProperty.cs index 28ef2fa7e..9d85a77af 100755 --- a/Snowflake.Data/Core/Session/SFSessionProperty.cs +++ b/Snowflake.Data/Core/Session/SFSessionProperty.cs @@ -89,7 +89,19 @@ internal enum SFSessionProperty [SFSessionPropertyAttr(required = false, defaultValue = "false")] DISABLEQUERYCONTEXTCACHE, [SFSessionPropertyAttr(required = false)] - CLIENT_CONFIG_FILE + CLIENT_CONFIG_FILE, + [SFSessionPropertyAttr(required = false, defaultValue = "10")] + MAXPOOLSIZE, + [SFSessionPropertyAttr(required = false, defaultValue = "2")] + MINPOOLSIZE, + [SFSessionPropertyAttr(required = false, defaultValue = "OriginalPool")] + CHANGEDSESSION, + [SFSessionPropertyAttr(required = false, defaultValue = "30s")] + WAITINGFORIDLESESSIONTIMEOUT, + [SFSessionPropertyAttr(required = false, defaultValue = "60m")] + EXPIRATIONTIMEOUT, + [SFSessionPropertyAttr(required = false, defaultValue = "true")] + POOLINGENABLED } class SFSessionPropertyAttr : Attribute diff --git a/Snowflake.Data/Core/Session/SessionCreationToken.cs b/Snowflake.Data/Core/Session/SessionCreationToken.cs index f0f56f739..8d26a3261 100644 --- a/Snowflake.Data/Core/Session/SessionCreationToken.cs +++ b/Snowflake.Data/Core/Session/SessionCreationToken.cs @@ -1,4 +1,5 @@ using System; +using Snowflake.Data.Core.Tools; namespace Snowflake.Data.Core.Session { @@ -6,15 +7,16 @@ internal class SessionCreationToken { public Guid Id { get; } private readonly long _grantedAtAsEpochMillis; - private readonly long _timeoutMillis; + private readonly TimeSpan _timeout; - public SessionCreationToken(long timeoutMillis) + public SessionCreationToken(TimeSpan timeout) { Id = Guid.NewGuid(); _grantedAtAsEpochMillis = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - _timeoutMillis = timeoutMillis; + _timeout = timeout; } - public bool IsExpired(long nowMillis) => nowMillis > _grantedAtAsEpochMillis + _timeoutMillis; + public bool IsExpired(long nowMillis) => + TimeoutHelper.IsExpired(_grantedAtAsEpochMillis, nowMillis, _timeout); } } diff --git a/Snowflake.Data/Core/Session/SessionCreationTokenCounter.cs b/Snowflake.Data/Core/Session/SessionCreationTokenCounter.cs index eae73e3f8..1bb9b0493 100644 --- a/Snowflake.Data/Core/Session/SessionCreationTokenCounter.cs +++ b/Snowflake.Data/Core/Session/SessionCreationTokenCounter.cs @@ -6,13 +6,13 @@ namespace Snowflake.Data.Core.Session { internal class SessionCreationTokenCounter: ISessionCreationTokenCounter { - private readonly long _timeoutMillis; + private readonly TimeSpan _timeout; private readonly ReaderWriterLockSlim _tokenLock = new ReaderWriterLockSlim(); private readonly List _tokens = new List(); - public SessionCreationTokenCounter(long timeoutMillis) + public SessionCreationTokenCounter(TimeSpan timeout) { - _timeoutMillis = timeoutMillis; + _timeout = timeout; } public SessionCreationToken NewToken() @@ -20,7 +20,7 @@ public SessionCreationToken NewToken() _tokenLock.EnterWriteLock(); try { - var token = new SessionCreationToken(_timeoutMillis); + var token = new SessionCreationToken(_timeout); _tokens.Add(token); var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); _tokens.RemoveAll(t => t.IsExpired(now)); diff --git a/Snowflake.Data/Core/Session/SessionPool.cs b/Snowflake.Data/Core/Session/SessionPool.cs index ecef9e23a..ad7d1f401 100644 --- a/Snowflake.Data/Core/Session/SessionPool.cs +++ b/Snowflake.Data/Core/Session/SessionPool.cs @@ -9,6 +9,7 @@ using System.Threading; using System.Threading.Tasks; using Snowflake.Data.Client; +using Snowflake.Data.Core.Tools; using Snowflake.Data.Log; namespace Snowflake.Data.Core.Session @@ -20,50 +21,47 @@ sealed class SessionPool : IDisposable private static ISessionFactory s_sessionFactory = new SessionFactory(); private readonly List _idleSessions; - private readonly IWaitingQueue _waitingForSessionToReuseQueue; + private readonly IWaitingQueue _waitingForIdleSessionQueue; private readonly ISessionCreationTokenCounter _sessionCreationTokenCounter; private readonly ISessionCreationTokenCounter _noPoolingSessionCreationTokenCounter = new NonCountingSessionCreationTokenCounter(); - private int _maxPoolSize; - private long _timeout; - private long _sessionCreationTimeoutMillis = SessionCreationTimeout30SecAsMillis; // TODO: in further PR (SNOW-1003113) make operation fail after - private const int MaxPoolSize = 10; - private const long Timeout = 3600; - private const long SessionCreationTimeout30SecAsMillis = 30000; internal string ConnectionString { get; } internal SecureString Password { get; } - private bool _pooling = true; private readonly ICounter _busySessionsCounter; private ISessionPoolEventHandler _sessionPoolEventHandler = new SessionPoolEventHandler(); // a way to inject some additional behaviour after certain events. Can be used for example to measure time of given steps. - + private readonly ConnectionPoolConfig _poolConfig; + private bool _configOverriden = false; + private SessionPool() { // acquiring a lock not needed because one is already acquired in SnowflakeDbConnectionPool _idleSessions = new List(); - _maxPoolSize = MaxPoolSize; - _timeout = Timeout; _busySessionsCounter = new FixedZeroCounter(); - _waitingForSessionToReuseQueue = new NonWaitingQueue(); + _waitingForIdleSessionQueue = new NonWaitingQueue(); _sessionCreationTokenCounter = new NonCountingSessionCreationTokenCounter(); + _poolConfig = new ConnectionPoolConfig(); } - private SessionPool(string connectionString, SecureString password) + private SessionPool(string connectionString, SecureString password, ConnectionPoolConfig poolConfig) { // acquiring a lock not needed because one is already acquired in ConnectionPoolManager _idleSessions = new List(); - _maxPoolSize = MaxPoolSize; - _timeout = Timeout; _busySessionsCounter = new NonNegativeCounter(); ConnectionString = connectionString; Password = password; - _waitingForSessionToReuseQueue = new WaitingQueue(); - _sessionCreationTokenCounter = new SessionCreationTokenCounter(_sessionCreationTimeoutMillis); + _waitingForIdleSessionQueue = new WaitingQueue(); + _poolConfig = poolConfig; + _sessionCreationTokenCounter = new SessionCreationTokenCounter(_poolConfig.ConnectionTimeout); } internal static SessionPool CreateSessionCache() => new SessionPool(); - internal static SessionPool CreateSessionPool(string connectionString, SecureString password) => - new SessionPool(connectionString, password); - + internal static SessionPool CreateSessionPool(string connectionString, SecureString password) + { + s_logger.Debug($"Creating a pool identified by connection string: {connectionString}"); + var poolConfig = ExtractConfig(connectionString, password); + return new SessionPool(connectionString, password, poolConfig); + } + ~SessionPool() { // Use async for the finalizer due to possible deadlock @@ -86,11 +84,11 @@ private void CleanExpiredSessions() s_logger.Debug("SessionPool::CleanExpiredSessions"); lock (_sessionPoolLock) { - long timeNow = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + var timeNow = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); foreach (var item in _idleSessions.ToList()) { - if (item.IsExpired(_timeout, timeNow)) + if (item.IsExpired(_poolConfig.ExpirationTimeout, timeNow)) { _idleSessions.Remove(item); item.close(); @@ -99,32 +97,59 @@ private void CleanExpiredSessions() } } + private static ConnectionPoolConfig ExtractConfig(string connectionString, SecureString password) + { + try + { + var properties = SFSessionProperties.parseConnectionString(connectionString, password); + var extractedProperties = SFSessionHttpClientProperties.ExtractAndValidate(properties); + return extractedProperties.BuildConnectionPoolConfig(); + } + catch (SnowflakeDbException exception) + { + s_logger.Error("Could not extract pool configuration, using default one", exception); + return new ConnectionPoolConfig(); + } + } + internal SFSession GetSession(string connStr, SecureString password) { s_logger.Debug("SessionPool::GetSession"); - if (!_pooling) + if (!GetPooling()) return NewNonPoolingSession(connStr, password); var sessionOrCreateToken = GetIdleSession(connStr); if (sessionOrCreateToken.Session != null) { _sessionPoolEventHandler.OnSessionProvided(this); } + WarnAboutOverridenConfig(); return sessionOrCreateToken.Session ?? NewSession(connStr, password, sessionOrCreateToken.SessionCreationToken); } internal async Task GetSessionAsync(string connStr, SecureString password, CancellationToken cancellationToken) { s_logger.Debug("SessionPool::GetSessionAsync"); - if (!_pooling) + if (!GetPooling()) return await NewNonPoolingSessionAsync(connStr, password, cancellationToken).ConfigureAwait(false); var sessionOrCreateToken = GetIdleSession(connStr); if (sessionOrCreateToken.Session != null) { _sessionPoolEventHandler.OnSessionProvided(this); } + WarnAboutOverridenConfig(); return sessionOrCreateToken.Session ?? await NewSessionAsync(connStr, password, sessionOrCreateToken.SessionCreationToken, cancellationToken).ConfigureAwait(false); } + private void WarnAboutOverridenConfig() + { + if (IsConfigOverridden() && GetPooling() && IsMultiplePoolsVersion()) + { + s_logger.Warn("Providing a connection from a pool for which technical configuration has been overriden by the user"); + } + } + + internal bool IsConfigOverridden() => _configOverriden; + internal SFSession GetSession() => GetSession(ConnectionString, Password); internal Task GetSessionAsync(CancellationToken cancellationToken) => @@ -140,7 +165,7 @@ private SessionOrCreationToken GetIdleSession(string connStr) s_logger.Debug("SessionPool::GetIdleSession"); lock (_sessionPoolLock) { - if (_waitingForSessionToReuseQueue.IsAnyoneWaiting()) + if (_waitingForIdleSessionQueue.IsAnyoneWaiting()) { s_logger.Debug("SessionPool::GetIdleSession - someone is already waiting for a session, request is going to be queued"); } @@ -165,33 +190,36 @@ private SessionOrCreationToken GetIdleSession(string connStr) private bool IsAllowedToCreateNewSession() { - if (!_waitingForSessionToReuseQueue.IsWaitingEnabled()) + if (!IsMultiplePoolsVersion()) { s_logger.Debug($"SessionPool - creating of new sessions is not limited"); return true; } var currentSize = GetCurrentPoolSize(); - if (currentSize < _maxPoolSize) + if (currentSize < _poolConfig.MaxPoolSize) { - s_logger.Debug($"SessionPool - allowed to create a session, current pool size is {currentSize} out of {_maxPoolSize}"); + s_logger.Debug($"SessionPool - allowed to create a session, current pool size is {currentSize} out of {_poolConfig.MaxPoolSize}"); return true; } - s_logger.Debug($"SessionPool - not allowed to create a session, current pool size is {currentSize} out of {_maxPoolSize}"); + s_logger.Debug($"SessionPool - not allowed to create a session, current pool size is {currentSize} out of {_poolConfig.MaxPoolSize}"); return false; } + + private bool IsMultiplePoolsVersion() => _waitingForIdleSessionQueue.IsWaitingEnabled(); private SFSession WaitForSession(string connStr) { - var timeout = _waitingForSessionToReuseQueue.GetWaitingTimeoutMillis(); - s_logger.Info($"SessionPool::WaitForSession for {timeout} ms timeout"); + if (TimeoutHelper.IsInfinite(_poolConfig.WaitingForIdleSessionTimeout)) + throw new Exception("WaitingForIdleSessionTimeout cannot be infinite"); + s_logger.Info($"SessionPool::WaitForSession for {(long) _poolConfig.WaitingForIdleSessionTimeout.TotalMilliseconds} ms timeout"); _sessionPoolEventHandler.OnWaitingForSessionStarted(this); - var beforeWaitingTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - long nowTime = beforeWaitingTime; - while (nowTime < beforeWaitingTime + timeout) // we loop to handle the case if someone overtook us after being woken or session which we were promised has just expired + var beforeWaitingTimeMillis = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + long nowTimeMillis = beforeWaitingTimeMillis; + while (!TimeoutHelper.IsExpired(beforeWaitingTimeMillis, nowTimeMillis, _poolConfig.WaitingForIdleSessionTimeout)) // we loop to handle the case if someone overtook us after being woken or session which we were promised has just expired { - var timeoutLeft = beforeWaitingTime + timeout - nowTime; - _sessionPoolEventHandler.OnWaitingForSessionStarted(this, timeoutLeft); - var successful = _waitingForSessionToReuseQueue.Wait((int) timeoutLeft, CancellationToken.None); + var timeoutLeftMillis = TimeoutHelper.FiniteTimeoutLeftMillis(beforeWaitingTimeMillis, nowTimeMillis, _poolConfig.WaitingForIdleSessionTimeout); + _sessionPoolEventHandler.OnWaitingForSessionStarted(this, timeoutLeftMillis); + var successful = _waitingForIdleSessionQueue.Wait((int) timeoutLeftMillis, CancellationToken.None); if (successful) { s_logger.Debug($"SessionPool::WaitForSession - woken with a session granted"); @@ -210,7 +238,7 @@ private SFSession WaitForSession(string connStr) { s_logger.Debug($"SessionPool::WaitForSession - woken without a session granted"); } - nowTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + nowTimeMillis = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); } s_logger.Info($"SessionPool::WaitForSession - could not find any idle session available withing a given timeout"); throw WaitingFailedException(); @@ -226,8 +254,8 @@ private SFSession ExtractIdleSession(string connStr) { SFSession session = _idleSessions[i]; _idleSessions.RemoveAt(i); - long timeNow = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - if (session.IsExpired(_timeout, timeNow)) + var timeNow = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + if (session.IsExpired(_poolConfig.ExpirationTimeout, timeNow)) { session.close(); // TODO: cherry-pick SNOW-984600 i--; @@ -254,7 +282,7 @@ private SFSession NewSession(String connectionString, SecureString password, Ses var session = s_sessionFactory.NewSession(connectionString, password); session.Open(); s_logger.Debug("SessionPool::NewSession - opened"); - if (_pooling) + if (GetPooling()) { lock (_sessionPoolLock) { @@ -310,7 +338,7 @@ private Task NewSessionAsync(String connectionString, SecureString pa if (!previousTask.IsCanceled) { - if (_pooling) + if (GetPooling()) { lock (_sessionPoolLock) { @@ -328,11 +356,18 @@ private Task NewSessionAsync(String connectionString, SecureString pa internal bool AddSession(SFSession session) { - s_logger.Debug("SessionPool::AddSession"); - if (!_pooling) + if (!GetPooling()) return false; - long timeNow = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - if (session.IsNotOpen() || session.IsExpired(_timeout, timeNow)) + if (IsMultiplePoolsVersion()) + { + s_logger.Debug($"SessionPool::AddSession - returning session to pool identified by connection string: {ConnectionString}"); + } + else + { + s_logger.Debug("SessionPool::AddSession"); + } + long timeNow = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + if (session.IsNotOpen() || session.IsExpired(_poolConfig.ExpirationTimeout, timeNow)) { lock (_sessionPoolLock) { @@ -345,7 +380,11 @@ internal bool AddSession(SFSession session) { _busySessionsCounter.Decrease(); CleanExpiredSessions(); - if (GetCurrentPoolSize() >= _maxPoolSize) + if (session.IsExpired(_poolConfig.ExpirationTimeout, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())) // checking again because we could have spent some time waiting for a lock + { + return false; + } + if (GetCurrentPoolSize() >= _poolConfig.MaxPoolSize) { s_logger.Warn($"Pool is full - unable to add session with sid {session.sessionId}"); return false; @@ -353,11 +392,28 @@ internal bool AddSession(SFSession session) s_logger.Debug($"pool connection with sid {session.sessionId}"); _idleSessions.Add(session); - _waitingForSessionToReuseQueue.OnResourceIncrease(); + _waitingForIdleSessionQueue.OnResourceIncrease(); return true; } } + internal void ClearSessions() + { + if (IsMultiplePoolsVersion()) + { + s_logger.Debug($"SessionPool::ClearSessions for connection string: {ConnectionString}"); + } + else + { + s_logger.Debug("SessionPool::ClearSessions"); + } + lock (_sessionPoolLock) + { + _busySessionsCounter.Reset(); + ClearIdleSessions(); + } + } + internal void ClearIdleSessions() { s_logger.Debug("SessionPool::ClearIdleSessions"); @@ -388,22 +444,25 @@ internal async void ClearAllPoolsAsync() public void SetMaxPoolSize(int size) { - _maxPoolSize = size; + _poolConfig.MaxPoolSize = size; + _configOverriden = true; } public int GetMaxPoolSize() { - return _maxPoolSize; + return _poolConfig.MaxPoolSize; } - public void SetTimeout(long time) + public void SetTimeout(long seconds) { - _timeout = time; + var timeout = seconds < 0 ? TimeoutHelper.Infinity() : TimeSpan.FromSeconds(seconds); + _poolConfig.ExpirationTimeout = timeout; + _configOverriden = true; } public long GetTimeout() { - return _timeout; + return TimeoutHelper.IsInfinite(_poolConfig.ExpirationTimeout) ? -1 : (long)_poolConfig.ExpirationTimeout.TotalSeconds; } public int GetCurrentPoolSize() @@ -414,21 +473,17 @@ public int GetCurrentPoolSize() public bool SetPooling(bool isEnable) { s_logger.Info($"SessionPool::SetPooling({isEnable})"); - if (_pooling == isEnable) + if (_poolConfig.PoolingEnabled == isEnable) return false; - _pooling = isEnable; - if (!_pooling) + _poolConfig.PoolingEnabled = isEnable; + if (!_poolConfig.PoolingEnabled) { - ClearIdleSessions(); + ClearSessions(); } + _configOverriden = true; return true; } - public bool GetPooling() - { - return _pooling; - } - - public void SetWaitingForSessionToReuseTimeout(long timeoutMillis) => _waitingForSessionToReuseQueue.SetWaitingTimeout(timeoutMillis); + public bool GetPooling() => _poolConfig.PoolingEnabled; } } diff --git a/Snowflake.Data/Core/Session/SessionPropertiesWithDefaultValuesExtractor.cs b/Snowflake.Data/Core/Session/SessionPropertiesWithDefaultValuesExtractor.cs new file mode 100644 index 000000000..b9d165db7 --- /dev/null +++ b/Snowflake.Data/Core/Session/SessionPropertiesWithDefaultValuesExtractor.cs @@ -0,0 +1,143 @@ +using System; +using System.Linq; +using System.Text.RegularExpressions; +using Snowflake.Data.Core.Tools; +using Snowflake.Data.Log; + +namespace Snowflake.Data.Core.Session +{ + internal class SessionPropertiesWithDefaultValuesExtractor + { + private static readonly SFLogger s_logger = SFLoggerFactory.GetLogger(); + private static readonly Regex s_timeoutFormatRegex = new Regex(@"^(-)?[0-9]{1,10}[mM]?[sS]?$"); + + private readonly SFSessionProperties _propertiesDictionary; + private readonly bool _failOnWrongValue; + + public SessionPropertiesWithDefaultValuesExtractor(SFSessionProperties propertiesDictionary, bool failOnWrongValue) + { + _propertiesDictionary = propertiesDictionary; + _failOnWrongValue = failOnWrongValue; + } + + public bool ExtractBooleanWithDefaultValue(SFSessionProperty property) => + ExtractPropertyWithDefaultValue( + property, + Boolean.Parse, + s => true, + b => true + ); + + public int ExtractPositiveIntegerWithDefaultValue( + SFSessionProperty property) => + ExtractPropertyWithDefaultValue( + property, + int.Parse, + s => true, + i => i > 0 + ); + + public int ExtractNonNegativeIntegerWithDefaultValue( + SFSessionProperty property) => + ExtractPropertyWithDefaultValue( + property, + int.Parse, + s => true, + i => i >= 0 + ); + + public TimeSpan ExtractTimeout( + SFSessionProperty property) => + ExtractPropertyWithDefaultValue( + property, + ExtractTimeout, + ValidateTimeoutFormat, + t => true + ); + + public T ExtractPropertyWithDefaultValue( + SFSessionProperty property, + Func extractor, + Func preExtractValidation, + Func postExtractValidation) + { + var propertyAttribute = property.GetAttribute(); + var defaultValueString = propertyAttribute.defaultValue; + var defaultValue = extractor(defaultValueString); + if (!postExtractValidation(defaultValue)) + { + throw new Exception($"Invalid default value of {property}"); + } + var valueString = _propertiesDictionary[property]; + if (string.IsNullOrEmpty(valueString)) + { + s_logger.Warn($"Parameter {property} not defined. Using a default value: {defaultValue}"); + return defaultValue; + } + if (!preExtractValidation(valueString)) + { + return handleFailedValidation(defaultValue, valueString, property); + } + T value; + try + { + value = extractor(valueString); + } + catch (Exception e) + { + if (_failOnWrongValue) + { + s_logger.Error($"Invalid value of parameter {property}. Error: {e}"); + throw new Exception($"Invalid value of parameter {property}", e); + } + s_logger.Warn($"Invalid value of parameter {property}. Using a default a default value: {defaultValue}"); + return defaultValue; + } + if (!postExtractValidation(value)) + { + return handleFailedValidation(defaultValue, value, property); + } + return value; + } + + private TResult handleFailedValidation( + TResult defaultValue, + TValue value, + SFSessionProperty property) + { + if (_failOnWrongValue) + { + s_logger.Error($"Invalid value of parameter {property}: {value}"); + throw new Exception($"Invalid value of parameter {property}"); + } + s_logger.Warn($"Invalid value of parameter {property}. Using a default value: {defaultValue}"); + return defaultValue; + } + + private static bool ValidateTimeoutFormat(string value) => + !string.IsNullOrEmpty(value) && s_timeoutFormatRegex.IsMatch(value); + + private static TimeSpan ExtractTimeout(string value) + { + var numericValueString = string.Concat(value.Where(IsNumberOrMinus)); + var unitValue = value.Substring(numericValueString.Length).ToLower(); + var numericValue = int.Parse(numericValueString); + if (numericValue < 0) + return TimeoutHelper.Infinity(); + switch (unitValue) + { + case "": + case "s": + return TimeSpan.FromSeconds(numericValue); + case "ms": + return TimeSpan.FromMilliseconds(numericValue); + case "m": + return TimeSpan.FromMinutes(numericValue); + default: + throw new Exception($"unknown timeout unit value: {unitValue}"); + } + } + + private static bool IsNumberOrMinus(char value) => char.IsNumber(value) || value.Equals('-'); + } +} diff --git a/Snowflake.Data/Core/Session/WaitingQueue.cs b/Snowflake.Data/Core/Session/WaitingQueue.cs index 869fd97ca..c8cd23390 100644 --- a/Snowflake.Data/Core/Session/WaitingQueue.cs +++ b/Snowflake.Data/Core/Session/WaitingQueue.cs @@ -7,8 +7,7 @@ namespace Snowflake.Data.Core.Session internal class WaitingQueue: IWaitingQueue { private readonly ReaderWriterLockSlim _lock = new ReaderWriterLockSlim(); - private long _waitingTimeoutMillis = 30000; // 30 seconds as default - private readonly List _queue= new List(); + private readonly List _queue = new List(); public bool Wait(int millisecondsTimeout, CancellationToken cancellationToken) { @@ -81,9 +80,5 @@ public bool IsAnyoneWaiting() { } public bool IsWaitingEnabled() => true; - - public long GetWaitingTimeoutMillis() => _waitingTimeoutMillis; - - public void SetWaitingTimeout(long timeoutMillis) => _waitingTimeoutMillis = timeoutMillis; } } diff --git a/Snowflake.Data/Core/Tools/TimeoutHelper.cs b/Snowflake.Data/Core/Tools/TimeoutHelper.cs new file mode 100644 index 000000000..180cee889 --- /dev/null +++ b/Snowflake.Data/Core/Tools/TimeoutHelper.cs @@ -0,0 +1,37 @@ +using System; +using System.Threading; + +namespace Snowflake.Data.Core.Tools +{ + internal class TimeoutHelper + { + public static bool IsExpired(long startedAtMillis, long nowMillis, TimeSpan timeout) + { + if (IsInfinite(timeout)) + return false; + var timeoutInMillis = (long) timeout.TotalMilliseconds; + return startedAtMillis + timeoutInMillis <= nowMillis; + } + + public static bool IsInfinite(TimeSpan timeout) => timeout == Timeout.InfiniteTimeSpan; + + public static bool IsZeroLength(TimeSpan timeout) + { + if (IsInfinite(timeout)) + return false; + return TimeSpan.Zero == timeout; + } + + public static TimeSpan Infinity() => Timeout.InfiniteTimeSpan; + + public static long FiniteTimeoutLeftMillis(long startedAtMillis, long nowMillis, TimeSpan timeout) + { + if (IsInfinite(timeout)) + { + throw new Exception("Infinite timeout cannot be used to determine milliseconds left"); + } + var passedMillis = nowMillis - startedAtMillis; + return Math.Max((long) timeout.TotalMilliseconds - passedMillis, 0); + } + } +}