From 46094518760bbfe4689610914882fd1d094ddfe8 Mon Sep 17 00:00:00 2001 From: Piotr Bulawa Date: Tue, 12 Sep 2023 17:22:44 +0200 Subject: [PATCH] SNOW-902588: Tests parallelization (#772) ### Description Enable the parallelization of the test cases. ### Checklist - [x] Code compiles correctly - [x] Code is formatted according to [Coding Conventions](../CodingConventions.md) - [x] Created tests which fail without the change (if possible) - [x] All tests passing (`dotnet test`) - [x] Extended the README / documentation, if necessary - [x] Provide JIRA issue id (if possible) or GitHub issue id in PR name --- .github/workflows/main.yml | 2 + .../IntegrationTests/SFConnectionIT.cs | 40 +++++++++---------- .../IntegrationTests/SFConnectionPoolT.cs | 21 ++++++++-- .../IntegrationTests/SFDbCommandIT.cs | 8 ++-- .../IntegrationTests/SFMultiStatementsIT.cs | 2 +- .../IntegrationTests/SFReusableChunkTest.cs | 2 +- Snowflake.Data.Tests/SFBaseTest.cs | 18 +++++++-- .../Snowflake.Data.Tests.csproj | 1 + .../UnitTests/ChunkDownloaderFactoryTest.cs | 2 +- .../UnitTests/SFAzureClientTest.cs | 10 +++-- .../UnitTests/SFGCSClientTest.cs | 14 ++++--- .../UnitTests/SFS3ClientTest.cs | 14 ++++--- 12 files changed, 87 insertions(+), 47 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ba6c4b2fb..1c8489a8e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,6 +24,8 @@ concurrency: env: DOTNET_VERSION: 6.0 DOTNET_LEGACY_VERSION: 4.7.1 + # uncomment to run the tests sequentially + #SEQUENTIAL_ENV: SEQUENTIAL_TEST_RUN jobs: test-windows: diff --git a/Snowflake.Data.Tests/IntegrationTests/SFConnectionIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFConnectionIT.cs index 70dd515c6..ed1969900 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFConnectionIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFConnectionIT.cs @@ -311,7 +311,9 @@ public void TestConnectString() //cmd.CommandText = "drop database \"dlTest\""; cmd.CommandText = $"drop schema \"{schemaName}\""; cmd.ExecuteNonQuery(); - cmd.CommandText = "use database "+ testConfig.database; + cmd.CommandText = "use database " + testConfig.database; + cmd.ExecuteNonQuery(); + cmd.CommandText = "use schema " + testConfig.schema; cmd.ExecuteNonQuery(); } conn.Close(); @@ -379,7 +381,7 @@ public void TestConnectViaSecureString() } } - [Test] + [Test, NonParallelizable] public void TestLoginTimeout() { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -417,7 +419,7 @@ public void TestLoginTimeout() } } - [Test] + [Test, NonParallelizable] public void TestLoginWithMaxRetryReached() { using (IDbConnection conn = new MockSnowflakeDbConnection()) @@ -445,7 +447,7 @@ public void TestLoginWithMaxRetryReached() } } - [Test] + [Test, NonParallelizable] [Ignore("Disable unstable test cases for now")] public void TestDefaultLoginTimeout() { @@ -638,6 +640,7 @@ public void TestSwitchDb() Assert.AreEqual("SNOWFLAKE_SAMPLE_DATA", conn.Database); } + conn.ChangeDatabase(testConfig.database); conn.Close(); } @@ -1666,7 +1669,7 @@ class SFConnectionITAsync : SFBaseTestAsync private static SFLogger logger = SFLoggerFactory.GetLogger(); - [Test] + [Test, NonParallelizable] public void TestCancelLoginBeforeTimeout() { using (var conn = new MockSnowflakeDbConnection()) @@ -1714,7 +1717,7 @@ public void TestCancelLoginBeforeTimeout() } } - [Test] + [Test, NonParallelizable] public void TestAsyncLoginTimeout() { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -1728,11 +1731,10 @@ public void TestAsyncLoginTimeout() Assert.AreEqual(conn.State, ConnectionState.Closed); - CancellationTokenSource connectionCancelToken = new CancellationTokenSource(); Stopwatch stopwatch = Stopwatch.StartNew(); try { - Task connectTask = conn.OpenAsync(connectionCancelToken.Token); + Task connectTask = conn.OpenAsync(CancellationToken.None); connectTask.Wait(); } catch (AggregateException e) @@ -1755,7 +1757,7 @@ public void TestAsyncLoginTimeout() } } - [Test] + [Test, NonParallelizable] public void TestAsyncDefaultLoginTimeout() { using (var conn = new MockSnowflakeDbConnection()) @@ -1765,11 +1767,10 @@ public void TestAsyncDefaultLoginTimeout() Assert.AreEqual(conn.State, ConnectionState.Closed); - CancellationTokenSource connectionCancelToken = new CancellationTokenSource(); Stopwatch stopwatch = Stopwatch.StartNew(); try { - Task connectTask = conn.OpenAsync(connectionCancelToken.Token); + Task connectTask = conn.OpenAsync(CancellationToken.None); connectTask.Wait(); } catch (AggregateException e) @@ -1802,11 +1803,10 @@ public void TestAsyncConnectionFailFast() conn.ConnectionString = invalidConnectionString; Assert.AreEqual(conn.State, ConnectionState.Closed); - CancellationTokenSource connectionCancelToken = new CancellationTokenSource(); Task connectTask = null; try { - connectTask = conn.OpenAsync(connectionCancelToken.Token); + connectTask = conn.OpenAsync(CancellationToken.None); connectTask.Wait(); Assert.Fail(); } @@ -1835,22 +1835,22 @@ public void TestCloseAsyncWithCancellation() Task task = null; // Close the connection. It's not opened yet, but it should not have any issue - task = conn.CloseAsync(new CancellationTokenSource().Token); + task = conn.CloseAsync(CancellationToken.None); task.Wait(); Assert.AreEqual(conn.State, ConnectionState.Closed); // Open the connection - task = conn.OpenAsync(new CancellationTokenSource().Token); + task = conn.OpenAsync(CancellationToken.None); task.Wait(); Assert.AreEqual(conn.State, ConnectionState.Open); // Close the opened connection - task = conn.CloseAsync(new CancellationTokenSource().Token); + task = conn.CloseAsync(CancellationToken.None); task.Wait(); Assert.AreEqual(conn.State, ConnectionState.Closed); // Close the connection again. - task = conn.CloseAsync(new CancellationTokenSource().Token); + task = conn.CloseAsync(CancellationToken.None); task.Wait(); Assert.AreEqual(conn.State, ConnectionState.Closed); } @@ -1893,7 +1893,7 @@ public void TestCloseAsync() } #endif - [Test] + [Test, NonParallelizable] public void TestCloseAsyncFailure() { using (var conn = new MockSnowflakeDbConnection(new MockCloseSessionException())) @@ -1904,12 +1904,12 @@ public void TestCloseAsyncFailure() Task task = null; // Open the connection - task = conn.OpenAsync(new CancellationTokenSource().Token); + task = conn.OpenAsync(CancellationToken.None); task.Wait(); Assert.AreEqual(conn.State, ConnectionState.Open); // Close the opened connection - task = conn.CloseAsync(new CancellationTokenSource().Token); + task = conn.CloseAsync(CancellationToken.None); try { task.Wait(); diff --git a/Snowflake.Data.Tests/IntegrationTests/SFConnectionPoolT.cs b/Snowflake.Data.Tests/IntegrationTests/SFConnectionPoolT.cs index aad1fb7f5..45148f066 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFConnectionPoolT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFConnectionPoolT.cs @@ -56,6 +56,12 @@ public void AfterTest() previousPoolConfig.Reset(); } + [OneTimeTearDown] + public static void AfterAllTests() + { + SnowflakeDbConnectionPool.ClearAllPools(); + } + [Test] [Ignore("dummy test case for showing test progress.")] public void ConnectionPoolTDone() @@ -339,9 +345,12 @@ public void TestConnectionPoolMultiThreading() t1.Start(); t2.Start(); + + t1.Join(); + t2.Join(); } - static void ThreadProcess1(string connstr) + void ThreadProcess1(string connstr) { var conn1 = new SnowflakeDbConnection(); conn1.ConnectionString = connstr; @@ -352,7 +361,7 @@ static void ThreadProcess1(string connstr) Assert.AreEqual(ConnectionState.Closed, conn1.State); } - static void ThreadProcess2(string connstr) + void ThreadProcess2(string connstr) { var conn1 = new SnowflakeDbConnection(); conn1.ConnectionString = connstr; @@ -429,7 +438,7 @@ public void TestConnectionPoolTurnOff() } } - [TestFixture] + [TestFixture, NonParallelizable] class SFConnectionPoolITAsync : SFBaseTestAsync { private static SFLogger logger = SFLoggerFactory.GetLogger(); @@ -448,6 +457,12 @@ public void AfterTest() { previousPoolConfig.Reset(); } + + [OneTimeTearDown] + public static void AfterAllTests() + { + SnowflakeDbConnectionPool.ClearAllPools(); + } [Test] public void TestConnectionPoolWithAsync() diff --git a/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs index a17d03a1e..4994ead35 100755 --- a/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs @@ -31,14 +31,12 @@ public void DbCommandITAsyncDone() [Test] public void TestExecAsyncAPI() { + SnowflakeDbConnectionPool.ClearAllPools(); using (DbConnection conn = new SnowflakeDbConnection()) { - SnowflakeDbConnectionPool.ClearAllPools(); conn.ConnectionString = ConnectionString; Task connectTask = conn.OpenAsync(CancellationToken.None); - Assert.AreEqual(ConnectionState.Connecting, conn.State); - connectTask.Wait(); Assert.AreEqual(ConnectionState.Open, conn.State); @@ -254,7 +252,7 @@ public void TestSimpleLargeResultSet() /* * Disabled to make sure that configuration changes does not cause problems with appveyor */ - [Test] + [Test, NonParallelizable] public void TestUseV1ResultParser() { var chunkParserVersion = SFConfiguration.Instance().ChunkParserVersion; @@ -283,7 +281,7 @@ public void TestUseV1ResultParser() SFConfiguration.Instance().ChunkDownloaderVersion = chunkDownloaderVersion; } - [Test] + [Test, NonParallelizable] public void TestUseV2ChunkDownloader() { var chunkParserVersion = SFConfiguration.Instance().ChunkParserVersion; diff --git a/Snowflake.Data.Tests/IntegrationTests/SFMultiStatementsIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFMultiStatementsIT.cs index 8a5b4ee12..b05389fba 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFMultiStatementsIT.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFMultiStatementsIT.cs @@ -310,7 +310,7 @@ public void testWithAllQueryTypes() $"remove @%{TableName};" + "create or replace temporary procedure P1() returns varchar language javascript as $$ return ''; $$;" + "call p1();" + - "use role public"; + $"use role {testConfig.role}"; // Set statement count var stmtCountParam = cmd.CreateParameter(); diff --git a/Snowflake.Data.Tests/IntegrationTests/SFReusableChunkTest.cs b/Snowflake.Data.Tests/IntegrationTests/SFReusableChunkTest.cs index bc216c9a4..dd7ad5693 100644 --- a/Snowflake.Data.Tests/IntegrationTests/SFReusableChunkTest.cs +++ b/Snowflake.Data.Tests/IntegrationTests/SFReusableChunkTest.cs @@ -9,7 +9,7 @@ namespace Snowflake.Data.Tests.IntegrationTests using Client; using System.Threading.Tasks; - [TestFixture] + [TestFixture, NonParallelizable] class SFReusableChunkTest : SFBaseTest { [Test] diff --git a/Snowflake.Data.Tests/SFBaseTest.cs b/Snowflake.Data.Tests/SFBaseTest.cs index 81c40c7c7..3f8c9207d 100755 --- a/Snowflake.Data.Tests/SFBaseTest.cs +++ b/Snowflake.Data.Tests/SFBaseTest.cs @@ -10,9 +10,12 @@ using System.IO; using System.Linq; using System.Runtime.InteropServices; +using NUnit.Framework; using Snowflake.Data.Client; using Snowflake.Data.Tests.Util; +[assembly:LevelOfParallelism(10)] + namespace Snowflake.Data.Tests { using NUnit.Framework; @@ -48,14 +51,18 @@ public static void TearDownContext() [TestFixture] [FixtureLifeCycle(LifeCycle.InstancePerTestCase)] [SetCulture("en-US")] + #if !SEQUENTIAL_TEST_RUN + [Parallelizable(ParallelScope.All)] + #endif public class SFBaseTestAsync { private const string ConnectionStringWithoutAuthFmt = "scheme={0};host={1};port={2};" + "account={3};role={4};db={5};schema={6};warehouse={7}"; private const string ConnectionStringSnowflakeAuthFmt = ";user={0};password={1};"; protected readonly string TestName = TestContext.CurrentContext.Test.MethodName; - - protected string TableName => TestName + TestContext.CurrentContext.WorkerId?.Replace("#", "_"); + protected string TestNameWithWorker => TestName + TestContext.CurrentContext.WorkerId?.Replace("#", "_"); + protected string TableName => TestNameWithWorker; + private Stopwatch _stopwatch; @@ -151,9 +158,14 @@ public class TestEnvironment private static Dictionary s_testPerformance; + private static readonly object s_testPerformanceLock = new object(); + public static void RecordTestPerformance(string name, TimeSpan time) { - s_testPerformance.Add(name, time); + lock (s_testPerformanceLock) + { + s_testPerformance.Add(name, time); + } } [OneTimeSetUp] diff --git a/Snowflake.Data.Tests/Snowflake.Data.Tests.csproj b/Snowflake.Data.Tests/Snowflake.Data.Tests.csproj index 0024e944f..811fb2b96 100644 --- a/Snowflake.Data.Tests/Snowflake.Data.Tests.csproj +++ b/Snowflake.Data.Tests/Snowflake.Data.Tests.csproj @@ -10,6 +10,7 @@ Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. true 7.3 + $(SEQUENTIAL_ENV) diff --git a/Snowflake.Data.Tests/UnitTests/ChunkDownloaderFactoryTest.cs b/Snowflake.Data.Tests/UnitTests/ChunkDownloaderFactoryTest.cs index 4bff69a0d..bb016376b 100644 --- a/Snowflake.Data.Tests/UnitTests/ChunkDownloaderFactoryTest.cs +++ b/Snowflake.Data.Tests/UnitTests/ChunkDownloaderFactoryTest.cs @@ -55,7 +55,7 @@ private SFResultSet mockSFResultSet(QueryExecResponseData responseData, Cancella return new SFResultSet(responseData, new SFStatement(session), token); } - [Test] + [Test, NonParallelizable] public void TestGetDownloader([Values(false, true)] bool useV2ChunkDownloader, [Values(1, 2, 3, 4)] int chunkDownloaderVersion) { // Set configuration settings diff --git a/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs index e2497479b..71db1466a 100644 --- a/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs @@ -2,6 +2,8 @@ * Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. */ +using System; + namespace Snowflake.Data.Tests.UnitTests { using NUnit.Framework; @@ -45,7 +47,7 @@ class SFAzureClientTest : SFBaseTest const int Parallel = 0; // File name for download tests - const string DownloadFileName = "mockFileName.txt"; + [ThreadStatic] private static string t_downloadFileName; // Token for async tests CancellationToken _cancellationToken; @@ -60,6 +62,8 @@ class SFAzureClientTest : SFBaseTest [SetUp] public void BeforeTest() { + t_downloadFileName = TestNameWithWorker + "_mockFileName.txt"; + _fileMetadata = new SFFileMetadata() { stageInfo = new PutGetStageInfo() @@ -329,7 +333,7 @@ public void TestDownloadFile(HttpStatusCode httpStatusCode, ResultStatus expecte _fileMetadata.stageInfo.location = httpStatusCode.ToString(); // Act - _client.DownloadFile(_fileMetadata, DownloadFileName, Parallel); + _client.DownloadFile(_fileMetadata, t_downloadFileName, Parallel); // Assert Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus); @@ -376,7 +380,7 @@ public async Task TestDownloadFileAsync(HttpStatusCode httpStatusCode, ResultSta _fileMetadata.stageInfo.location = httpStatusCode.ToString(); // Act - await _client.DownloadFileAsync(_fileMetadata, DownloadFileName, Parallel, _cancellationToken).ConfigureAwait(false); + await _client.DownloadFileAsync(_fileMetadata, t_downloadFileName, Parallel, _cancellationToken).ConfigureAwait(false); // Assert Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus); diff --git a/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs index 7167de22c..f2c812fad 100644 --- a/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs @@ -2,6 +2,8 @@ * Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. */ +using System; + namespace Snowflake.Data.Tests.UnitTests { using NUnit.Framework; @@ -38,7 +40,7 @@ class SFGCSClientTest : SFBaseTest const int Parallel = 0; // File name for download tests - const string DownloadFileName = "mockFileName.txt"; + [ThreadStatic] private static string t_downloadFileName; // Token for async tests CancellationToken _cancellationToken; @@ -53,6 +55,8 @@ class SFGCSClientTest : SFBaseTest [SetUp] public void BeforeTest() { + t_downloadFileName = TestNameWithWorker + "_mockFileName.txt"; + _fileMetadata = new SFFileMetadata() { stageInfo = new PutGetStageInfo() @@ -310,7 +314,7 @@ public void TestDownloadFile(HttpStatusCode httpStatusCode, ResultStatus expecte _client.SetCustomWebRequest(mockWebRequest.Object); // Act - _client.DownloadFile(_fileMetadata, DownloadFileName, Parallel); + _client.DownloadFile(_fileMetadata, t_downloadFileName, Parallel); // Assert AssertForDownloadFileTests(expectedResultStatus); @@ -334,7 +338,7 @@ public async Task TestDownloadFileAsync(HttpStatusCode httpStatusCode, ResultSta _client.SetCustomWebRequest(mockWebRequest.Object); // Act - await _client.DownloadFileAsync(_fileMetadata, DownloadFileName, Parallel, _cancellationToken).ConfigureAwait(false); + await _client.DownloadFileAsync(_fileMetadata, t_downloadFileName, Parallel, _cancellationToken).ConfigureAwait(false); // Assert AssertForDownloadFileTests(expectedResultStatus); @@ -344,9 +348,9 @@ private void AssertForDownloadFileTests(ResultStatus expectedResultStatus) { if (expectedResultStatus == ResultStatus.DOWNLOADED) { - string text = File.ReadAllText(DownloadFileName); + string text = File.ReadAllText(t_downloadFileName); Assert.AreEqual(MockGCSClient.GcsFileContent, text); - File.Delete(DownloadFileName); + File.Delete(t_downloadFileName); } Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus); diff --git a/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs index 10f8b0ff1..0a3671299 100644 --- a/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs @@ -2,6 +2,8 @@ * Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. */ +using System; + namespace Snowflake.Data.Tests.UnitTests { using NUnit.Framework; @@ -61,7 +63,7 @@ class SFS3ClientTest : SFBaseTest const int Parallel = 0; // File name for download tests - const string DownloadFileName = "mockFileName.txt"; + [ThreadStatic] private static string t_downloadFileName; // Token for async tests CancellationToken _cancellationToken; @@ -77,6 +79,8 @@ class SFS3ClientTest : SFBaseTest [SetUp] public void BeforeTest() { + t_downloadFileName = TestNameWithWorker + "_mockFileName.txt"; + _fileMetadata = new SFFileMetadata() { stageInfo = new PutGetStageInfo() @@ -280,7 +284,7 @@ public void TestDownloadFile(string requestKey, ResultStatus expectedResultStatu _fileMetadata.stageInfo.location = requestKey; // Act - _client.DownloadFile(_fileMetadata, DownloadFileName, Parallel); + _client.DownloadFile(_fileMetadata, t_downloadFileName, Parallel); // Assert AssertForDownloadFileTests(expectedResultStatus); @@ -304,7 +308,7 @@ public async Task TestDownloadFileAsync(string requestKey, ResultStatus expected _fileMetadata.stageInfo.location = requestKey; // Act - await _client.DownloadFileAsync(_fileMetadata, DownloadFileName, Parallel, _cancellationToken).ConfigureAwait(false); + await _client.DownloadFileAsync(_fileMetadata, t_downloadFileName, Parallel, _cancellationToken).ConfigureAwait(false); // Assert AssertForDownloadFileTests(expectedResultStatus); @@ -314,9 +318,9 @@ private void AssertForDownloadFileTests(ResultStatus expectedResultStatus) { if (expectedResultStatus == ResultStatus.DOWNLOADED) { - string text = File.ReadAllText(DownloadFileName); + string text = File.ReadAllText(t_downloadFileName); Assert.AreEqual(MockS3Client.S3FileContent, text); - File.Delete(DownloadFileName); + File.Delete(t_downloadFileName); } Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus);