SNOW-902588: Tests parallelization (#772)

### Description Enable the parallelization of the test cases. ### Checklist - [x] Code compiles correctly - [x] Code is formatted according to [Coding Conventions](../CodingConventions.md) - [x] Created tests which fail without the change (if possible) - [x] All tests passing (`dotnet test`) - [x] Extended the README / documentation, if necessary - [x] Provide JIRA issue id (if possible) or GitHub issue id in PR name
snowflakedb · Sep 12, 2023 · 4609451 · 4609451
1 parent a175090
commit 4609451
Show file tree

Hide file tree

Showing 12 changed files with 87 additions and 47 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -24,6 +24,8 @@ concurrency:
 env:
   DOTNET_VERSION: 6.0
   DOTNET_LEGACY_VERSION: 4.7.1
+  # uncomment to run the tests sequentially
+  #SEQUENTIAL_ENV: SEQUENTIAL_TEST_RUN
 
 jobs:
   test-windows:

diff --git a/Snowflake.Data.Tests/IntegrationTests/SFConnectionIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFConnectionIT.cs
@@ -311,7 +311,9 @@ public void TestConnectString()
                 //cmd.CommandText = "drop database \"dlTest\"";
                 cmd.CommandText = $"drop schema \"{schemaName}\"";
                 cmd.ExecuteNonQuery();
-                cmd.CommandText = "use database "+ testConfig.database;
+                cmd.CommandText = "use database " + testConfig.database;
+                cmd.ExecuteNonQuery();
+                cmd.CommandText = "use schema " + testConfig.schema;
                 cmd.ExecuteNonQuery();
             }
             conn.Close();
@@ -379,7 +381,7 @@ public void TestConnectViaSecureString()
             }
         }
 
-        [Test]
+        [Test, NonParallelizable]
         public void TestLoginTimeout()
         {
             if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
@@ -417,7 +419,7 @@ public void TestLoginTimeout()
             }
         }
 
-        [Test]
+        [Test, NonParallelizable]
         public void TestLoginWithMaxRetryReached()
         {
             using (IDbConnection conn = new MockSnowflakeDbConnection())
@@ -445,7 +447,7 @@ public void TestLoginWithMaxRetryReached()
             }
         }
 
-        [Test]
+        [Test, NonParallelizable]
         [Ignore("Disable unstable test cases for now")]
         public void TestDefaultLoginTimeout()
         {
@@ -638,6 +640,7 @@ public void TestSwitchDb()
                     Assert.AreEqual("SNOWFLAKE_SAMPLE_DATA", conn.Database);
                 }
 
+                conn.ChangeDatabase(testConfig.database);
                 conn.Close();
             }
 
@@ -1666,7 +1669,7 @@ class SFConnectionITAsync : SFBaseTestAsync
         private static SFLogger logger = SFLoggerFactory.GetLogger<SFConnectionITAsync>();
 
 
-        [Test]
+        [Test, NonParallelizable]
         public void TestCancelLoginBeforeTimeout()
         {
             using (var conn = new MockSnowflakeDbConnection())
@@ -1714,7 +1717,7 @@ public void TestCancelLoginBeforeTimeout()
             }
         }
 
-        [Test]
+        [Test, NonParallelizable]
         public void TestAsyncLoginTimeout()
         {
             if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
@@ -1728,11 +1731,10 @@ public void TestAsyncLoginTimeout()
 
                     Assert.AreEqual(conn.State, ConnectionState.Closed);
 
-                    CancellationTokenSource connectionCancelToken = new CancellationTokenSource();
                     Stopwatch stopwatch = Stopwatch.StartNew();
                     try
                     {
-                        Task connectTask = conn.OpenAsync(connectionCancelToken.Token);
+                        Task connectTask = conn.OpenAsync(CancellationToken.None);
                         connectTask.Wait();
                     }
                     catch (AggregateException e)
@@ -1755,7 +1757,7 @@ public void TestAsyncLoginTimeout()
             }
         }
 
-        [Test]
+        [Test, NonParallelizable]
         public void TestAsyncDefaultLoginTimeout()
         {
             using (var conn = new MockSnowflakeDbConnection())
@@ -1765,11 +1767,10 @@ public void TestAsyncDefaultLoginTimeout()
 
                 Assert.AreEqual(conn.State, ConnectionState.Closed);
 
-                CancellationTokenSource connectionCancelToken = new CancellationTokenSource();
                 Stopwatch stopwatch = Stopwatch.StartNew();
                 try
                 {
-                    Task connectTask = conn.OpenAsync(connectionCancelToken.Token);
+                    Task connectTask = conn.OpenAsync(CancellationToken.None);
                     connectTask.Wait();
                 }
                 catch (AggregateException e)
@@ -1802,11 +1803,10 @@ public void TestAsyncConnectionFailFast()
                 conn.ConnectionString = invalidConnectionString;
 
                 Assert.AreEqual(conn.State, ConnectionState.Closed);
-                CancellationTokenSource connectionCancelToken = new CancellationTokenSource();
                 Task connectTask = null;
                 try
                 {
-                    connectTask = conn.OpenAsync(connectionCancelToken.Token);
+                    connectTask = conn.OpenAsync(CancellationToken.None);
                     connectTask.Wait();
                     Assert.Fail();
                 }
@@ -1835,22 +1835,22 @@ public void TestCloseAsyncWithCancellation()
                 Task task = null;
 
                 // Close the connection. It's not opened yet, but it should not have any issue
-                task = conn.CloseAsync(new CancellationTokenSource().Token);
+                task = conn.CloseAsync(CancellationToken.None);
                 task.Wait();
                 Assert.AreEqual(conn.State, ConnectionState.Closed);
 
                 // Open the connection
-                task = conn.OpenAsync(new CancellationTokenSource().Token);
+                task = conn.OpenAsync(CancellationToken.None);
                 task.Wait();
                 Assert.AreEqual(conn.State, ConnectionState.Open);
 
                 // Close the opened connection
-                task = conn.CloseAsync(new CancellationTokenSource().Token);
+                task = conn.CloseAsync(CancellationToken.None);
                 task.Wait();
                 Assert.AreEqual(conn.State, ConnectionState.Closed);
 
                 // Close the connection again.
-                task = conn.CloseAsync(new CancellationTokenSource().Token);
+                task = conn.CloseAsync(CancellationToken.None);
                 task.Wait();
                 Assert.AreEqual(conn.State, ConnectionState.Closed);
             }
@@ -1893,7 +1893,7 @@ public void TestCloseAsync()
 		}
 #endif
 
-		[Test]
+        [Test, NonParallelizable]
         public void TestCloseAsyncFailure()
         {
             using (var conn = new MockSnowflakeDbConnection(new MockCloseSessionException()))
@@ -1904,12 +1904,12 @@ public void TestCloseAsyncFailure()
                 Task task = null;
 
                 // Open the connection
-                task = conn.OpenAsync(new CancellationTokenSource().Token);
+                task = conn.OpenAsync(CancellationToken.None);
                 task.Wait();
                 Assert.AreEqual(conn.State, ConnectionState.Open);
 
                 // Close the opened connection
-                task =  conn.CloseAsync(new CancellationTokenSource().Token);
+                task =  conn.CloseAsync(CancellationToken.None);
                 try
                 {
                     task.Wait();

diff --git a/Snowflake.Data.Tests/IntegrationTests/SFConnectionPoolT.cs b/Snowflake.Data.Tests/IntegrationTests/SFConnectionPoolT.cs
@@ -56,6 +56,12 @@ public void AfterTest()
             previousPoolConfig.Reset();
         }
 
+        [OneTimeTearDown]
+        public static void AfterAllTests()
+        {
+            SnowflakeDbConnectionPool.ClearAllPools();
+        }
+
         [Test]
         [Ignore("dummy test case for showing test progress.")]
         public void ConnectionPoolTDone()
@@ -339,9 +345,12 @@ public void TestConnectionPoolMultiThreading()
 
             t1.Start();
             t2.Start();
+
+            t1.Join();
+            t2.Join();
         }
 
-        static void ThreadProcess1(string connstr)
+        void ThreadProcess1(string connstr)
         {
             var conn1 = new SnowflakeDbConnection();
             conn1.ConnectionString = connstr;
@@ -352,7 +361,7 @@ static void ThreadProcess1(string connstr)
             Assert.AreEqual(ConnectionState.Closed, conn1.State);
         }
 
-        static void ThreadProcess2(string connstr)
+        void ThreadProcess2(string connstr)
         {
             var conn1 = new SnowflakeDbConnection();
             conn1.ConnectionString = connstr;
@@ -429,7 +438,7 @@ public void TestConnectionPoolTurnOff()
         }
     }
 
-    [TestFixture]
+    [TestFixture, NonParallelizable]
     class SFConnectionPoolITAsync : SFBaseTestAsync
     {
         private static SFLogger logger = SFLoggerFactory.GetLogger<SFConnectionPoolITAsync>();
@@ -448,6 +457,12 @@ public void AfterTest()
         {
             previousPoolConfig.Reset();
         }
+
+        [OneTimeTearDown]
+        public static void AfterAllTests()
+        {
+            SnowflakeDbConnectionPool.ClearAllPools();
+        }
 
         [Test]
         public void TestConnectionPoolWithAsync()

diff --git a/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFDbCommandIT.cs
@@ -31,14 +31,12 @@ public void DbCommandITAsyncDone()
         [Test]
         public void TestExecAsyncAPI()
         {
+            SnowflakeDbConnectionPool.ClearAllPools();
             using (DbConnection conn = new SnowflakeDbConnection())
             {
-                SnowflakeDbConnectionPool.ClearAllPools();
                 conn.ConnectionString = ConnectionString;
 
                 Task connectTask = conn.OpenAsync(CancellationToken.None);
-                Assert.AreEqual(ConnectionState.Connecting, conn.State);
-
                 connectTask.Wait();
                 Assert.AreEqual(ConnectionState.Open, conn.State);
 
@@ -254,7 +252,7 @@ public void TestSimpleLargeResultSet()
         /*
          * Disabled to make sure that configuration changes does not cause problems with appveyor
          */
-        [Test]
+        [Test, NonParallelizable]
         public void TestUseV1ResultParser()
         {
             var chunkParserVersion = SFConfiguration.Instance().ChunkParserVersion;
@@ -283,7 +281,7 @@ public void TestUseV1ResultParser()
             SFConfiguration.Instance().ChunkDownloaderVersion = chunkDownloaderVersion;
         }
 
-        [Test]
+        [Test, NonParallelizable]
         public void TestUseV2ChunkDownloader()
         {
             var chunkParserVersion = SFConfiguration.Instance().ChunkParserVersion;

diff --git a/Snowflake.Data.Tests/IntegrationTests/SFMultiStatementsIT.cs b/Snowflake.Data.Tests/IntegrationTests/SFMultiStatementsIT.cs
@@ -310,7 +310,7 @@ public void testWithAllQueryTypes()
                                       $"remove @%{TableName};" +
                                       "create or replace temporary procedure P1() returns varchar language javascript as $$ return ''; $$;" +
                                       "call p1();" +
-                                      "use role public";
+                                      $"use role {testConfig.role}";
 
                     // Set statement count
                     var stmtCountParam = cmd.CreateParameter();

diff --git a/Snowflake.Data.Tests/IntegrationTests/SFReusableChunkTest.cs b/Snowflake.Data.Tests/IntegrationTests/SFReusableChunkTest.cs
@@ -9,7 +9,7 @@ namespace Snowflake.Data.Tests.IntegrationTests
     using Client;
     using System.Threading.Tasks;
 
-    [TestFixture]
+    [TestFixture, NonParallelizable]
     class SFReusableChunkTest : SFBaseTest
     {
         [Test]

diff --git a/Snowflake.Data.Tests/SFBaseTest.cs b/Snowflake.Data.Tests/SFBaseTest.cs
@@ -10,9 +10,12 @@
 using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
+using NUnit.Framework;
 using Snowflake.Data.Client;
 using Snowflake.Data.Tests.Util;
 
+[assembly:LevelOfParallelism(10)]
+
 namespace Snowflake.Data.Tests
 {
     using NUnit.Framework;
@@ -48,14 +51,18 @@ public static void TearDownContext()
     [TestFixture]
     [FixtureLifeCycle(LifeCycle.InstancePerTestCase)]
     [SetCulture("en-US")]
+    #if !SEQUENTIAL_TEST_RUN
+    [Parallelizable(ParallelScope.All)]
+    #endif
     public class SFBaseTestAsync
     {
         private const string ConnectionStringWithoutAuthFmt = "scheme={0};host={1};port={2};" +
                                                               "account={3};role={4};db={5};schema={6};warehouse={7}";
         private const string ConnectionStringSnowflakeAuthFmt = ";user={0};password={1};";
         protected readonly string TestName = TestContext.CurrentContext.Test.MethodName;
-
-        protected string TableName => TestName + TestContext.CurrentContext.WorkerId?.Replace("#", "_");
+        protected string TestNameWithWorker => TestName + TestContext.CurrentContext.WorkerId?.Replace("#", "_");
+        protected string TableName => TestNameWithWorker;
+
 
         private Stopwatch _stopwatch;
 
@@ -151,9 +158,14 @@ public class TestEnvironment
 
         private static Dictionary<string, TimeSpan> s_testPerformance;
 
+        private static readonly object s_testPerformanceLock = new object();
+
         public static void RecordTestPerformance(string name, TimeSpan time)
         {
-            s_testPerformance.Add(name, time);
+            lock (s_testPerformanceLock)
+            {
+                s_testPerformance.Add(name, time);
+            }
         }
 
         [OneTimeSetUp]

diff --git a/Snowflake.Data.Tests/Snowflake.Data.Tests.csproj b/Snowflake.Data.Tests/Snowflake.Data.Tests.csproj
@@ -10,6 +10,7 @@
     <Copyright>Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.</Copyright>
     <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
     <LangVersion>7.3</LangVersion>
+    <DefineConstants>$(SEQUENTIAL_ENV)</DefineConstants>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/Snowflake.Data.Tests/UnitTests/ChunkDownloaderFactoryTest.cs b/Snowflake.Data.Tests/UnitTests/ChunkDownloaderFactoryTest.cs
@@ -55,7 +55,7 @@ private SFResultSet mockSFResultSet(QueryExecResponseData responseData, Cancella
             return new SFResultSet(responseData, new SFStatement(session), token);
         }
 
-        [Test]
+        [Test, NonParallelizable]
         public void TestGetDownloader([Values(false, true)] bool useV2ChunkDownloader, [Values(1, 2, 3, 4)] int chunkDownloaderVersion)
         {
             // Set configuration settings

diff --git a/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs
@@ -2,6 +2,8 @@
  * Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.
  */
 
+using System;
+
 namespace Snowflake.Data.Tests.UnitTests
 {
     using NUnit.Framework;
@@ -45,7 +47,7 @@ class SFAzureClientTest : SFBaseTest
         const int Parallel = 0;
 
         // File name for download tests
-        const string DownloadFileName = "mockFileName.txt";
+        [ThreadStatic] private static string t_downloadFileName;
 
         // Token for async tests
         CancellationToken _cancellationToken;
@@ -60,6 +62,8 @@ class SFAzureClientTest : SFBaseTest
         [SetUp]
         public void BeforeTest()
         {
+            t_downloadFileName = TestNameWithWorker + "_mockFileName.txt";
+
             _fileMetadata = new SFFileMetadata()
             {
                 stageInfo = new PutGetStageInfo()
@@ -329,7 +333,7 @@ public void TestDownloadFile(HttpStatusCode httpStatusCode, ResultStatus expecte
             _fileMetadata.stageInfo.location = httpStatusCode.ToString();
 
             // Act
-            _client.DownloadFile(_fileMetadata, DownloadFileName, Parallel);
+            _client.DownloadFile(_fileMetadata, t_downloadFileName, Parallel);
 
             // Assert
             Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus);
@@ -376,7 +380,7 @@ public async Task TestDownloadFileAsync(HttpStatusCode httpStatusCode, ResultSta
             _fileMetadata.stageInfo.location = httpStatusCode.ToString();
 
             // Act
-            await _client.DownloadFileAsync(_fileMetadata, DownloadFileName, Parallel, _cancellationToken).ConfigureAwait(false);
+            await _client.DownloadFileAsync(_fileMetadata, t_downloadFileName, Parallel, _cancellationToken).ConfigureAwait(false);
 
             // Assert
             Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus);