Skip to content

Commit

Permalink
SNOW-950923 fix to provide QueryID for failures during GET/PUT file o…
Browse files Browse the repository at this point in the history 
…perations (#825)

### Description
QueryID is super helpful when solving issues but it was not reachable
when PUT/GET operation got failed at the command/exception level.
It was not available for successful scenario for PUT/GET commands as
well.
This change actually introduces a new type of exception prior to the one
used previously - FileNotFoundException/DirectoryNotFoundException and
this previous exception is now an inner exception of
SnowflakeDbException which we had to use to provide QueryId.
In rare cases (for instance when a connection was not opened) request to
PUT/GET file on stage might still end up with a base Exception since
QueryId won't be created in such a case.

### Checklist
- [x] Code compiles correctly
- [x] Code is formatted according to [Coding
Conventions](../CodingConventions.md)
- [x] Created tests which fail without the change (if possible)
- [x] All tests passing (`dotnet test`)
- [x] Extended the README / documentation, if necessary
- [x] Provide JIRA issue id (if possible) or GitHub issue id in PR name
  • Loading branch information
@sfc-gh-mhofman
sfc-gh-mhofman authored Feb 21, 2024
1 parent 9d61e81 commit fd193e1
Show file tree
Hide file tree
Showing 12 changed files with 555 additions and 156 deletions.
62 changes: 62 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -672,6 +672,68 @@ using (IDbConnection conn = new SnowflakeDbConnection())
}
```

PUT local files to stage
------------------------

PUT command can be used to upload files of a local directory or a single local file to the Snowflake stages (named, internal table stage or internal user stage).
Such staging files can be used to load data into a table.
More on this topic: [File staging with PUT](https://docs.snowflake.com/en/sql-reference/sql/put).

In the driver the command can be executed in a bellow way:
```cs
using (IDbConnection conn = new SnowflakeDbConnection())
{
try
{
conn.ConnectionString = "<connection parameters>";
conn.Open();
var cmd = (SnowflakeDbCommand)conn.CreateCommand(); // cast allows get QueryId from the command
cmd.CommandText = "PUT file://some_data.csv @my_schema.my_stage AUTO_COMPRESS=TRUE";
var reader = cmd.ExecuteReader();
Assert.IsTrue(reader.read());
Assert.DoesNotThrow(() => Guid.Parse(cmd.GetQueryId()));
}
catch (SnowflakeDbException e)
{
Assert.DoesNotThrow(() => Guid.Parse(e.QueryId)); // when failed
Assert.That(e.InnerException.GetType(), Is.EqualTo(typeof(FileNotFoundException)));
}
```
In case of a failure a SnowflakeDbException exception will be thrown with affected QueryId if possible.
If it was after the query got executed this exception will be a SnowflakeDbException containing affected QueryId.
In case of the initial phase of execution QueryId might not be provided.
Inner exception (if applicable) will provide some details on the failure cause and
it will be for example: FileNotFoundException, DirectoryNotFoundException.

GET stage files
---------------
GET command allows to download stage directories or files to a local directory.
It can be used in connection with named stage, table internal stage or user stage.
Detailed information on the command: [Downloading files with GET](https://docs.snowflake.com/en/sql-reference/sql/get).
To use the command in a driver similar code can be executed in a client app:
```cs
try
{
conn.ConnectionString = "<connection parameters>";
conn.Open();
var cmd = (SnowflakeDbCommand)conn.CreateCommand(); // cast allows get QueryId from the command
cmd.CommandText = "GET @my_schema.my_stage/stage_file.csv file://local_file.csv AUTO_COMPRESS=TRUE";
var reader = cmd.ExecuteReader();
Assert.IsTrue(reader.read()); // True on success, False if failure
Assert.DoesNotThrow(() => Guid.Parse(cmd.GetQueryId()));
}
catch (SnowflakeDbException e)
{
Assert.DoesNotThrow(() => Guid.Parse(e.QueryId)); // on failure
}
```
In case of a failure a SnowflakeDbException will be thrown with affected QueryId if possible.
When no technical or syntax errors occurred but the DBDataReader has no data to process it returns False
without throwing an exception.

Close the Connection
--------------------

Expand Down
60 changes: 34 additions & 26 deletions Snowflake.Data.Tests/IntegrationTests/SFConnectionIT.cs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
/*
* Copyright (c) 2012-2021 Snowflake Computing Inc. All rights reserved.
* Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.
*/

using System.Data.Common;
using System.Net;
using Snowflake.Data.Tests.Util;

namespace Snowflake.Data.Tests.IntegrationTests
{
Expand Down Expand Up @@ -503,8 +505,7 @@ public void TestDefaultLoginTimeout()
{
if (e.InnerException is SnowflakeDbException)
{
Assert.AreEqual(SFError.REQUEST_TIMEOUT.GetAttribute<SFErrorAttr>().errorCode,
((SnowflakeDbException)e.InnerException).ErrorCode);
SnowflakeDbExceptionAssert.HasErrorCode(e.InnerException, SFError.REQUEST_TIMEOUT);

stopwatch.Stop();
int delta = 10; // in case server time slower.
Expand All @@ -519,12 +520,12 @@ public void TestDefaultLoginTimeout()
}

[Test]
public void TestConnectionFailFast()
public void TestConnectionFailFastForNonRetried404OnLogin()
{
using (var conn = new SnowflakeDbConnection())
{
// Just a way to get a 404 on the login request and make sure there are no retry
string invalidConnectionString = "host=learn.microsoft.com;"
string invalidConnectionString = "host=google.com/404;"
+ "connection_timeout=0;account=testFailFast;user=testFailFast;password=testFailFast;";

conn.ConnectionString = invalidConnectionString;
Expand All @@ -537,20 +538,24 @@ public void TestConnectionFailFast()
}
catch (SnowflakeDbException e)
{
Assert.AreEqual(SFError.INTERNAL_ERROR.GetAttribute<SFErrorAttr>().errorCode,
e.ErrorCode);
SnowflakeDbExceptionAssert.HasHttpErrorCodeInExceptionChain(e, HttpStatusCode.NotFound);
SnowflakeDbExceptionAssert.HasMessageInExceptionChain(e, "404 (Not Found)");
}
catch (Exception unexpected)
{
Assert.Fail($"Unexpected {unexpected.GetType()} exception occurred");
}

Assert.AreEqual(ConnectionState.Closed, conn.State);
}
}

[Test]
public void TestEnableRetry()
public void TestEnableLoginRetryOn404()
{
using (var conn = new SnowflakeDbConnection())
{
string invalidConnectionString = "host=learn.microsoft.com;"
string invalidConnectionString = "host=google.com/404;"
+ "connection_timeout=0;account=testFailFast;user=testFailFast;password=testFailFast;disableretry=true;forceretryon404=true";
conn.ConnectionString = invalidConnectionString;

Expand All @@ -562,8 +567,12 @@ public void TestEnableRetry()
}
catch (SnowflakeDbException e)
{
Assert.AreEqual(SFError.INTERNAL_ERROR.GetAttribute<SFErrorAttr>().errorCode,
e.ErrorCode);
SnowflakeDbExceptionAssert.HasErrorCode(e, SFError.INTERNAL_ERROR);
SnowflakeDbExceptionAssert.HasHttpErrorCodeInExceptionChain(e, HttpStatusCode.NotFound);
}
catch (Exception unexpected)
{
Assert.Fail($"Unexpected {unexpected.GetType()} exception occurred");
}

Assert.AreEqual(ConnectionState.Closed, conn.State);
Expand Down Expand Up @@ -794,7 +803,7 @@ public void TestUnknownAuthenticator()
}
catch (SnowflakeDbException e)
{
Assert.AreEqual(SFError.UNKNOWN_AUTHENTICATOR.GetAttribute<SFErrorAttr>().errorCode, e.ErrorCode);
SnowflakeDbExceptionAssert.HasErrorCode(e, SFError.UNKNOWN_AUTHENTICATOR);
}

}
Expand Down Expand Up @@ -849,7 +858,7 @@ public void TestOktaConnectionUntilMaxTimeout()
catch (Exception e)
{
Assert.IsInstanceOf<SnowflakeDbException>(e);
Assert.AreEqual(SFError.INTERNAL_ERROR.GetAttribute<SFErrorAttr>().errorCode, ((SnowflakeDbException)e).ErrorCode);
SnowflakeDbExceptionAssert.HasErrorCode(e, SFError.INTERNAL_ERROR);
Assert.IsTrue(e.Message.Contains(
$"The retry count has reached its limit of {expectedMaxRetryCount} and " +
$"the timeout elapsed has reached its limit of {expectedMaxConnectionTimeout} " +
Expand Down Expand Up @@ -1856,9 +1865,7 @@ public void TestAsyncLoginTimeout()
}
catch (AggregateException e)
{
Assert.AreEqual(SFError.INTERNAL_ERROR.GetAttribute<SFErrorAttr>().errorCode,
((SnowflakeDbException)e.InnerException).ErrorCode);

SnowflakeDbExceptionAssert.HasErrorCode(e.InnerException, SFError.INTERNAL_ERROR);
}
stopwatch.Stop();
int delta = 10; // in case server time slower.
Expand Down Expand Up @@ -1894,9 +1901,7 @@ public void TestAsyncLoginTimeoutWithRetryTimeoutLesserThanConnectionTimeout()
}
catch (AggregateException e)
{
Assert.AreEqual(SFError.INTERNAL_ERROR.GetAttribute<SFErrorAttr>().errorCode,
((SnowflakeDbException)e.InnerException).ErrorCode);

SnowflakeDbExceptionAssert.HasErrorCode(e.InnerException, SFError.INTERNAL_ERROR);
}
stopwatch.Stop();
int delta = 10; // in case server time slower.
Expand Down Expand Up @@ -1929,8 +1934,7 @@ public void TestAsyncDefaultLoginTimeout()
}
catch (AggregateException e)
{
Assert.AreEqual(SFError.INTERNAL_ERROR.GetAttribute<SFErrorAttr>().errorCode,
((SnowflakeDbException)e.InnerException).ErrorCode);
SnowflakeDbExceptionAssert.HasErrorCode(e.InnerException, SFError.INTERNAL_ERROR);
}
stopwatch.Stop();
int delta = 10; // in case server time slower.
Expand All @@ -1946,12 +1950,12 @@ public void TestAsyncDefaultLoginTimeout()
}

[Test]
public void TestAsyncConnectionFailFast()
public void TestAsyncConnectionFailFastForNonRetried404OnLogin()
{
using (var conn = new SnowflakeDbConnection())
{
// Just a way to get a 404 on the login request and make sure there are no retry
string invalidConnectionString = "host=learn.microsoft.com;"
string invalidConnectionString = "host=google.com/404;"
+ "connection_timeout=0;account=testFailFast;user=testFailFast;password=testFailFast;";

conn.ConnectionString = invalidConnectionString;
Expand All @@ -1966,8 +1970,12 @@ public void TestAsyncConnectionFailFast()
}
catch (AggregateException e)
{
Assert.AreEqual(SFError.INTERNAL_ERROR.GetAttribute<SFErrorAttr>().errorCode,
((SnowflakeDbException)e.InnerException).ErrorCode);
SnowflakeDbExceptionAssert.HasHttpErrorCodeInExceptionChain(e, HttpStatusCode.NotFound);
SnowflakeDbExceptionAssert.HasMessageInExceptionChain(e, "404 (Not Found)");
}
catch (Exception unexpected)
{
Assert.Fail($"Unexpected {unexpected.GetType()} exception occurred");
}

Assert.AreEqual(ConnectionState.Closed, conn.State);
Expand Down Expand Up @@ -2131,7 +2139,7 @@ public void TestAsyncOktaConnectionUntilMaxTimeout()
catch (Exception e)
{
Assert.IsInstanceOf<SnowflakeDbException>(e.InnerException);
Assert.AreEqual(SFError.INTERNAL_ERROR.GetAttribute<SFErrorAttr>().errorCode, ((SnowflakeDbException)e.InnerException).ErrorCode);
SnowflakeDbExceptionAssert.HasErrorCode(e.InnerException, SFError.INTERNAL_ERROR);
Exception oktaException;
#if NETFRAMEWORK
oktaException = e.InnerException.InnerException.InnerException;
Expand Down
Loading

0 comments on commit fd193e1

Please sign in to comment.