From a2253d0cf50b52dd64f4e696acc90975bb2947e0 Mon Sep 17 00:00:00 2001 From: Krzysztof Nozderko Date: Mon, 21 Oct 2024 16:03:42 +0200 Subject: [PATCH 1/4] SNOW-1524245 Gcm encryption using BouncyCastle --- .../UnitTests/GcmEncryptionProviderTest.cs | 281 ++++++++++++++++++ .../Util/TestDataGenarator.cs | 23 +- .../Core/FileTransfer/EncryptionProvider.cs | 13 +- .../FileTransfer/GcmEncryptionProvider.cs | 182 ++++++++++++ .../Core/FileTransfer/MaterialDescriptor.cs | 11 + .../Core/FileTransfer/SFFileMetadata.cs | 21 +- 6 files changed, 504 insertions(+), 27 deletions(-) create mode 100644 Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs create mode 100644 Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs create mode 100644 Snowflake.Data/Core/FileTransfer/MaterialDescriptor.cs diff --git a/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs b/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs new file mode 100644 index 000000000..ced5e3c1c --- /dev/null +++ b/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs @@ -0,0 +1,281 @@ +using System; +using System.IO; +using System.Text; +using NUnit.Framework; +using Org.BouncyCastle.Crypto; +using Snowflake.Data.Core; +using Snowflake.Data.Core.FileTransfer; +using Snowflake.Data.Tests.Util; + +namespace Snowflake.Data.Tests.UnitTests +{ + [TestFixture] + public class GcmEncryptionProviderTest + { + private const string PlainText = "there is no rose without a smoke"; + private static readonly byte[] s_plainTextBytes = Encoding.UTF8.GetBytes(PlainText); + private static readonly byte[] s_qsmkBytes = TestDataGenarator.NextBytes(GcmEncryptionProvider.BlockSizeInBytes); + private static readonly string s_qsmk = Convert.ToBase64String(s_qsmkBytes); + private static readonly string s_queryId = Guid.NewGuid().ToString(); + private const long SmkId = 1234L; + private const string KeyAad = "key additional information"; + private static readonly byte[] s_keyAadBytes = Encoding.UTF8.GetBytes(KeyAad); + private static readonly string s_keyAadBase64 = Convert.ToBase64String(s_keyAadBytes); + private const string ContentAad = "content additional information"; + private static readonly byte[] s_contentAadBytes = Encoding.UTF8.GetBytes(ContentAad); + private static readonly string s_contentAadBase64 = Convert.ToBase64String(s_contentAadBytes); + private const string InvalidAad = "invalid additional information"; + private static readonly byte[] s_invalidAadBytes = Encoding.UTF8.GetBytes(InvalidAad); + private static readonly string s_invalidAadBase64 = Convert.ToBase64String(s_invalidAadBytes); + private static readonly PutGetEncryptionMaterial s_encryptionMaterial = new PutGetEncryptionMaterial + { + queryStageMasterKey = s_qsmk, + queryId = s_queryId, + smkId = SmkId + }; + private static readonly FileTransferConfiguration s_fileTransferConfiguration = new FileTransferConfiguration + { + TempDir = Path.GetTempPath(), + MaxBytesInMemory = FileTransferConfiguration.DefaultMaxBytesInMemory + }; + + [Test] + public void TestEncryptAndDecryptWithoutAad() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + + // act + using (var encryptedStream = GcmEncryptionProvider.Encrypt( + s_encryptionMaterial, + encryptionMetadata, + s_fileTransferConfiguration,// this is output parameter + new MemoryStream(s_plainTextBytes), + null, + null)) + { + var encryptedContent = ExtractContentBytes(encryptedStream); + + // assert + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + Assert.IsNull(encryptionMetadata.keyAad); + Assert.IsNull(encryptionMetadata.aad); + + // act + using (var decryptedStream = GcmEncryptionProvider.Decrypt(new MemoryStream(encryptedContent), s_encryptionMaterial, encryptionMetadata, s_fileTransferConfiguration)) + { + // assert + var decryptedText = ExtractContent(decryptedStream); + CollectionAssert.AreEqual(s_plainTextBytes, decryptedText); + } + } + } + + [Test] + public void TestEncryptAndDEncryptWithAad() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + + // act + using (var encryptedStream = GcmEncryptionProvider.Encrypt( + s_encryptionMaterial, + encryptionMetadata, // this is output parameter + s_fileTransferConfiguration, + new MemoryStream(s_plainTextBytes), + s_contentAadBytes, + s_keyAadBytes)) + { + var encryptedContent = ExtractContentBytes(encryptedStream); + + // assert + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + CollectionAssert.AreEqual(s_keyAadBase64, encryptionMetadata.keyAad); + CollectionAssert.AreEqual(s_contentAadBase64, encryptionMetadata.aad); + + // act + using (var decryptedStream = GcmEncryptionProvider.Decrypt(new MemoryStream(encryptedContent), s_encryptionMaterial, encryptionMetadata, s_fileTransferConfiguration)) + { + // assert + var decryptedText = ExtractContent(decryptedStream); + CollectionAssert.AreEqual(s_plainTextBytes, decryptedText); + } + } + } + + [Test] + public void TestFailDecryptWithInvalidKeyAad() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + using (var encryptedStream = GcmEncryptionProvider.Encrypt( + s_encryptionMaterial, + encryptionMetadata, // this is output parameter + s_fileTransferConfiguration, + new MemoryStream(s_plainTextBytes), + null, + s_keyAadBytes)) + { + var encryptedContent = ExtractContentBytes(encryptedStream); + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + CollectionAssert.AreEqual(s_keyAadBase64, encryptionMetadata.keyAad); + Assert.IsNull(encryptionMetadata.aad); + encryptionMetadata.keyAad = s_invalidAadBase64; + + // act + var thrown = Assert.Throws(() => + GcmEncryptionProvider.Decrypt(new MemoryStream(encryptedContent), s_encryptionMaterial, encryptionMetadata, s_fileTransferConfiguration)); + + // assert + Assert.AreEqual("mac check in GCM failed", thrown.Message); + } + } + + [Test] + public void TestFailDecryptWithInvalidContentAad() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + using (var encryptedStream = GcmEncryptionProvider.Encrypt( + s_encryptionMaterial, + encryptionMetadata, // this is output parameter + s_fileTransferConfiguration, + new MemoryStream(s_plainTextBytes), + s_contentAadBytes, + null)) + { + var encryptedContent = ExtractContentBytes(encryptedStream); + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + Assert.IsNull(encryptionMetadata.keyAad); + CollectionAssert.AreEqual(s_contentAadBase64, encryptionMetadata.aad); + encryptionMetadata.aad = s_invalidAadBase64; + + // act + var thrown = Assert.Throws(() => + GcmEncryptionProvider.Decrypt(new MemoryStream(encryptedContent), s_encryptionMaterial, encryptionMetadata, s_fileTransferConfiguration)); + + // assert + Assert.AreEqual("mac check in GCM failed", thrown.Message); + } + } + + [Test] + public void TestFailDecryptWhenMissingAad() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + using (var encryptedStream = GcmEncryptionProvider.Encrypt( + s_encryptionMaterial, + encryptionMetadata, // this is output parameter + s_fileTransferConfiguration, + new MemoryStream(s_plainTextBytes), + s_contentAadBytes, + s_keyAadBytes)) + { + var encryptedContent = ExtractContentBytes(encryptedStream); + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + CollectionAssert.AreEqual(s_keyAadBase64, encryptionMetadata.keyAad); + CollectionAssert.AreEqual(s_contentAadBase64, encryptionMetadata.aad); + encryptionMetadata.keyAad = null; + encryptionMetadata.aad = null; + + // act + var thrown = Assert.Throws(() => + GcmEncryptionProvider.Decrypt(new MemoryStream(encryptedContent), s_encryptionMaterial, encryptionMetadata,s_fileTransferConfiguration)); + + // assert + Assert.AreEqual("mac check in GCM failed", thrown.Message); + } + } + + [Test] + public void TestEncryptAndDecryptFile() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + var plainTextFilePath = Path.Combine(Path.GetTempPath(), "plaintext.txt"); + var encryptedFilePath = Path.Combine(Path.GetTempPath(), "encrypted.txt"); + try + { + CreateFile(plainTextFilePath, PlainText); + + // act + using (var encryptedStream = GcmEncryptionProvider.EncryptFile(plainTextFilePath, s_encryptionMaterial, encryptionMetadata, + s_fileTransferConfiguration, s_contentAadBytes, s_keyAadBytes)) + { + CreateFile(encryptedFilePath, encryptedStream); + } + + // assert + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + CollectionAssert.AreEqual(s_keyAadBase64, encryptionMetadata.keyAad); + CollectionAssert.AreEqual(s_contentAadBase64, encryptionMetadata.aad); + + // act + string result; + using (var decryptedStream = GcmEncryptionProvider.DecryptFile(encryptedFilePath, s_encryptionMaterial, encryptionMetadata, + s_fileTransferConfiguration)) + { + decryptedStream.Position = 0; + var resultBytes = new byte[decryptedStream.Length]; + var bytesRead = decryptedStream.Read(resultBytes, 0, resultBytes.Length); + Assert.AreEqual(decryptedStream.Length, bytesRead); + result = Encoding.UTF8.GetString(resultBytes); + } + + // assert + CollectionAssert.AreEqual(PlainText, result); + } + finally + { + File.Delete(plainTextFilePath); + File.Delete(encryptedFilePath); + } + } + + private static void CreateFile(string filePath, string content) + { + using (var writer = File.CreateText(filePath)) + { + writer.Write(content); + } + } + + private static void CreateFile(string filePath, Stream content) + { + using (var writer = File.Create(filePath)) + { + var buffer = new byte[1024]; + int bytesRead; + content.Position = 0; + while ((bytesRead = content.Read(buffer, 0, 1024)) > 0) + { + writer.Write(buffer, 0, bytesRead); + } + } + } + + private string ExtractContent(Stream stream) => + Encoding.UTF8.GetString(ExtractContentBytes(stream)); + + private byte[] ExtractContentBytes(Stream stream) + { + var memoryStream = new MemoryStream(); + stream.Position = 0; + stream.CopyTo(memoryStream); + return memoryStream.ToArray(); + } + } +} diff --git a/Snowflake.Data.Tests/Util/TestDataGenarator.cs b/Snowflake.Data.Tests/Util/TestDataGenarator.cs index 27dda5ab0..760f1820b 100644 --- a/Snowflake.Data.Tests/Util/TestDataGenarator.cs +++ b/Snowflake.Data.Tests/Util/TestDataGenarator.cs @@ -22,7 +22,7 @@ public class TestDataGenarator public static char SnowflakeUnicode => '\u2744'; public static string EmojiUnicode => "\uD83D\uDE00"; public static string StringWithUnicode => AsciiCodes + SnowflakeUnicode + EmojiUnicode; - + public static bool NextBool() { return s_random.Next(0, 1) == 1; @@ -32,7 +32,7 @@ public static int NextInt(int minValueInclusive, int maxValueExclusive) { return s_random.Next(minValueInclusive, maxValueExclusive); } - + public static string NextAlphaNumeric() { return NextAlphaNumeric(s_random.Next(5, 12)); @@ -72,17 +72,24 @@ public static string NextDigitsString(int length) } return new string(buffer); } - + + public static byte[] NextBytes(int length) + { + var buffer = new byte[length]; + s_random.NextBytes(buffer); + return buffer; + } + private static char NextAlphaNumericChar() => NextChar(s_alphanumericChars); - + public static string NextNonZeroDigitAsString() => NextNonZeroDigitChar().ToString(); private static char NextNonZeroDigitChar() => NextChar(s_nonZeroDigits); - - private static string NextDigitAsString() => NextDigitChar().ToString(); - + + private static string NextDigitAsString() => NextDigitChar().ToString(); + private static char NextDigitChar() => NextChar(s_digitChars); - + private static string NextLetterAsString() => NextLetterChar().ToString(); private static char NextLetterChar() => NextChar(s_letterChars); diff --git a/Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs b/Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs index 411a6eeab..a35f1b11d 100644 --- a/Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs +++ b/Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs @@ -9,20 +9,9 @@ namespace Snowflake.Data.Core.FileTransfer { - /// - /// The encryption materials. - /// - internal class MaterialDescriptor - { - public string smkId { get; set; } - - public string queryId { get; set; } - - public string keySize { get; set; } - } - /// /// The encryptor/decryptor for PUT/GET files. + /// Handles encryption and decryption using EAS CBC (for files) and ECB (for keys). /// class EncryptionProvider { diff --git a/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs b/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs new file mode 100644 index 000000000..f432bb552 --- /dev/null +++ b/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs @@ -0,0 +1,182 @@ +using System; +using System.IO; +using Org.BouncyCastle.Crypto; +using Org.BouncyCastle.Crypto.IO; +using Org.BouncyCastle.Crypto.Parameters; +using Org.BouncyCastle.Security; + +namespace Snowflake.Data.Core.FileTransfer +{ + internal class GcmEncryptionProvider + { + private const int AesBlockSize = 128; + internal const int BlockSizeInBytes = AesBlockSize / 8; + private const string AesGcmNoPaddingCipher = "AES/GCM/NoPadding"; + + private static readonly SecureRandom s_random = SecureRandom.GetInstance("SHA1PRNG"); + + public static Stream EncryptFile( + string inFile, + PutGetEncryptionMaterial encryptionMaterial, + SFEncryptionMetadata encryptionMetadata, + FileTransferConfiguration transferConfiguration, + byte[] contentAad, + byte[] keyAad + ) + { + using (var fileStream = File.OpenRead(inFile)) + { + return Encrypt(encryptionMaterial, encryptionMetadata, transferConfiguration, fileStream, contentAad, keyAad); + } + } + + public static Stream DecryptFile( + string inFile, + PutGetEncryptionMaterial encryptionMaterial, + SFEncryptionMetadata encryptionMetadata, + FileTransferConfiguration transferConfiguration) + { + using (var fileStream = File.OpenRead(inFile)) + { + return Decrypt(fileStream, encryptionMaterial, encryptionMetadata, transferConfiguration); + } + } + + public static Stream Encrypt( + PutGetEncryptionMaterial encryptionMaterial, + SFEncryptionMetadata encryptionMetadata, // this is output parameter + FileTransferConfiguration fileTransferConfiguration, + Stream inputStream, + byte[] contentAad, + byte[] keyAad) + { + byte[] decodedMasterKey = Convert.FromBase64String(encryptionMaterial.queryStageMasterKey); + int masterKeySize = decodedMasterKey.Length; + // s_logger.Debug($"Master key size : {masterKeySize}"); + + var contentIV = new byte[BlockSizeInBytes]; + var keyIV = new byte[BlockSizeInBytes]; + var fileKeyBytes = new byte[masterKeySize]; // we choose a random fileKey to encrypt it with qsmk key with GCM + s_random.NextBytes(contentIV); + s_random.NextBytes(keyIV); + s_random.NextBytes(fileKeyBytes); + + var encryptedKey = EncryptKey(fileKeyBytes, decodedMasterKey, keyIV, keyAad); + var result = EncryptContent(inputStream, fileKeyBytes, contentIV, contentAad, fileTransferConfiguration); + + MaterialDescriptor matDesc = new MaterialDescriptor + { + smkId = encryptionMaterial.smkId.ToString(), + queryId = encryptionMaterial.queryId, + keySize = (masterKeySize * 8).ToString() + }; + + encryptionMetadata.key = Convert.ToBase64String(encryptedKey); + encryptionMetadata.iv = Convert.ToBase64String(contentIV); + encryptionMetadata.keyIV = Convert.ToBase64String(keyIV); + encryptionMetadata.keyAad = keyAad == null ? null : Convert.ToBase64String(keyAad); + encryptionMetadata.aad = contentAad == null ? null : Convert.ToBase64String(contentAad); + encryptionMetadata.matDesc = Newtonsoft.Json.JsonConvert.SerializeObject(matDesc); + + return result; + } + + public static Stream Decrypt( + Stream inputStream, + PutGetEncryptionMaterial encryptionMaterial, + SFEncryptionMetadata encryptionMetadata, + FileTransferConfiguration fileTransferConfiguration) + { + var decodedMasterKey = Convert.FromBase64String(encryptionMaterial.queryStageMasterKey); + var keyBytes = Convert.FromBase64String(encryptionMetadata.key); + var keyIVBytes = Convert.FromBase64String(encryptionMetadata.keyIV); + var keyAad = encryptionMetadata.keyAad == null ? null : Convert.FromBase64String(encryptionMetadata.keyAad); + var ivBytes = Convert.FromBase64String(encryptionMetadata.iv); + var contentAad = encryptionMetadata.aad == null ? null : Convert.FromBase64String(encryptionMetadata.aad); + var decryptedFileKey = DecryptKey(keyBytes, decodedMasterKey, keyIVBytes, keyAad); + return DecryptContent(inputStream, decryptedFileKey, ivBytes, contentAad, fileTransferConfiguration); + } + + private static byte[] EncryptKey(byte[] fileKeyBytes, byte[] qsmk, byte[] keyIV, byte[] keyAad) + { + var keyCipher = BuildAesGcmNoPaddingCipher(true, qsmk, keyIV, keyAad); + var cipherKeyData = new byte[keyCipher.GetOutputSize(fileKeyBytes.Length)]; + var processLength = keyCipher.ProcessBytes(fileKeyBytes, 0, fileKeyBytes.Length, cipherKeyData, 0); + keyCipher.DoFinal(cipherKeyData, processLength); + return cipherKeyData; + } + + private static Stream EncryptContent(Stream inputStream, byte[] fileKeyBytes, byte[] contentIV, byte[] contentAad, + FileTransferConfiguration transferConfiguration) + { + var contentCipher = BuildAesGcmNoPaddingCipher(true, fileKeyBytes, contentIV, contentAad); + var targetStream = new FileBackedOutputStream(transferConfiguration.MaxBytesInMemory, transferConfiguration.TempDir); + try + { + var cipherStream = new CipherStream(targetStream, null, contentCipher); + byte[] buffer = new byte[transferConfiguration.MaxBytesInMemory]; + int bytesRead; + while ((bytesRead = inputStream.Read(buffer, 0, buffer.Length)) > 0) + { + cipherStream.Write(buffer, 0, bytesRead); + } + + cipherStream.Flush(); // we cannot close or dispose cipherStream because: 1) it would do additional DoFinal resulting in an exception 2) closing cipherStream would close target stream + var mac = contentCipher.DoFinal(); // getting authentication tag for the whole content + targetStream.Write(mac, 0, mac.Length); + return targetStream; + } + catch (Exception) + { + targetStream.Dispose(); + throw; + } + } + + private static byte[] DecryptKey(byte[] fileKey, byte[] qsmk, byte[] keyIV, byte[] keyAad) + { + var keyCipher = BuildAesGcmNoPaddingCipher(false, qsmk, keyIV, keyAad); + var decryptedKeyData = new byte[keyCipher.GetOutputSize(fileKey.Length)]; + var processLength = keyCipher.ProcessBytes(fileKey, 0, fileKey.Length, decryptedKeyData, 0); + keyCipher.DoFinal(decryptedKeyData, processLength); + return decryptedKeyData; + } + + private static Stream DecryptContent(Stream inputStream, byte[] fileKeyBytes, byte[] contentIV, byte[] contentAad, + FileTransferConfiguration transferConfiguration) + { + var contentCipher = BuildAesGcmNoPaddingCipher(false, fileKeyBytes, contentIV, contentAad); + var targetStream = new FileBackedOutputStream(transferConfiguration.MaxBytesInMemory, transferConfiguration.TempDir); + try + { + var cipherStream = new CipherStream(targetStream, null, contentCipher); + byte[] buffer = new byte[transferConfiguration.MaxBytesInMemory]; + int bytesRead; + while ((bytesRead = inputStream.Read(buffer, 0, buffer.Length)) > 0) + { + cipherStream.Write(buffer, 0, bytesRead); + } + + cipherStream.Flush(); // we cannot close or dispose cipherStream because closing cipherStream would close target stream + contentCipher.DoFinal(); // in case of decrypting we ignore the result which has to be empty + return targetStream; + } + catch (Exception) + { + targetStream.Dispose(); + throw; + } + } + + private static IBufferedCipher BuildAesGcmNoPaddingCipher(bool forEncryption, byte[] keyBytes, byte[] initialisationVector, byte[] aadData) + { + var cipher = CipherUtilities.GetCipher(AesGcmNoPaddingCipher); + KeyParameter keyParameter = new KeyParameter(keyBytes); + var keyParameterAead = aadData == null + ? new AeadParameters(keyParameter, AesBlockSize, initialisationVector) + : new AeadParameters(keyParameter, AesBlockSize, initialisationVector, aadData); + cipher.Init(forEncryption, keyParameterAead); + return cipher; + } + } +} diff --git a/Snowflake.Data/Core/FileTransfer/MaterialDescriptor.cs b/Snowflake.Data/Core/FileTransfer/MaterialDescriptor.cs new file mode 100644 index 000000000..e0b352910 --- /dev/null +++ b/Snowflake.Data/Core/FileTransfer/MaterialDescriptor.cs @@ -0,0 +1,11 @@ +namespace Snowflake.Data.Core.FileTransfer +{ + internal class MaterialDescriptor + { + public string smkId { get; set; } + + public string queryId { get; set; } + + public string keySize { get; set; } + } +} diff --git a/Snowflake.Data/Core/FileTransfer/SFFileMetadata.cs b/Snowflake.Data/Core/FileTransfer/SFFileMetadata.cs index 605de0be1..e1647257b 100644 --- a/Snowflake.Data/Core/FileTransfer/SFFileMetadata.cs +++ b/Snowflake.Data/Core/FileTransfer/SFFileMetadata.cs @@ -3,21 +3,28 @@ */ using System; -using System.Collections.Generic; using System.IO; -using System.Text; using static Snowflake.Data.Core.FileTransfer.SFFileCompressionTypes; namespace Snowflake.Data.Core.FileTransfer { public class SFEncryptionMetadata { - /// Initialization vector + /// Initialization vector for file content encryption public string iv { set; get; } /// File key public string key { set; get; } + /// Additional Authentication Data for file content encryption + public string aad { set; get; } + + /// Initialization vector for key encryption + public string keyIV { set; get; } + + /// Additional Authentication Data for key encryption + public string keyAad { set; get; } + /// Encryption material descriptor public string matDesc { set; get; } } @@ -89,7 +96,7 @@ internal class SFFileMetadata /// File message digest (after compression if required) public string sha256Digest { set; get; } - /// Source compression + /// Source compression public SFFileCompressionType sourceCompression { set; get; } /// Target compression @@ -122,9 +129,9 @@ internal class SFFileMetadata // Proxy credentials of the remote storage client. public ProxyCredentials proxyCredentials { get; set; } - + public int MaxBytesInMemory { get; set; } - + internal CommandTypes _operationType; internal string RemoteFileName() @@ -142,7 +149,7 @@ internal class FileTransferConfiguration { private const int OneMegabyteInBytes = 1024 * 1024; - + public string TempDir { get; set; } public int MaxBytesInMemory { get; set; } From 5ea1cacaafc64a051336d6824b2faf987a297cc5 Mon Sep 17 00:00:00 2001 From: Krzysztof Nozderko Date: Mon, 21 Oct 2024 16:30:23 +0200 Subject: [PATCH 2/4] add logger --- Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs b/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs index f432bb552..e60eccba1 100644 --- a/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs +++ b/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs @@ -4,6 +4,7 @@ using Org.BouncyCastle.Crypto.IO; using Org.BouncyCastle.Crypto.Parameters; using Org.BouncyCastle.Security; +using Snowflake.Data.Log; namespace Snowflake.Data.Core.FileTransfer { @@ -13,6 +14,8 @@ internal class GcmEncryptionProvider internal const int BlockSizeInBytes = AesBlockSize / 8; private const string AesGcmNoPaddingCipher = "AES/GCM/NoPadding"; + private static readonly SFLogger s_logger = SFLoggerFactory.GetLogger(); + private static readonly SecureRandom s_random = SecureRandom.GetInstance("SHA1PRNG"); public static Stream EncryptFile( @@ -52,7 +55,7 @@ public static Stream Encrypt( { byte[] decodedMasterKey = Convert.FromBase64String(encryptionMaterial.queryStageMasterKey); int masterKeySize = decodedMasterKey.Length; - // s_logger.Debug($"Master key size : {masterKeySize}"); + s_logger.Debug($"Master key size : {masterKeySize}"); var contentIV = new byte[BlockSizeInBytes]; var keyIV = new byte[BlockSizeInBytes]; From 4f4bc6bd21e3a3ae054937d1e2ccd4de520d5aa4 Mon Sep 17 00:00:00 2001 From: Krzysztof Nozderko Date: Tue, 22 Oct 2024 11:23:43 +0200 Subject: [PATCH 3/4] changes from review --- Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs | 4 ++-- Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs b/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs index ced5e3c1c..a8df88ff8 100644 --- a/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs +++ b/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs @@ -12,7 +12,7 @@ namespace Snowflake.Data.Tests.UnitTests [TestFixture] public class GcmEncryptionProviderTest { - private const string PlainText = "there is no rose without a smoke"; + private const string PlainText = "there is no rose without thorns"; private static readonly byte[] s_plainTextBytes = Encoding.UTF8.GetBytes(PlainText); private static readonly byte[] s_qsmkBytes = TestDataGenarator.NextBytes(GcmEncryptionProvider.BlockSizeInBytes); private static readonly string s_qsmk = Convert.ToBase64String(s_qsmkBytes); @@ -74,7 +74,7 @@ public void TestEncryptAndDecryptWithoutAad() } [Test] - public void TestEncryptAndDEncryptWithAad() + public void TestEncryptAndDecryptWithAad() { // arrange SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); diff --git a/Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs b/Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs index a35f1b11d..b625f80d3 100644 --- a/Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs +++ b/Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs @@ -11,7 +11,7 @@ namespace Snowflake.Data.Core.FileTransfer { /// /// The encryptor/decryptor for PUT/GET files. - /// Handles encryption and decryption using EAS CBC (for files) and ECB (for keys). + /// Handles encryption and decryption using AES CBC (for files) and ECB (for keys). /// class EncryptionProvider { From 1aebc4b565f002b6ecf81609c087afe39cf991e9 Mon Sep 17 00:00:00 2001 From: Krzysztof Nozderko Date: Tue, 22 Oct 2024 13:58:46 +0200 Subject: [PATCH 4/4] fix bug in decoding, add tests for empty aad --- .../UnitTests/GcmEncryptionProviderTest.cs | 41 ++++++++++++++++++- .../FileTransfer/GcmEncryptionProvider.cs | 7 +++- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs b/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs index a8df88ff8..60c0c2059 100644 --- a/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs +++ b/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs @@ -27,6 +27,9 @@ public class GcmEncryptionProviderTest private const string InvalidAad = "invalid additional information"; private static readonly byte[] s_invalidAadBytes = Encoding.UTF8.GetBytes(InvalidAad); private static readonly string s_invalidAadBase64 = Convert.ToBase64String(s_invalidAadBytes); + private static readonly string s_emptyAad = string.Empty; + private static readonly byte[] s_emptyAadBytes = Encoding.UTF8.GetBytes(s_emptyAad); + private static readonly string s_emptyAadBase64 = Convert.ToBase64String(s_emptyAadBytes); private static readonly PutGetEncryptionMaterial s_encryptionMaterial = new PutGetEncryptionMaterial { queryStageMasterKey = s_qsmk, @@ -48,8 +51,8 @@ public void TestEncryptAndDecryptWithoutAad() // act using (var encryptedStream = GcmEncryptionProvider.Encrypt( s_encryptionMaterial, - encryptionMetadata, - s_fileTransferConfiguration,// this is output parameter + encryptionMetadata, // this is output parameter + s_fileTransferConfiguration, new MemoryStream(s_plainTextBytes), null, null)) @@ -73,6 +76,40 @@ public void TestEncryptAndDecryptWithoutAad() } } + [Test] + public void TestEncryptAndDecryptWithEmptyAad() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + + // act + using (var encryptedStream = GcmEncryptionProvider.Encrypt( + s_encryptionMaterial, + encryptionMetadata, // this is output parameter + s_fileTransferConfiguration, + new MemoryStream(s_plainTextBytes), + s_emptyAadBytes, + s_emptyAadBytes)) + { + var encryptedContent = ExtractContentBytes(encryptedStream); + + // assert + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + Assert.AreEqual(s_emptyAadBase64, encryptionMetadata.keyAad); + Assert.AreEqual(s_emptyAadBase64, encryptionMetadata.aad); + + // act + using (var decryptedStream = GcmEncryptionProvider.Decrypt(new MemoryStream(encryptedContent), s_encryptionMaterial, encryptionMetadata, s_fileTransferConfiguration)) + { + // assert + var decryptedText = ExtractContent(decryptedStream); + CollectionAssert.AreEqual(s_plainTextBytes, decryptedText); + } + } + } + [Test] public void TestEncryptAndDecryptWithAad() { diff --git a/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs b/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs index e60eccba1..50b80dd05 100644 --- a/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs +++ b/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs @@ -159,9 +159,12 @@ private static Stream DecryptContent(Stream inputStream, byte[] fileKeyBytes, by { cipherStream.Write(buffer, 0, bytesRead); } - cipherStream.Flush(); // we cannot close or dispose cipherStream because closing cipherStream would close target stream - contentCipher.DoFinal(); // in case of decrypting we ignore the result which has to be empty + var lastBytes = contentCipher.DoFinal(); + if (lastBytes != null && lastBytes.Length > 0) + { + targetStream.Write(lastBytes, 0, lastBytes.Length); + } return targetStream; } catch (Exception)