diff --git a/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs b/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs new file mode 100644 index 000000000..ced5e3c1c --- /dev/null +++ b/Snowflake.Data.Tests/UnitTests/GcmEncryptionProviderTest.cs @@ -0,0 +1,281 @@ +using System; +using System.IO; +using System.Text; +using NUnit.Framework; +using Org.BouncyCastle.Crypto; +using Snowflake.Data.Core; +using Snowflake.Data.Core.FileTransfer; +using Snowflake.Data.Tests.Util; + +namespace Snowflake.Data.Tests.UnitTests +{ + [TestFixture] + public class GcmEncryptionProviderTest + { + private const string PlainText = "there is no rose without a smoke"; + private static readonly byte[] s_plainTextBytes = Encoding.UTF8.GetBytes(PlainText); + private static readonly byte[] s_qsmkBytes = TestDataGenarator.NextBytes(GcmEncryptionProvider.BlockSizeInBytes); + private static readonly string s_qsmk = Convert.ToBase64String(s_qsmkBytes); + private static readonly string s_queryId = Guid.NewGuid().ToString(); + private const long SmkId = 1234L; + private const string KeyAad = "key additional information"; + private static readonly byte[] s_keyAadBytes = Encoding.UTF8.GetBytes(KeyAad); + private static readonly string s_keyAadBase64 = Convert.ToBase64String(s_keyAadBytes); + private const string ContentAad = "content additional information"; + private static readonly byte[] s_contentAadBytes = Encoding.UTF8.GetBytes(ContentAad); + private static readonly string s_contentAadBase64 = Convert.ToBase64String(s_contentAadBytes); + private const string InvalidAad = "invalid additional information"; + private static readonly byte[] s_invalidAadBytes = Encoding.UTF8.GetBytes(InvalidAad); + private static readonly string s_invalidAadBase64 = Convert.ToBase64String(s_invalidAadBytes); + private static readonly PutGetEncryptionMaterial s_encryptionMaterial = new PutGetEncryptionMaterial + { + queryStageMasterKey = s_qsmk, + queryId = s_queryId, + smkId = SmkId + }; + private static readonly FileTransferConfiguration s_fileTransferConfiguration = new FileTransferConfiguration + { + TempDir = Path.GetTempPath(), + MaxBytesInMemory = FileTransferConfiguration.DefaultMaxBytesInMemory + }; + + [Test] + public void TestEncryptAndDecryptWithoutAad() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + + // act + using (var encryptedStream = GcmEncryptionProvider.Encrypt( + s_encryptionMaterial, + encryptionMetadata, + s_fileTransferConfiguration,// this is output parameter + new MemoryStream(s_plainTextBytes), + null, + null)) + { + var encryptedContent = ExtractContentBytes(encryptedStream); + + // assert + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + Assert.IsNull(encryptionMetadata.keyAad); + Assert.IsNull(encryptionMetadata.aad); + + // act + using (var decryptedStream = GcmEncryptionProvider.Decrypt(new MemoryStream(encryptedContent), s_encryptionMaterial, encryptionMetadata, s_fileTransferConfiguration)) + { + // assert + var decryptedText = ExtractContent(decryptedStream); + CollectionAssert.AreEqual(s_plainTextBytes, decryptedText); + } + } + } + + [Test] + public void TestEncryptAndDEncryptWithAad() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + + // act + using (var encryptedStream = GcmEncryptionProvider.Encrypt( + s_encryptionMaterial, + encryptionMetadata, // this is output parameter + s_fileTransferConfiguration, + new MemoryStream(s_plainTextBytes), + s_contentAadBytes, + s_keyAadBytes)) + { + var encryptedContent = ExtractContentBytes(encryptedStream); + + // assert + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + CollectionAssert.AreEqual(s_keyAadBase64, encryptionMetadata.keyAad); + CollectionAssert.AreEqual(s_contentAadBase64, encryptionMetadata.aad); + + // act + using (var decryptedStream = GcmEncryptionProvider.Decrypt(new MemoryStream(encryptedContent), s_encryptionMaterial, encryptionMetadata, s_fileTransferConfiguration)) + { + // assert + var decryptedText = ExtractContent(decryptedStream); + CollectionAssert.AreEqual(s_plainTextBytes, decryptedText); + } + } + } + + [Test] + public void TestFailDecryptWithInvalidKeyAad() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + using (var encryptedStream = GcmEncryptionProvider.Encrypt( + s_encryptionMaterial, + encryptionMetadata, // this is output parameter + s_fileTransferConfiguration, + new MemoryStream(s_plainTextBytes), + null, + s_keyAadBytes)) + { + var encryptedContent = ExtractContentBytes(encryptedStream); + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + CollectionAssert.AreEqual(s_keyAadBase64, encryptionMetadata.keyAad); + Assert.IsNull(encryptionMetadata.aad); + encryptionMetadata.keyAad = s_invalidAadBase64; + + // act + var thrown = Assert.Throws(() => + GcmEncryptionProvider.Decrypt(new MemoryStream(encryptedContent), s_encryptionMaterial, encryptionMetadata, s_fileTransferConfiguration)); + + // assert + Assert.AreEqual("mac check in GCM failed", thrown.Message); + } + } + + [Test] + public void TestFailDecryptWithInvalidContentAad() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + using (var encryptedStream = GcmEncryptionProvider.Encrypt( + s_encryptionMaterial, + encryptionMetadata, // this is output parameter + s_fileTransferConfiguration, + new MemoryStream(s_plainTextBytes), + s_contentAadBytes, + null)) + { + var encryptedContent = ExtractContentBytes(encryptedStream); + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + Assert.IsNull(encryptionMetadata.keyAad); + CollectionAssert.AreEqual(s_contentAadBase64, encryptionMetadata.aad); + encryptionMetadata.aad = s_invalidAadBase64; + + // act + var thrown = Assert.Throws(() => + GcmEncryptionProvider.Decrypt(new MemoryStream(encryptedContent), s_encryptionMaterial, encryptionMetadata, s_fileTransferConfiguration)); + + // assert + Assert.AreEqual("mac check in GCM failed", thrown.Message); + } + } + + [Test] + public void TestFailDecryptWhenMissingAad() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + using (var encryptedStream = GcmEncryptionProvider.Encrypt( + s_encryptionMaterial, + encryptionMetadata, // this is output parameter + s_fileTransferConfiguration, + new MemoryStream(s_plainTextBytes), + s_contentAadBytes, + s_keyAadBytes)) + { + var encryptedContent = ExtractContentBytes(encryptedStream); + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + CollectionAssert.AreEqual(s_keyAadBase64, encryptionMetadata.keyAad); + CollectionAssert.AreEqual(s_contentAadBase64, encryptionMetadata.aad); + encryptionMetadata.keyAad = null; + encryptionMetadata.aad = null; + + // act + var thrown = Assert.Throws(() => + GcmEncryptionProvider.Decrypt(new MemoryStream(encryptedContent), s_encryptionMaterial, encryptionMetadata,s_fileTransferConfiguration)); + + // assert + Assert.AreEqual("mac check in GCM failed", thrown.Message); + } + } + + [Test] + public void TestEncryptAndDecryptFile() + { + // arrange + SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata(); + var plainTextFilePath = Path.Combine(Path.GetTempPath(), "plaintext.txt"); + var encryptedFilePath = Path.Combine(Path.GetTempPath(), "encrypted.txt"); + try + { + CreateFile(plainTextFilePath, PlainText); + + // act + using (var encryptedStream = GcmEncryptionProvider.EncryptFile(plainTextFilePath, s_encryptionMaterial, encryptionMetadata, + s_fileTransferConfiguration, s_contentAadBytes, s_keyAadBytes)) + { + CreateFile(encryptedFilePath, encryptedStream); + } + + // assert + Assert.NotNull(encryptionMetadata.key); + Assert.NotNull(encryptionMetadata.iv); + Assert.NotNull(encryptionMetadata.matDesc); + CollectionAssert.AreEqual(s_keyAadBase64, encryptionMetadata.keyAad); + CollectionAssert.AreEqual(s_contentAadBase64, encryptionMetadata.aad); + + // act + string result; + using (var decryptedStream = GcmEncryptionProvider.DecryptFile(encryptedFilePath, s_encryptionMaterial, encryptionMetadata, + s_fileTransferConfiguration)) + { + decryptedStream.Position = 0; + var resultBytes = new byte[decryptedStream.Length]; + var bytesRead = decryptedStream.Read(resultBytes, 0, resultBytes.Length); + Assert.AreEqual(decryptedStream.Length, bytesRead); + result = Encoding.UTF8.GetString(resultBytes); + } + + // assert + CollectionAssert.AreEqual(PlainText, result); + } + finally + { + File.Delete(plainTextFilePath); + File.Delete(encryptedFilePath); + } + } + + private static void CreateFile(string filePath, string content) + { + using (var writer = File.CreateText(filePath)) + { + writer.Write(content); + } + } + + private static void CreateFile(string filePath, Stream content) + { + using (var writer = File.Create(filePath)) + { + var buffer = new byte[1024]; + int bytesRead; + content.Position = 0; + while ((bytesRead = content.Read(buffer, 0, 1024)) > 0) + { + writer.Write(buffer, 0, bytesRead); + } + } + } + + private string ExtractContent(Stream stream) => + Encoding.UTF8.GetString(ExtractContentBytes(stream)); + + private byte[] ExtractContentBytes(Stream stream) + { + var memoryStream = new MemoryStream(); + stream.Position = 0; + stream.CopyTo(memoryStream); + return memoryStream.ToArray(); + } + } +} diff --git a/Snowflake.Data.Tests/Util/TestDataGenarator.cs b/Snowflake.Data.Tests/Util/TestDataGenarator.cs index 27dda5ab0..760f1820b 100644 --- a/Snowflake.Data.Tests/Util/TestDataGenarator.cs +++ b/Snowflake.Data.Tests/Util/TestDataGenarator.cs @@ -22,7 +22,7 @@ public class TestDataGenarator public static char SnowflakeUnicode => '\u2744'; public static string EmojiUnicode => "\uD83D\uDE00"; public static string StringWithUnicode => AsciiCodes + SnowflakeUnicode + EmojiUnicode; - + public static bool NextBool() { return s_random.Next(0, 1) == 1; @@ -32,7 +32,7 @@ public static int NextInt(int minValueInclusive, int maxValueExclusive) { return s_random.Next(minValueInclusive, maxValueExclusive); } - + public static string NextAlphaNumeric() { return NextAlphaNumeric(s_random.Next(5, 12)); @@ -72,17 +72,24 @@ public static string NextDigitsString(int length) } return new string(buffer); } - + + public static byte[] NextBytes(int length) + { + var buffer = new byte[length]; + s_random.NextBytes(buffer); + return buffer; + } + private static char NextAlphaNumericChar() => NextChar(s_alphanumericChars); - + public static string NextNonZeroDigitAsString() => NextNonZeroDigitChar().ToString(); private static char NextNonZeroDigitChar() => NextChar(s_nonZeroDigits); - - private static string NextDigitAsString() => NextDigitChar().ToString(); - + + private static string NextDigitAsString() => NextDigitChar().ToString(); + private static char NextDigitChar() => NextChar(s_digitChars); - + private static string NextLetterAsString() => NextLetterChar().ToString(); private static char NextLetterChar() => NextChar(s_letterChars); diff --git a/Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs b/Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs index 411a6eeab..a35f1b11d 100644 --- a/Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs +++ b/Snowflake.Data/Core/FileTransfer/EncryptionProvider.cs @@ -9,20 +9,9 @@ namespace Snowflake.Data.Core.FileTransfer { - /// - /// The encryption materials. - /// - internal class MaterialDescriptor - { - public string smkId { get; set; } - - public string queryId { get; set; } - - public string keySize { get; set; } - } - /// /// The encryptor/decryptor for PUT/GET files. + /// Handles encryption and decryption using EAS CBC (for files) and ECB (for keys). /// class EncryptionProvider { diff --git a/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs b/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs new file mode 100644 index 000000000..f432bb552 --- /dev/null +++ b/Snowflake.Data/Core/FileTransfer/GcmEncryptionProvider.cs @@ -0,0 +1,182 @@ +using System; +using System.IO; +using Org.BouncyCastle.Crypto; +using Org.BouncyCastle.Crypto.IO; +using Org.BouncyCastle.Crypto.Parameters; +using Org.BouncyCastle.Security; + +namespace Snowflake.Data.Core.FileTransfer +{ + internal class GcmEncryptionProvider + { + private const int AesBlockSize = 128; + internal const int BlockSizeInBytes = AesBlockSize / 8; + private const string AesGcmNoPaddingCipher = "AES/GCM/NoPadding"; + + private static readonly SecureRandom s_random = SecureRandom.GetInstance("SHA1PRNG"); + + public static Stream EncryptFile( + string inFile, + PutGetEncryptionMaterial encryptionMaterial, + SFEncryptionMetadata encryptionMetadata, + FileTransferConfiguration transferConfiguration, + byte[] contentAad, + byte[] keyAad + ) + { + using (var fileStream = File.OpenRead(inFile)) + { + return Encrypt(encryptionMaterial, encryptionMetadata, transferConfiguration, fileStream, contentAad, keyAad); + } + } + + public static Stream DecryptFile( + string inFile, + PutGetEncryptionMaterial encryptionMaterial, + SFEncryptionMetadata encryptionMetadata, + FileTransferConfiguration transferConfiguration) + { + using (var fileStream = File.OpenRead(inFile)) + { + return Decrypt(fileStream, encryptionMaterial, encryptionMetadata, transferConfiguration); + } + } + + public static Stream Encrypt( + PutGetEncryptionMaterial encryptionMaterial, + SFEncryptionMetadata encryptionMetadata, // this is output parameter + FileTransferConfiguration fileTransferConfiguration, + Stream inputStream, + byte[] contentAad, + byte[] keyAad) + { + byte[] decodedMasterKey = Convert.FromBase64String(encryptionMaterial.queryStageMasterKey); + int masterKeySize = decodedMasterKey.Length; + // s_logger.Debug($"Master key size : {masterKeySize}"); + + var contentIV = new byte[BlockSizeInBytes]; + var keyIV = new byte[BlockSizeInBytes]; + var fileKeyBytes = new byte[masterKeySize]; // we choose a random fileKey to encrypt it with qsmk key with GCM + s_random.NextBytes(contentIV); + s_random.NextBytes(keyIV); + s_random.NextBytes(fileKeyBytes); + + var encryptedKey = EncryptKey(fileKeyBytes, decodedMasterKey, keyIV, keyAad); + var result = EncryptContent(inputStream, fileKeyBytes, contentIV, contentAad, fileTransferConfiguration); + + MaterialDescriptor matDesc = new MaterialDescriptor + { + smkId = encryptionMaterial.smkId.ToString(), + queryId = encryptionMaterial.queryId, + keySize = (masterKeySize * 8).ToString() + }; + + encryptionMetadata.key = Convert.ToBase64String(encryptedKey); + encryptionMetadata.iv = Convert.ToBase64String(contentIV); + encryptionMetadata.keyIV = Convert.ToBase64String(keyIV); + encryptionMetadata.keyAad = keyAad == null ? null : Convert.ToBase64String(keyAad); + encryptionMetadata.aad = contentAad == null ? null : Convert.ToBase64String(contentAad); + encryptionMetadata.matDesc = Newtonsoft.Json.JsonConvert.SerializeObject(matDesc); + + return result; + } + + public static Stream Decrypt( + Stream inputStream, + PutGetEncryptionMaterial encryptionMaterial, + SFEncryptionMetadata encryptionMetadata, + FileTransferConfiguration fileTransferConfiguration) + { + var decodedMasterKey = Convert.FromBase64String(encryptionMaterial.queryStageMasterKey); + var keyBytes = Convert.FromBase64String(encryptionMetadata.key); + var keyIVBytes = Convert.FromBase64String(encryptionMetadata.keyIV); + var keyAad = encryptionMetadata.keyAad == null ? null : Convert.FromBase64String(encryptionMetadata.keyAad); + var ivBytes = Convert.FromBase64String(encryptionMetadata.iv); + var contentAad = encryptionMetadata.aad == null ? null : Convert.FromBase64String(encryptionMetadata.aad); + var decryptedFileKey = DecryptKey(keyBytes, decodedMasterKey, keyIVBytes, keyAad); + return DecryptContent(inputStream, decryptedFileKey, ivBytes, contentAad, fileTransferConfiguration); + } + + private static byte[] EncryptKey(byte[] fileKeyBytes, byte[] qsmk, byte[] keyIV, byte[] keyAad) + { + var keyCipher = BuildAesGcmNoPaddingCipher(true, qsmk, keyIV, keyAad); + var cipherKeyData = new byte[keyCipher.GetOutputSize(fileKeyBytes.Length)]; + var processLength = keyCipher.ProcessBytes(fileKeyBytes, 0, fileKeyBytes.Length, cipherKeyData, 0); + keyCipher.DoFinal(cipherKeyData, processLength); + return cipherKeyData; + } + + private static Stream EncryptContent(Stream inputStream, byte[] fileKeyBytes, byte[] contentIV, byte[] contentAad, + FileTransferConfiguration transferConfiguration) + { + var contentCipher = BuildAesGcmNoPaddingCipher(true, fileKeyBytes, contentIV, contentAad); + var targetStream = new FileBackedOutputStream(transferConfiguration.MaxBytesInMemory, transferConfiguration.TempDir); + try + { + var cipherStream = new CipherStream(targetStream, null, contentCipher); + byte[] buffer = new byte[transferConfiguration.MaxBytesInMemory]; + int bytesRead; + while ((bytesRead = inputStream.Read(buffer, 0, buffer.Length)) > 0) + { + cipherStream.Write(buffer, 0, bytesRead); + } + + cipherStream.Flush(); // we cannot close or dispose cipherStream because: 1) it would do additional DoFinal resulting in an exception 2) closing cipherStream would close target stream + var mac = contentCipher.DoFinal(); // getting authentication tag for the whole content + targetStream.Write(mac, 0, mac.Length); + return targetStream; + } + catch (Exception) + { + targetStream.Dispose(); + throw; + } + } + + private static byte[] DecryptKey(byte[] fileKey, byte[] qsmk, byte[] keyIV, byte[] keyAad) + { + var keyCipher = BuildAesGcmNoPaddingCipher(false, qsmk, keyIV, keyAad); + var decryptedKeyData = new byte[keyCipher.GetOutputSize(fileKey.Length)]; + var processLength = keyCipher.ProcessBytes(fileKey, 0, fileKey.Length, decryptedKeyData, 0); + keyCipher.DoFinal(decryptedKeyData, processLength); + return decryptedKeyData; + } + + private static Stream DecryptContent(Stream inputStream, byte[] fileKeyBytes, byte[] contentIV, byte[] contentAad, + FileTransferConfiguration transferConfiguration) + { + var contentCipher = BuildAesGcmNoPaddingCipher(false, fileKeyBytes, contentIV, contentAad); + var targetStream = new FileBackedOutputStream(transferConfiguration.MaxBytesInMemory, transferConfiguration.TempDir); + try + { + var cipherStream = new CipherStream(targetStream, null, contentCipher); + byte[] buffer = new byte[transferConfiguration.MaxBytesInMemory]; + int bytesRead; + while ((bytesRead = inputStream.Read(buffer, 0, buffer.Length)) > 0) + { + cipherStream.Write(buffer, 0, bytesRead); + } + + cipherStream.Flush(); // we cannot close or dispose cipherStream because closing cipherStream would close target stream + contentCipher.DoFinal(); // in case of decrypting we ignore the result which has to be empty + return targetStream; + } + catch (Exception) + { + targetStream.Dispose(); + throw; + } + } + + private static IBufferedCipher BuildAesGcmNoPaddingCipher(bool forEncryption, byte[] keyBytes, byte[] initialisationVector, byte[] aadData) + { + var cipher = CipherUtilities.GetCipher(AesGcmNoPaddingCipher); + KeyParameter keyParameter = new KeyParameter(keyBytes); + var keyParameterAead = aadData == null + ? new AeadParameters(keyParameter, AesBlockSize, initialisationVector) + : new AeadParameters(keyParameter, AesBlockSize, initialisationVector, aadData); + cipher.Init(forEncryption, keyParameterAead); + return cipher; + } + } +} diff --git a/Snowflake.Data/Core/FileTransfer/MaterialDescriptor.cs b/Snowflake.Data/Core/FileTransfer/MaterialDescriptor.cs new file mode 100644 index 000000000..e0b352910 --- /dev/null +++ b/Snowflake.Data/Core/FileTransfer/MaterialDescriptor.cs @@ -0,0 +1,11 @@ +namespace Snowflake.Data.Core.FileTransfer +{ + internal class MaterialDescriptor + { + public string smkId { get; set; } + + public string queryId { get; set; } + + public string keySize { get; set; } + } +} diff --git a/Snowflake.Data/Core/FileTransfer/SFFileMetadata.cs b/Snowflake.Data/Core/FileTransfer/SFFileMetadata.cs index 605de0be1..e1647257b 100644 --- a/Snowflake.Data/Core/FileTransfer/SFFileMetadata.cs +++ b/Snowflake.Data/Core/FileTransfer/SFFileMetadata.cs @@ -3,21 +3,28 @@ */ using System; -using System.Collections.Generic; using System.IO; -using System.Text; using static Snowflake.Data.Core.FileTransfer.SFFileCompressionTypes; namespace Snowflake.Data.Core.FileTransfer { public class SFEncryptionMetadata { - /// Initialization vector + /// Initialization vector for file content encryption public string iv { set; get; } /// File key public string key { set; get; } + /// Additional Authentication Data for file content encryption + public string aad { set; get; } + + /// Initialization vector for key encryption + public string keyIV { set; get; } + + /// Additional Authentication Data for key encryption + public string keyAad { set; get; } + /// Encryption material descriptor public string matDesc { set; get; } } @@ -89,7 +96,7 @@ internal class SFFileMetadata /// File message digest (after compression if required) public string sha256Digest { set; get; } - /// Source compression + /// Source compression public SFFileCompressionType sourceCompression { set; get; } /// Target compression @@ -122,9 +129,9 @@ internal class SFFileMetadata // Proxy credentials of the remote storage client. public ProxyCredentials proxyCredentials { get; set; } - + public int MaxBytesInMemory { get; set; } - + internal CommandTypes _operationType; internal string RemoteFileName() @@ -142,7 +149,7 @@ internal class FileTransferConfiguration { private const int OneMegabyteInBytes = 1024 * 1024; - + public string TempDir { get; set; } public int MaxBytesInMemory { get; set; }