From a4349d7fdaaab8ee821dfc7ec3a770f83a55775e Mon Sep 17 00:00:00 2001 From: Krzysztof Nozderko Date: Thu, 6 Jun 2024 15:09:01 +0200 Subject: [PATCH] SNOW-1452613 Update secret detector (#961) ### Description SNOW-1452613 Update secret detector ### Checklist - [x] Code compiles correctly - [x] Code is formatted according to [Coding Conventions](../blob/master/CodingConventions.md) - [x] Created tests which fail without the change (if possible) - [x] All tests passing (`dotnet test`) - [x] Extended the README / documentation, if necessary - [x] Provide JIRA issue id (if possible) or GitHub issue id in PR name --- .pre-commit-config.yaml | 2 +- .../UnitTests/SecretDetectorTest.cs | 126 +++++++++++++----- Snowflake.Data/Logger/SecretDetector.cs | 102 +++++++++----- 3 files changed, 167 insertions(+), 63 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 895581a9c..c2af46b46 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ repos: - repo: git@github.com:snowflakedb/casec_precommit.git - rev: v1.20 + rev: v1.35.4 hooks: - id: secret-scanner diff --git a/Snowflake.Data.Tests/UnitTests/SecretDetectorTest.cs b/Snowflake.Data.Tests/UnitTests/SecretDetectorTest.cs index 862a7c248..82c59a63c 100644 --- a/Snowflake.Data.Tests/UnitTests/SecretDetectorTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SecretDetectorTest.cs @@ -1,16 +1,15 @@ /* - * Copyright (c) 2021 Snowflake Computing Inc. All rights reserved. + * Copyright (c) 2021-2024 Snowflake Computing Inc. All rights reserved. */ -using Amazon.S3.Model.Internal.MarshallTransformations; +using NUnit.Framework; +using Snowflake.Data.Log; +using Snowflake.Data.Tests.Mock; +using System; +using System.Text; namespace Snowflake.Data.Tests.UnitTests { - using NUnit.Framework; - using Snowflake.Data.Log; - using Snowflake.Data.Tests.Mock; - using System; - using System.Collections.Generic; [TestFixture] class SecretDetectorTest @@ -95,7 +94,7 @@ public void TestAWSKeys() BasicMasking(@"""aws_key_id""='aaaaaaaa'", @"""aws_key_id""='****'"); //aws_key_id|aws_secret_key|access_key_id|secret_access_key)('|"")?(\s*[:|=]\s*)'([^']+)' - // Delimiters before start of value to mask + // Delimiters before start of value to mask BasicMasking(@"aws_key_id:'aaaaaaaa'", @"aws_key_id:'****'"); BasicMasking(@"aws_key_id='aaaaaaaa'", @"aws_key_id='****'"); } @@ -144,7 +143,7 @@ public void TestSASTokens() BasicMasking(@"sig=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", @"sig=****"); // signature - BasicMasking(@"signature=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", @"signature=****"); + BasicMasking(@"signature=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", @"signature=****"); // AWSAccessKeyId BasicMasking(@"AWSAccessKeyId=ABCDEFGHIJKL01234", @"AWSAccessKeyId=****"); // pragma: allowlist secret @@ -167,6 +166,32 @@ public void TestPrivateKey() "-----BEGIN PRIVATE KEY-----\\\\nXXXX\\\\n-----END PRIVATE KEY-----"); // pragma: allowlist secret } + [Test] + public void TestPrivateKeyProperty() + { + BasicMasking(@"something=anything;private_key=aaaaaa", @"something=anything;private_key=****"); + BasicMasking("something=anything;private_key \r\n =aaaaaa", "something=anything;private_key \r\n =****"); + BasicMasking(@"something=anything;private_key=aaaaaaaaaaaaaaaaaa", @"something=anything;private_key=****"); + BasicMasking(@"something=anything;private_key=a", @"something=anything;private_key=****"); + BasicMasking(@"something=anything;private_key=""a"";someOtherProperty=someValue", @"something=anything;private_key=****"); + BasicMasking(@"something=anything;private_key='a';someOtherProperty=someValue", @"something=anything;private_key=****"); + BasicMasking($"something=anything;private_key ={GetStringWithManyWeirdCharacters()}\r\nxxxxxx\r\nyyyyyy;someOtherProperty=someValue", @"something=anything;private_key =****"); + } + + private string GetStringWithManyWeirdCharacters() + { + var bytes = new byte[256]; + for (var i = 0; i < 256; i++) + { + if (i < 20) + { + bytes[i] = 58; + } + bytes[i] = (byte) i; + } + return Encoding.Default.GetString(bytes); + } + [Test] public void TestPrivateKeyData() { @@ -185,12 +210,12 @@ public void TestConnectionTokens() // assertion content BasicMasking(@"assertion content:aaaaaaaa", @"assertion content:****"); - // Delimiters before start of value to mask + // Delimiters before start of value to mask BasicMasking(@"token""aaaaaaaa", @"token""****"); // " BasicMasking(@"token'aaaaaaaa", @"token'****"); // ' BasicMasking(@"token=aaaaaaaa", @"token=****"); // = BasicMasking(@"token aaaaaaaa", @"token ****"); // {space} - BasicMasking(@"token ="" 'aaaaaaaa", @"token ="" '****"); // Mix + BasicMasking(@"token ="" 'aaaaaaaa", @"token =****"); // Mix // Verify that all allowed characters are correctly supported BasicMasking(@"Token:a=b/c_d-e+F:025", @"Token:****"); @@ -211,17 +236,57 @@ public void TestPassword() // passcode BasicMasking(@"passcode:aaaaaaaa", @"passcode:****"); - // Delimiters before start of value to mask + // Delimiters before start of value to mask BasicMasking(@"password""aaaaaaaa", @"password""****"); // " BasicMasking(@"password'aaaaaaaa", @"password'****"); // ' BasicMasking(@"password=aaaaaaaa", @"password=****"); // = BasicMasking(@"password aaaaaaaa", @"password ****"); // {space} - BasicMasking(@"password ="" 'aaaaaaaa", @"password ="" '****"); // Mix + BasicMasking(@"password ="" 'aaaaaaaa", @"password =****"); // Mix // Verify that all allowed characters are correctly supported BasicMasking(@"password:a!b""c#d$e%f&g'h(i)k*k+l,m;nq?r@s[t]u^v_w`x{y|z}Az0123", @"password:****"); } + [Test] + public void TestPasswordProperty() + { + BasicMasking(@"somethingBefore=cccc;password=aa", @"somethingBefore=cccc;password=****"); + BasicMasking(@"somethingBefore=cccc;password=aa;somethingNext=bbbb", @"somethingBefore=cccc;password=****"); + BasicMasking(@"somethingBefore=cccc;password=""aa"";somethingNext=bbbb", @"somethingBefore=cccc;password=****"); + BasicMasking(@"somethingBefore=cccc;password=;somethingNext=bbbb", @"somethingBefore=cccc;password=****"); + BasicMasking(@"somethingBefore=cccc;password=", @"somethingBefore=cccc;password=****"); + BasicMasking(@"somethingBefore=cccc;password =aa;somethingNext=bbbb", @"somethingBefore=cccc;password =****"); + BasicMasking(@"somethingBefore=cccc;password="" 'aa", @"somethingBefore=cccc;password=****"); + + BasicMasking(@"somethingBefore=cccc;proxypassword=aa", @"somethingBefore=cccc;proxypassword=****"); + BasicMasking(@"somethingBefore=cccc;proxypassword=aa;somethingNext=bbbb", @"somethingBefore=cccc;proxypassword=****"); + BasicMasking(@"somethingBefore=cccc;proxypassword=""aa"";somethingNext=bbbb", @"somethingBefore=cccc;proxypassword=****"); + BasicMasking(@"somethingBefore=cccc;proxypassword=;somethingNext=bbbb", @"somethingBefore=cccc;proxypassword=****"); + BasicMasking(@"somethingBefore=cccc;proxypassword=", @"somethingBefore=cccc;proxypassword=****"); + BasicMasking(@"somethingBefore=cccc;proxypassword =aa;somethingNext=bbbb", @"somethingBefore=cccc;proxypassword =****"); + BasicMasking(@"somethingBefore=cccc;proxypassword="" 'aa", @"somethingBefore=cccc;proxypassword=****"); + + BasicMasking(@"somethingBefore=cccc;private_key_pwd=aa", @"somethingBefore=cccc;private_key_pwd=****"); + BasicMasking(@"somethingBefore=cccc;private_key_pwd=aa;somethingNext=bbbb", @"somethingBefore=cccc;private_key_pwd=****"); + BasicMasking(@"somethingBefore=cccc;private_key_pwd=""aa"";somethingNext=bbbb", @"somethingBefore=cccc;private_key_pwd=****"); + BasicMasking(@"somethingBefore=cccc;private_key_pwd=;somethingNext=bbbb", @"somethingBefore=cccc;private_key_pwd=****"); + BasicMasking(@"somethingBefore=cccc;private_key_pwd=", @"somethingBefore=cccc;private_key_pwd=****"); + BasicMasking(@"somethingBefore=cccc;private_key_pwd =aa;somethingNext=bbbb", @"somethingBefore=cccc;private_key_pwd =****"); + BasicMasking(@"somethingBefore=cccc;private_key_pwd="" 'aa", @"somethingBefore=cccc;private_key_pwd=****"); + } + + [Test] + [TestCase("2020-04-30 23:06:04,069 - MainThread auth.py:397 - write_temporary_credential() - DEBUG - no ID password was not given")] + [TestCase("2020-04-30 23:06:04,069 - MainThread auth.py:397 - write_temporary_credential() - DEBUG - no ID proxyPassword was not given")] + [TestCase("2020-04-30 23:06:04,069 - MainThread auth.py:397 - write_temporary_credential() - DEBUG - no ID private_key_pwd was not given")] + public void TestPasswordFalsePositive(string falsePositiveMessage) + { + mask = SecretDetector.MaskSecrets(falsePositiveMessage); + Assert.IsFalse(mask.isMasked); + Assert.AreEqual(falsePositiveMessage, mask.maskedText); + Assert.IsNull(mask.errStr); + } + [Test] public void TestMaskToken() { @@ -268,7 +333,7 @@ public void TestMaskToken() string snowFlakeAuthToken = "Authorization: Snowflake Token=\"ver:1-hint:92019676298218-ETMsDgAAAXswwgJhABRBRVMvQ0JDL1BLQ1M1UGFkZGluZwEAABAAEF1tbNM3myWX6A9sNSK6rpIAAACA6StojDJS4q1Vi3ID+dtFEucCEvGMOte0eapK+reb39O6hTHYxLfOgSGsbvbM5grJ4dYdNJjrzDf1r07tID4I2RJJRYjS4/DWBJn98Untd3xeNnXE1/45HgvwKVHlmZQLVwfWAxI7ifl2MVDwJlcXBufLZoVMYhUd4np121d7zFwAFGQzKyzUYQwI3M9Nqja9syHgaotG\""; mask = SecretDetector.MaskSecrets(snowFlakeAuthToken); Assert.IsTrue(mask.isMasked); - Assert.AreEqual(@"Authorization: Snowflake Token=""****""", mask.maskedText); + Assert.AreEqual(@"Authorization: Snowflake Token=****", mask.maskedText); Assert.IsNull(mask.errStr); } @@ -311,7 +376,7 @@ public void TestPasswords() string randomPasswordEqualSign = "password = " + randomPassword; mask = SecretDetector.MaskSecrets(randomPasswordEqualSign); Assert.IsTrue(mask.isMasked); - Assert.AreEqual(@"password = ****", mask.maskedText); + Assert.AreEqual(@"password =****", mask.maskedText); Assert.IsNull(mask.errStr); string randomPwdWithPrefix = "pwd:" + randomPassword; @@ -350,9 +415,7 @@ public void TestTokenPassword() mask = SecretDetector.MaskSecrets(testStringWithPrefix); Assert.IsTrue(mask.isMasked); Assert.AreEqual( - "token=****" + - " random giberish " + - "password:****", + "token=****", mask.maskedText); Assert.IsNull(mask.errStr); @@ -378,11 +441,7 @@ public void TestTokenPassword() mask = SecretDetector.MaskSecrets(testStringWithPrefix); Assert.IsTrue(mask.isMasked); Assert.AreEqual( - "token=****" + - " random giberish " + - "password:****" + - " random giberish " + - "idToken:****", + "token=****", mask.maskedText); Assert.IsNull(mask.errStr); @@ -393,10 +452,7 @@ public void TestTokenPassword() mask = SecretDetector.MaskSecrets(testStringWithPrefix); Assert.IsTrue(mask.isMasked); Assert.AreEqual( - "password=****" + - " random giberish " + - "pwd:****", - mask.maskedText); + "password=****", mask.maskedText); Assert.IsNull(mask.errStr); // multiple passwords @@ -408,15 +464,23 @@ public void TestTokenPassword() mask = SecretDetector.MaskSecrets(testStringWithPrefix); Assert.IsTrue(mask.isMasked); Assert.AreEqual( - "password=****" + - " random giberish " + - "password=****" + - " random giberish " + "password=****", mask.maskedText); Assert.IsNull(mask.errStr); } + [Test] + public void TestTokenProperty() + { + BasicMasking(@"somethingBefore=cccc;token=aa", @"somethingBefore=cccc;token=****"); + BasicMasking(@"somethingBefore=cccc;token=aa;somethingNext=bbbb", @"somethingBefore=cccc;token=****"); + BasicMasking(@"somethingBefore=cccc;token=""aa"";somethingNext=bbbb", @"somethingBefore=cccc;token=****"); + BasicMasking(@"somethingBefore=cccc;token=;somethingNext=bbbb", @"somethingBefore=cccc;token=****"); + BasicMasking(@"somethingBefore=cccc;token=", @"somethingBefore=cccc;token=****"); + BasicMasking(@"somethingBefore=cccc;token =aa;somethingNext=bbbb", @"somethingBefore=cccc;token =****"); + BasicMasking(@"somethingBefore=cccc;token="" 'aa", @"somethingBefore=cccc;token=****"); + } + [Test] public void TestCustomPattern() { diff --git a/Snowflake.Data/Logger/SecretDetector.cs b/Snowflake.Data/Logger/SecretDetector.cs index 2c0524fb1..59cd810d6 100644 --- a/Snowflake.Data/Logger/SecretDetector.cs +++ b/Snowflake.Data/Logger/SecretDetector.cs @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Snowflake Computing Inc. All rights reserved. + * Copyright (c) 2021-2024 Snowflake Computing Inc. All rights reserved. */ using System; @@ -76,69 +76,107 @@ private static string MaskCustomPatterns(string text) /* * https://docs.microsoft.com/en-us/dotnet/standard/base-types/character-escapes-in-regular-expressions * . $ ^ { [ ( | ) * + ? \ - * The characters are special regular expression language elements. + * The characters are special regular expression language elements. * To match them in a regular expression, they must be escaped or included in a positive character group. * [ ] \ - ^ * The characters are special character group element. * To match them in a character group, they must be escaped. */ - private static readonly string AWS_KEY_PATTERN = @"(aws_key_id|aws_secret_key|access_key_id|secret_access_key)('|"")?(\s*[:=]\s*)'([^']+)'"; - private static readonly string AWS_TOKEN_PATTERN = @"(accessToken|tempToken|keySecret)\""\s*:\s*\""([a-z0-9/+]{32,}={0,2})\"""; - private static readonly string AWS_SERVER_SIDE_PATTERN = @"((x-amz-server-side-encryption)([a-z0-9\-])*)\s*(:|=)\s*([a-z0-9/_\-+:=])+"; - private static readonly string SAS_TOKEN_PATTERN = @"(sig|signature|AWSAccessKeyId|password|passcode)=([a-z0-9%/+]{16,})"; - private static readonly string PRIVATE_KEY_PATTERN = @"-----BEGIN PRIVATE KEY-----\n([a-z0-9/+=\n]{32,})\n-----END PRIVATE KEY-----"; - private static readonly string PRIVATE_KEY_DATA_PATTERN = @"""privateKeyData"": ""([a-z0-9/+=\n]{10,})"""; - private static readonly string CONNECTION_TOKEN_PATTERN = @"(token|assertion content)(['""\s:=]+)([a-z0-9=/_\-+:]{8,})"; - private static readonly string PASSWORD_PATTERN = @"(password|passcode|pwd|proxypassword)(['""\s:=]+)([a-z0-9!""#$%&'\()*+,-./:;<=>?@\[\]\^_`{|}~]{6,})"; + private const string AwsKeyPattern = @"(aws_key_id|aws_secret_key|access_key_id|secret_access_key)('|"")?(\s*[:=]\s*)'([^']+)'"; + private const string AwsTokenPattern = @"(accessToken|tempToken|keySecret)\""\s*:\s*\""([a-z0-9/+]{32,}={0,2})\"""; + private const string AwsServerSidePattern = @"((x-amz-server-side-encryption)([a-z0-9\-])*)\s*(:|=)\s*([a-z0-9/_\-+:=])+"; + private const string SasTokenPattern = @"(sig|signature|AWSAccessKeyId|password|passcode)=([a-z0-9%/+]{16,})"; + private const string PrivateKeyPattern = @"-----BEGIN PRIVATE KEY-----\n([a-z0-9/+=\n]{32,})\n-----END PRIVATE KEY-----"; // pragma: allowlist secret + private const string PrivateKeyDataPattern = @"""privateKeyData"": ""([a-z0-9/+=\n]{10,})"""; + private const string PrivateKeyPropertyPrefixPattern = @"(private_key\s*=)"; + private const string ConnectionTokenPattern = @"(token|assertion content)(['""\s:=]+)([a-z0-9=/_\-+:]{8,})"; + private const string TokenPropertyPattern = @"(token)(\s*=)(.*)"; + private const string PasswordPattern = @"(password|passcode|pwd|proxypassword|private_key_pwd)(['""\s:=]+)([a-z0-9!""#$%&'\()*+,-./:;<=>?@\[\]\^_`{|}~]{6,})"; + private const string PasswordPropertyPattern = @"(password|proxypassword|private_key_pwd)(\s*=)(.*)"; + + private static readonly Func[] s_maskFunctions = { + MaskAWSServerSide, + MaskAWSKeys, + MaskSASTokens, + MaskAWSTokens, + MaskPrivateKey, + MaskPrivateKeyData, + MaskPrivateKeyProperty, + MaskPassword, + MaskPasswordProperty, + MaskConnectionTokens, + MaskTokenProperty + }; private static string MaskAWSKeys(string text) { - return Regex.Replace(text, AWS_KEY_PATTERN, @"$1$2$3'****'", + return Regex.Replace(text, AwsKeyPattern, @"$1$2$3'****'", RegexOptions.IgnoreCase); } private static string MaskAWSTokens(string text) { - return Regex.Replace(text, AWS_TOKEN_PATTERN, @"$1"":""XXXX""", + return Regex.Replace(text, AwsTokenPattern, @"$1"":""XXXX""", RegexOptions.IgnoreCase); } private static string MaskAWSServerSide(string text) { - return Regex.Replace(text, AWS_SERVER_SIDE_PATTERN, @"$1:....", + return Regex.Replace(text, AwsServerSidePattern, @"$1:....", RegexOptions.IgnoreCase); } private static string MaskSASTokens(string text) { - return Regex.Replace(text, SAS_TOKEN_PATTERN, @"$1=****", + return Regex.Replace(text, SasTokenPattern, @"$1=****", RegexOptions.IgnoreCase); } private static string MaskPrivateKey(string text) { - return Regex.Replace(text, PRIVATE_KEY_PATTERN, "-----BEGIN PRIVATE KEY-----\\\\nXXXX\\\\n-----END PRIVATE KEY-----", + return Regex.Replace(text, PrivateKeyPattern, "-----BEGIN PRIVATE KEY-----\\\\nXXXX\\\\n-----END PRIVATE KEY-----", // pragma: allowlist secret RegexOptions.IgnoreCase | RegexOptions.Multiline); } + private static string MaskPrivateKeyProperty(string text) + { + var match = Regex.Match(text, PrivateKeyPropertyPrefixPattern, RegexOptions.IgnoreCase); + if (match.Success) + { + int length = match.Index + match.Value.Length; + return text.Substring(0, length) + "****"; + } + return text; + } + private static string MaskPrivateKeyData(string text) { - return Regex.Replace(text, PRIVATE_KEY_DATA_PATTERN, @"""privateKeyData"": ""XXXX""", + return Regex.Replace(text, PrivateKeyDataPattern, @"""privateKeyData"": ""XXXX""", RegexOptions.IgnoreCase | RegexOptions.Multiline); } private static string MaskConnectionTokens(string text) { - return Regex.Replace(text, CONNECTION_TOKEN_PATTERN, @"$1$2****", + return Regex.Replace(text, ConnectionTokenPattern, @"$1$2****", RegexOptions.IgnoreCase); } private static string MaskPassword(string text) { - return Regex.Replace(text, PASSWORD_PATTERN, @"$1$2****", + return Regex.Replace(text, PasswordPattern, @"$1$2****", RegexOptions.IgnoreCase); } + private static string MaskPasswordProperty(string text) + { + return Regex.Replace(text, PasswordPropertyPattern, @"$1$2****", RegexOptions.IgnoreCase); + } + + private static string MaskTokenProperty(string text) + { + return Regex.Replace(text, TokenPropertyPattern, @"$1$2****", RegexOptions.IgnoreCase); + } + public static Mask MaskSecrets(string text) { Mask result = new Mask(maskedText: text); @@ -150,19 +188,7 @@ public static Mask MaskSecrets(string text) try { - result.maskedText = - MaskConnectionTokens( - MaskPassword( - MaskPrivateKeyData( - MaskPrivateKey( - MaskAWSTokens( - MaskSASTokens( - MaskAWSKeys( - MaskAWSServerSide(text)))))))); - if (CUSTOM_PATTERNS_LENGTH > 0) - { - result.maskedText = MaskCustomPatterns(result.maskedText); - } + result.maskedText = MaskAllPatterns(text); if (result.maskedText != text) { result.isMasked = true; @@ -179,5 +205,19 @@ public static Mask MaskSecrets(string text) } return result; } + + private static string MaskAllPatterns(string text) + { + string result = text; + foreach (var maskFunction in s_maskFunctions) + { + result = maskFunction.Invoke(result); + } + if (CUSTOM_PATTERNS_LENGTH > 0) + { + result = MaskCustomPatterns(result); + } + return result; + } } }