From d19a8bab517180e55fb17344afa989936411b173 Mon Sep 17 00:00:00 2001 From: Terry Quigley <77437788+terryquigleysas@users.noreply.github.com> Date: Wed, 15 May 2024 16:10:10 +0100 Subject: [PATCH] Configure masking algorithm default (#4336) Signed-off-by: Terry Quigley Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../security/OpenSearchSecurityPlugin.java | 17 +++ .../configuration/DlsFlsFilterLeafReader.java | 15 ++- .../security/configuration/MaskedField.java | 76 ++++------- .../dlic/rest/api/RolesApiAction.java | 7 +- .../security/support/ConfigConstants.java | 1 + .../dlic/dlsfls/CustomFieldMaskedTest.java | 47 +++++++ .../security/dlic/dlsfls/FieldMaskedTest.java | 118 ++++++++++++++++++ 7 files changed, 226 insertions(+), 55 deletions(-) diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java index 6e3c22e695..4a33d685e9 100644 --- a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java +++ b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java @@ -58,6 +58,7 @@ import java.util.stream.Stream; import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.search.QueryCachingPolicy; @@ -430,6 +431,19 @@ public List run() { } } + try { + String maskingAlgorithmDefault = settings.get(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT); + if (StringUtils.isNotEmpty(maskingAlgorithmDefault)) { + MessageDigest.getInstance(maskingAlgorithmDefault); + } + } catch (Exception ex) { + throw new OpenSearchSecurityException( + "JVM does not support algorithm for {}", + ex, + ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT + ); + } + if (!client && !settings.getAsBoolean(ConfigConstants.SECURITY_ALLOW_UNSAFE_DEMOCERTIFICATES, false)) { // check for demo certificates final List files = AccessController.doPrivileged(new PrivilegedAction>() { @@ -1383,6 +1397,9 @@ public List> getSettings() { settings.add( Setting.boolSetting(ConfigConstants.OPENDISTRO_SECURITY_AUDIT_ENABLE_TRANSPORT, true, Property.NodeScope, Property.Filtered) ); + settings.add( + Setting.simpleString(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT, Property.NodeScope, Property.Filtered) + ); final List disabledCategories = new ArrayList(2); disabledCategories.add("AUTHENTICATED"); disabledCategories.add("GRANTED_PRIVILEGES"); diff --git a/src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java b/src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java index ac769e37dd..b09745727f 100644 --- a/src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java +++ b/src/main/java/org/opensearch/security/configuration/DlsFlsFilterLeafReader.java @@ -106,6 +106,7 @@ class DlsFlsFilterLeafReader extends SequentialStoredFieldsLeafReader { private final ShardId shardId; private final boolean maskFields; private final Salt salt; + private final String maskingAlgorithmDefault; private DlsGetEvaluator dge = null; @@ -130,7 +131,8 @@ class DlsFlsFilterLeafReader extends SequentialStoredFieldsLeafReader { this.clusterService = clusterService; this.auditlog = auditlog; this.salt = salt; - this.maskedFieldsMap = MaskedFieldsMap.extractMaskedFields(maskFields, maskedFields, salt); + this.maskingAlgorithmDefault = clusterService.getSettings().get(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT); + this.maskedFieldsMap = MaskedFieldsMap.extractMaskedFields(maskFields, maskedFields, salt, maskingAlgorithmDefault); this.shardId = shardId; flsEnabled = includesExcludes != null && !includesExcludes.isEmpty(); @@ -292,11 +294,16 @@ private MaskedFieldsMap(Map maskedFieldsMap) { this.maskedFieldsMap = maskedFieldsMap; } - public static MaskedFieldsMap extractMaskedFields(boolean maskFields, Set maskedFields, final Salt salt) { + public static MaskedFieldsMap extractMaskedFields( + boolean maskFields, + Set maskedFields, + final Salt salt, + String algorithmDefault + ) { if (maskFields) { return new MaskedFieldsMap( maskedFields.stream() - .map(mf -> new MaskedField(mf, salt)) + .map(mf -> new MaskedField(mf, salt, algorithmDefault)) .collect(ImmutableMap.toImmutableMap(mf -> WildcardMatcher.from(mf.getName()), Function.identity())) ); } else { @@ -1210,7 +1217,7 @@ private MaskedFieldsMap getRuntimeMaskedFieldInfo() { if (maskedEval != null) { final Set mf = maskedFieldsMap.get(maskedEval); if (mf != null && !mf.isEmpty()) { - return MaskedFieldsMap.extractMaskedFields(true, mf, salt); + return MaskedFieldsMap.extractMaskedFields(true, mf, salt, maskingAlgorithmDefault); } } diff --git a/src/main/java/org/opensearch/security/configuration/MaskedField.java b/src/main/java/org/opensearch/security/configuration/MaskedField.java index 2636047568..579b9f476d 100644 --- a/src/main/java/org/opensearch/security/configuration/MaskedField.java +++ b/src/main/java/org/opensearch/security/configuration/MaskedField.java @@ -20,6 +20,7 @@ import java.util.Objects; import com.google.common.base.Splitter; +import org.apache.commons.lang3.StringUtils; import org.apache.lucene.util.BytesRef; import org.bouncycastle.util.encoders.Hex; @@ -31,9 +32,11 @@ public class MaskedField { private String algo = null; private List regexReplacements; private final byte[] defaultSalt; + private final String defaultAlgorithm; - public MaskedField(final String value, final Salt salt) { + public MaskedField(final String value, final Salt salt, final String defaultAlgorithm) { this.defaultSalt = salt.getSalt16(); + this.defaultAlgorithm = defaultAlgorithm; final List tokens = Splitter.on("::").splitToList(Objects.requireNonNull(value)); final int tokenCount = tokens.size(); if (tokenCount == 1) { @@ -57,31 +60,31 @@ public final void isValid() throws Exception { } public byte[] mask(byte[] value) { - if (isDefault()) { - return blake2bHash(value); + if (algo != null) { + return customHash(value, algo); + } else if (regexReplacements != null) { + String cur = new String(value, StandardCharsets.UTF_8); + for (RegexReplacement rr : regexReplacements) { + cur = cur.replaceAll(rr.getRegex(), rr.getReplacement()); + } + return cur.getBytes(StandardCharsets.UTF_8); + } else if (StringUtils.isNotEmpty(defaultAlgorithm)) { + return customHash(value, defaultAlgorithm); } else { - return customHash(value); + return blake2bHash(value); } } public String mask(String value) { - if (isDefault()) { - return blake2bHash(value); - } else { - return customHash(value); - } + return new String(mask(value.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); } public BytesRef mask(BytesRef value) { if (value == null) { return null; } - - if (isDefault()) { - return blake2bHash(value); - } else { - return customHash(value); - } + final BytesRef copy = BytesRef.deepCopyOf(value); + return new BytesRef(mask(copy.bytes)); } public String getName() { @@ -126,6 +129,8 @@ public String toString() { + regexReplacements + ", defaultSalt=" + Arrays.toString(defaultSalt) + + ", defaultAlgorithm=" + + defaultAlgorithm + ", isDefault()=" + isDefault() + "]"; @@ -135,35 +140,15 @@ private boolean isDefault() { return regexReplacements == null && algo == null; } - private byte[] customHash(byte[] in) { - if (algo != null) { - try { - MessageDigest digest = MessageDigest.getInstance(algo); - return Hex.encode(digest.digest(in)); - } catch (NoSuchAlgorithmException e) { - throw new IllegalArgumentException(e); - } - } else if (regexReplacements != null) { - String cur = new String(in, StandardCharsets.UTF_8); - for (RegexReplacement rr : regexReplacements) { - cur = cur.replaceAll(rr.getRegex(), rr.getReplacement()); - } - return cur.getBytes(StandardCharsets.UTF_8); - - } else { - throw new IllegalArgumentException(); + private static byte[] customHash(byte[] in, final String algorithm) { + try { + MessageDigest digest = MessageDigest.getInstance(algorithm); + return Hex.encode(digest.digest(in)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalArgumentException(e); } } - private BytesRef customHash(BytesRef in) { - final BytesRef copy = BytesRef.deepCopyOf(in); - return new BytesRef(customHash(copy.bytes)); - } - - private String customHash(String in) { - return new String(customHash(in.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); - } - private byte[] blake2bHash(byte[] in) { // Salt is passed incorrectly but order of parameters is retained at present to ensure full backwards compatibility // Tracking with https://github.com/opensearch-project/security/issues/4274 @@ -174,15 +159,6 @@ private byte[] blake2bHash(byte[] in) { return Hex.encode(out); } - private BytesRef blake2bHash(BytesRef in) { - final BytesRef copy = BytesRef.deepCopyOf(in); - return new BytesRef(blake2bHash(copy.bytes)); - } - - private String blake2bHash(String in) { - return new String(blake2bHash(in.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); - } - private static class RegexReplacement { private final String regex; private final String replacement; diff --git a/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java b/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java index 8b25fd5702..3af971e208 100644 --- a/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java +++ b/src/main/java/org/opensearch/security/dlic/rest/api/RolesApiAction.java @@ -36,6 +36,7 @@ import org.opensearch.security.dlic.rest.validation.RequestContentValidator.DataType; import org.opensearch.security.dlic.rest.validation.ValidationResult; import org.opensearch.security.securityconf.impl.CType; +import org.opensearch.security.support.ConfigConstants; import org.opensearch.threadpool.ThreadPool; import static org.opensearch.security.dlic.rest.api.RequestHandler.methodNotImplementedHandler; @@ -91,7 +92,11 @@ private ValidationResult validateMaskedFields(final JsonNode content) private Pair validateMaskedFieldSyntax(final JsonNode maskedFieldNode) { try { - new MaskedField(maskedFieldNode.asText(), SALT).isValid(); + new MaskedField( + maskedFieldNode.asText(), + SALT, + validationContext.settings().get(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT) + ).isValid(); } catch (Exception e) { return Pair.of(maskedFieldNode.asText(), e.getMessage()); } diff --git a/src/main/java/org/opensearch/security/support/ConfigConstants.java b/src/main/java/org/opensearch/security/support/ConfigConstants.java index 9c671a80f9..a17678ea80 100644 --- a/src/main/java/org/opensearch/security/support/ConfigConstants.java +++ b/src/main/java/org/opensearch/security/support/ConfigConstants.java @@ -327,6 +327,7 @@ public enum RolesMappingResolution { public static final Boolean SECURITY_SYSTEM_INDICES_PERMISSIONS_DEFAULT = false; public static final String SECURITY_SYSTEM_INDICES_KEY = "plugins.security.system_indices.indices"; public static final List SECURITY_SYSTEM_INDICES_DEFAULT = Collections.emptyList(); + public static final String SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT = "plugins.security.masked_fields.algorithm.default"; public static final String TENANCY_PRIVATE_TENANT_NAME = "private"; public static final String TENANCY_GLOBAL_TENANT_NAME = "global"; diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedTest.java index 226574c588..f699bd6505 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/CustomFieldMaskedTest.java @@ -18,7 +18,9 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.WriteRequest.RefreshPolicy; import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.test.helper.rest.RestHelper.HttpResponse; public class CustomFieldMaskedTest extends AbstractDlsFlsTest { @@ -272,4 +274,49 @@ public void testCustomMaskedGet() throws Exception { Assert.assertTrue(res.getBody().contains("***.100.1.XXX")); Assert.assertTrue(res.getBody().contains("123.123.1.XXX")); } + + @Test + public void testCustomMaskedGetWithClusterDefaultSHA3() throws Exception { + + final Settings settings = Settings.builder().put(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT, "SHA3-224").build(); + setup(settings); + + HttpResponse res; + + Assert.assertEquals( + HttpStatus.SC_OK, + (res = rh.executeGetRequest("/deals/_doc/0?pretty", encodeBasicHeader("admin", "admin"))).getStatusCode() + ); + Assert.assertTrue(res.getBody().contains("\"found\" : true")); + Assert.assertTrue(res.getBody().contains("cust1")); + Assert.assertFalse(res.getBody().contains("cust2")); + Assert.assertTrue(res.getBody().contains("100.100.1.1")); + Assert.assertFalse(res.getBody().contains("100.100.2.2")); + Assert.assertFalse( + res.getBody() + .contains( + "8976994d0491e35f74fcac67ede9c83334a6ad34dae07c176df32f10225f93c5077ddd302c02ddd618b2406b1e4dfe50a727cbc880cfe264c552decf2d224ffc" + ) + ); + Assert.assertFalse(res.getBody().contains("***")); + Assert.assertFalse(res.getBody().contains("XXX")); + + Assert.assertEquals( + HttpStatus.SC_OK, + (res = rh.executeGetRequest("/deals/_doc/0?pretty", encodeBasicHeader("user_masked_custom", "password"))).getStatusCode() + ); + Assert.assertTrue(res.getBody().contains("\"found\" : true")); + Assert.assertFalse(res.getBody().contains("cust1")); + Assert.assertFalse(res.getBody().contains("cust2")); + Assert.assertFalse(res.getBody().contains("100.100.1.1")); + Assert.assertFalse(res.getBody().contains("100.100.2.2")); + Assert.assertTrue( + res.getBody() + .contains( + "8976994d0491e35f74fcac67ede9c83334a6ad34dae07c176df32f10225f93c5077ddd302c02ddd618b2406b1e4dfe50a727cbc880cfe264c552decf2d224ffc" + ) + ); + Assert.assertTrue(res.getBody().contains("***.100.1.XXX")); + Assert.assertTrue(res.getBody().contains("123.123.1.XXX")); + } } diff --git a/src/test/java/org/opensearch/security/dlic/dlsfls/FieldMaskedTest.java b/src/test/java/org/opensearch/security/dlic/dlsfls/FieldMaskedTest.java index e18eae5780..c83388345a 100644 --- a/src/test/java/org/opensearch/security/dlic/dlsfls/FieldMaskedTest.java +++ b/src/test/java/org/opensearch/security/dlic/dlsfls/FieldMaskedTest.java @@ -11,6 +11,7 @@ package org.opensearch.security.dlic.dlsfls; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Test; @@ -18,7 +19,9 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.WriteRequest.RefreshPolicy; import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.security.support.ConfigConstants; import org.opensearch.security.test.helper.rest.RestHelper.HttpResponse; public class FieldMaskedTest extends AbstractDlsFlsTest { @@ -217,6 +220,42 @@ public void testMaskedSearch() throws Exception { } + @Test + public void testMaskedSearchWithClusterDefaultSHA512() throws Exception { + + final Settings settings = Settings.builder().put(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT, "SHA-512").build(); + setup(settings); + + HttpResponse res; + + Assert.assertEquals( + HttpStatus.SC_OK, + (res = rh.executeGetRequest("/deals/_search?pretty&size=100", encodeBasicHeader("admin", "admin"))).getStatusCode() + ); + Assert.assertTrue(res.getBody().contains("\"value\" : 32,\n \"relation")); + Assert.assertTrue(res.getBody().contains("\"failed\" : 0")); + Assert.assertTrue(res.getBody().contains("cust1")); + Assert.assertTrue(res.getBody().contains("cust2")); + Assert.assertTrue(res.getBody().contains("100.100.1.1")); + Assert.assertTrue(res.getBody().contains("100.100.2.2")); + Assert.assertFalse(res.getBody().contains("87873bdb698e5f0f60e0b02b76dad1ec11b2787c628edbc95b7ff0e82274b140")); + Assert.assertFalse(res.getBody().contains(DigestUtils.sha512Hex("100.100.1.1"))); + + Assert.assertEquals( + HttpStatus.SC_OK, + (res = rh.executeGetRequest("/deals/_search?pretty&size=100", encodeBasicHeader("user_masked", "password"))).getStatusCode() + ); + Assert.assertTrue(res.getBody().contains("\"value\" : 32,\n \"relation")); + Assert.assertTrue(res.getBody().contains("\"failed\" : 0")); + Assert.assertTrue(res.getBody().contains("cust1")); + Assert.assertTrue(res.getBody().contains("cust2")); + Assert.assertFalse(res.getBody().contains("100.100.1.1")); + Assert.assertFalse(res.getBody().contains("100.100.2.2")); + Assert.assertFalse(res.getBody().contains("87873bdb698e5f0f60e0b02b76dad1ec11b2787c628edbc95b7ff0e82274b140")); + Assert.assertTrue(res.getBody().contains(DigestUtils.sha512Hex("100.100.1.1"))); + + } + @Test public void testMaskedGet() throws Exception { @@ -247,4 +286,83 @@ public void testMaskedGet() throws Exception { Assert.assertTrue(res.getBody().contains("87873bdb698e5f0f60e0b02b76dad1ec11b2787c628edbc95b7ff0e82274b140")); } + @Test + public void testMaskedGetWithClusterDefaultSHA512() throws Exception { + + final Settings settings = Settings.builder().put(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT, "SHA-512").build(); + setup(settings); + + HttpResponse res; + + Assert.assertEquals( + HttpStatus.SC_OK, + (res = rh.executeGetRequest("/deals/_doc/0?pretty", encodeBasicHeader("admin", "admin"))).getStatusCode() + ); + Assert.assertTrue(res.getBody().contains("\"found\" : true")); + Assert.assertTrue(res.getBody().contains("cust1")); + Assert.assertFalse(res.getBody().contains("cust2")); + Assert.assertTrue(res.getBody().contains("100.100.1.1")); + Assert.assertFalse(res.getBody().contains("100.100.2.2")); + Assert.assertFalse(res.getBody().contains("87873bdb698e5f0f60e0b02b76dad1ec11b2787c628edbc95b7ff0e82274b140")); + Assert.assertFalse(res.getBody().contains(DigestUtils.sha3_224Hex("100.100.1.1"))); + Assert.assertFalse(res.getBody().contains(DigestUtils.sha512Hex("100.100.1.1"))); + + Assert.assertEquals( + HttpStatus.SC_OK, + (res = rh.executeGetRequest("/deals/_doc/0?pretty", encodeBasicHeader("user_masked", "password"))).getStatusCode() + ); + + Assert.assertTrue(res.getBody().contains("\"found\" : true")); + Assert.assertTrue(res.getBody().contains("cust1")); + Assert.assertFalse(res.getBody().contains("cust2")); + Assert.assertFalse(res.getBody().contains("100.100.1.1")); + Assert.assertFalse(res.getBody().contains("100.100.2.2")); + Assert.assertFalse(res.getBody().contains("87873bdb698e5f0f60e0b02b76dad1ec11b2787c628edbc95b7ff0e82274b140")); + Assert.assertFalse(res.getBody().contains(DigestUtils.sha3_224Hex("100.100.1.1"))); + Assert.assertTrue(res.getBody().contains(DigestUtils.sha512Hex("100.100.1.1"))); + } + + @Test + public void testMaskedGetWithClusterDefaultSHA3() throws Exception { + + final Settings settings = Settings.builder().put(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT, "SHA3-224").build(); + setup(settings); + + HttpResponse res; + + Assert.assertEquals( + HttpStatus.SC_OK, + (res = rh.executeGetRequest("/deals/_doc/0?pretty", encodeBasicHeader("admin", "admin"))).getStatusCode() + ); + Assert.assertTrue(res.getBody().contains("\"found\" : true")); + Assert.assertTrue(res.getBody().contains("cust1")); + Assert.assertFalse(res.getBody().contains("cust2")); + Assert.assertTrue(res.getBody().contains("100.100.1.1")); + Assert.assertFalse(res.getBody().contains("100.100.2.2")); + Assert.assertFalse(res.getBody().contains("87873bdb698e5f0f60e0b02b76dad1ec11b2787c628edbc95b7ff0e82274b140")); + Assert.assertFalse(res.getBody().contains(DigestUtils.sha3_224Hex("100.100.1.1"))); + Assert.assertFalse(res.getBody().contains(DigestUtils.sha512Hex("100.100.1.1"))); + + Assert.assertEquals( + HttpStatus.SC_OK, + (res = rh.executeGetRequest("/deals/_doc/0?pretty", encodeBasicHeader("user_masked", "password"))).getStatusCode() + ); + + Assert.assertTrue(res.getBody().contains("\"found\" : true")); + Assert.assertTrue(res.getBody().contains("cust1")); + Assert.assertFalse(res.getBody().contains("cust2")); + Assert.assertFalse(res.getBody().contains("100.100.1.1")); + Assert.assertFalse(res.getBody().contains("100.100.2.2")); + Assert.assertFalse(res.getBody().contains("87873bdb698e5f0f60e0b02b76dad1ec11b2787c628edbc95b7ff0e82274b140")); + Assert.assertTrue(res.getBody().contains(DigestUtils.sha3_224Hex("100.100.1.1"))); + Assert.assertFalse(res.getBody().contains(DigestUtils.sha512Hex("100.100.1.1"))); + } + + @Test(expected = IllegalStateException.class) + public void testMaskedGetClusterDefaultDoesNotExist() throws Exception { + final Settings settings = Settings.builder() + .put(ConfigConstants.SECURITY_MASKED_FIELDS_ALGORITHM_DEFAULT, "SHA6-FORCEFAIL") + .build(); + setup(settings); + } }