From 87de7e2a9a540c03de1ea5669b30ec78e05f95c8 Mon Sep 17 00:00:00 2001 From: Prabhas Kurapati <66924475+prabhask5@users.noreply.github.com> Date: Sat, 2 Dec 2023 02:31:36 -0800 Subject: [PATCH] [Maintenance-2930] Review use of JSON Flattener (#3674) ### Description Implement JsonFlattener helper class as written in #2926 to deprecate the use of the unnecessary JsonFlattener third party module. * Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation) Maintenance * Why these changes are required? The JsonFlattener module was being utilized in only one place for one specific purpose, so these functions can be implemented as part of the OpenSearch codebase instead of importing an unnecessary third party module. * What is the old behavior before changes and new behavior after changes? Hopefully nothing. ### Issues Resolved - #2930 Is this a backport? If so, please add backport PR # and/or commits # No ### Testing Tests checked to make sure functions are not broken. ### Check List - [x] New functionality includes testing - [x] New functionality has been documented - [x] Commits are signed per the DCO using --signoff By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). --------- Signed-off-by: Prabhas Kurapati --- build.gradle | 7 - .../compliance/FieldReadCallback.java | 3 +- .../security/support/JsonFlattener.java | 66 +++ .../security/support/JsonFlattenerTest.java | 404 ++++++++++++++++++ 4 files changed, 471 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/opensearch/security/support/JsonFlattener.java create mode 100644 src/test/java/org/opensearch/security/support/JsonFlattenerTest.java diff --git a/build.gradle b/build.gradle index 125fb5e39e..92de00aa26 100644 --- a/build.gradle +++ b/build.gradle @@ -585,13 +585,6 @@ dependencies { implementation "io.jsonwebtoken:jjwt-api:${jjwt_version}" implementation "io.jsonwebtoken:jjwt-impl:${jjwt_version}" implementation "io.jsonwebtoken:jjwt-jackson:${jjwt_version}" - // JSON flattener - implementation ("com.github.wnameless.json:json-base:2.4.3") { - exclude group: "org.glassfish", module: "jakarta.json" - exclude group: "com.google.code.gson", module: "gson" - exclude group: "org.json", module: "json" - } - implementation 'com.github.wnameless.json:json-flattener:0.16.6' // JSON patch implementation 'com.flipkart.zjsonpatch:zjsonpatch:0.4.14' implementation 'org.apache.commons:commons-collections4:4.4' diff --git a/src/main/java/org/opensearch/security/compliance/FieldReadCallback.java b/src/main/java/org/opensearch/security/compliance/FieldReadCallback.java index 210a198e2e..4cce5bb61f 100644 --- a/src/main/java/org/opensearch/security/compliance/FieldReadCallback.java +++ b/src/main/java/org/opensearch/security/compliance/FieldReadCallback.java @@ -33,11 +33,10 @@ import org.opensearch.security.auditlog.AuditLog; import org.opensearch.security.dlic.rest.support.Utils; import org.opensearch.security.support.HeaderHelper; +import org.opensearch.security.support.JsonFlattener; import org.opensearch.security.support.SourceFieldsContext; import org.opensearch.security.support.WildcardMatcher; -import com.github.wnameless.json.flattener.JsonFlattener; - //TODO We need to deal with caching!! //Currently we disable caching (and realtime requests) when FLS or DLS is applied //Check if we can hook in into the caches diff --git a/src/main/java/org/opensearch/security/support/JsonFlattener.java b/src/main/java/org/opensearch/security/support/JsonFlattener.java new file mode 100644 index 0000000000..ba2819a886 --- /dev/null +++ b/src/main/java/org/opensearch/security/support/JsonFlattener.java @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.security.support; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + +import com.fasterxml.jackson.core.type.TypeReference; + +import org.opensearch.core.common.Strings; +import org.opensearch.security.DefaultObjectMapper; + +public class JsonFlattener { + + public static Map flattenAsMap(String jsonString) { + try { + final TypeReference> typeReference = new TypeReference<>() { + }; + final Map jsonMap = DefaultObjectMapper.objectMapper.readValue(jsonString, typeReference); + final Map flattenMap = new LinkedHashMap<>(); + flattenEntries("", jsonMap.entrySet(), flattenMap); + return flattenMap; + } catch (final IOException ioe) { + throw new IllegalArgumentException("Unparseable json", ioe); + } + } + + private static void flattenEntries(String prefix, final Iterable> entries, final Map result) { + if (!Strings.isNullOrEmpty(prefix)) { + prefix += "."; + } + + for (final Map.Entry e : entries) { + flattenElement(prefix.concat(e.getKey()), e.getValue(), result); + } + } + + @SuppressWarnings("unchecked") + private static void flattenElement(String prefix, final Object source, final Map result) { + if (source instanceof Iterable) { + flattenCollection(prefix, (Iterable) source, result); + } + if (source instanceof Map) { + flattenEntries(prefix, ((Map) source).entrySet(), result); + } + result.put(prefix, source); + } + + private static void flattenCollection(String prefix, final Iterable objects, final Map result) { + int counter = 0; + for (final Object o : objects) { + flattenElement(prefix + "[" + counter + "]", o, result); + counter++; + } + } + +} diff --git a/src/test/java/org/opensearch/security/support/JsonFlattenerTest.java b/src/test/java/org/opensearch/security/support/JsonFlattenerTest.java new file mode 100644 index 0000000000..2880de387b --- /dev/null +++ b/src/test/java/org/opensearch/security/support/JsonFlattenerTest.java @@ -0,0 +1,404 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.security.support; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; + +public class JsonFlattenerTest { + @Test + public void testFlattenAsMapBasic() { + Map flattenedMap1 = JsonFlattener.flattenAsMap("{\"key\": {\"nested\": 1}, \"another.key\": [\"one\", \"two\"] }"); + assertThat(flattenedMap1.keySet(), containsInAnyOrder("key.nested", "key", "another.key[0]", "another.key[1]", "another.key")); + assertThat( + flattenedMap1.values(), + containsInAnyOrder(1, "one", "two", Arrays.asList("one", "two"), Collections.singletonMap("nested", 1)) + ); + Map flattenedMap2 = JsonFlattener.flattenAsMap("{\"a\":1, \"b\":2, \"cn\":{\"c\":[3,4]}}"); + assertThat(flattenedMap2.keySet(), containsInAnyOrder("a", "b", "cn.c[0]", "cn.c[1]", "cn.c", "cn")); + assertThat( + flattenedMap2.values(), + containsInAnyOrder(1, 2, 3, 4, Arrays.asList(3, 4), Collections.singletonMap("c", Arrays.asList(3, 4))) + ); + Map flattenedMap3 = JsonFlattener.flattenAsMap("{}"); + assertThat(flattenedMap3.keySet(), is(empty())); + assertThat(flattenedMap3.values(), is(empty())); + } + + @Test + public void testFlattenAsMapComplex() { + Map flattenedMap1 = JsonFlattener.flattenAsMap("{\n" + // + " \"a\": {\n" + // + " \"b\": 1,\n" + // + " \"c\": null,\n" + // + " \"d\": [\n" + // + " false,\n" + // + " true\n" + // + " ]\n" + // + " },\n" + // + " \"e\": \"f\",\n" + // + " \"g\": 2.30\n" + // + "}"); + assertThat(flattenedMap1.keySet(), containsInAnyOrder("a.b", "a.c", "a.d[0]", "a.d[1]", "a.d", "a", "e", "g")); + HashMap subMap1 = new HashMap<>(); + subMap1.put("b", 1); + subMap1.put("c", null); + subMap1.put("d", Arrays.asList(false, true)); + assertThat(flattenedMap1.values(), containsInAnyOrder(1, null, false, true, Arrays.asList(false, true), subMap1, "f", 2.3)); + Map flattenedMap2 = JsonFlattener.flattenAsMap( + "{\"a\":{\"b\":1,\"c\":null,\"d\":[false,{\"i\":{\"j\":[false,true,\"xy\"]}}]},\"e\":\"f\",\"g\":2.3,\"z\":[]}" + ); + assertThat( + flattenedMap2.keySet(), + containsInAnyOrder( + "a.b", + "a.c", + "a.d[0]", + "a.d[1].i.j[0]", + "a.d[1].i.j[1]", + "a.d[1].i.j[2]", + "a.d[1].i.j", + "a.d[1].i", + "a.d[1]", + "a.d", + "a", + "e", + "g", + "z" + ) + ); + subMap1 = new HashMap<>(); + subMap1.put("b", 1); + subMap1.put("c", null); + subMap1.put( + "d", + Arrays.asList(false, Collections.singletonMap("i", Collections.singletonMap("j", Arrays.asList(false, true, "xy")))) + ); + assertThat( + flattenedMap2.values(), + containsInAnyOrder( + 1, + null, + false, + false, + true, + "xy", + Arrays.asList(false, true, "xy"), + Collections.singletonMap("j", Arrays.asList(false, true, "xy")), + Collections.singletonMap("i", Collections.singletonMap("j", Arrays.asList(false, true, "xy"))), + Arrays.asList(false, Collections.singletonMap("i", Collections.singletonMap("j", Arrays.asList(false, true, "xy")))), + subMap1, + "f", + 2.3, + Collections.emptyList() + ) + ); + Map flattenedMap3 = JsonFlattener.flattenAsMap("{\n" + // + "\t\"glossary\": {\n" + // + "\t\t\"title\": \"example glossary\",\n" + // + "\t\t\"GlossDiv\": {\n" + // + "\t\t\t\"title\": \"S\",\n" + // + "\t\t\t\"GlossList\": {\n" + // + "\t\t\t\t\"GlossEntry\": {\n" + // + "\t\t\t\t\t\"ID\": \"SGML\",\n" + // + "\t\t\t\t\t\"SortAs\": \"SGML\",\n" + // + "\t\t\t\t\t\"GlossTerm\": \"Standard Generalized Markup Language\",\n" + // + "\t\t\t\t\t\"Acronym\": \"SGML\",\n" + // + "\t\t\t\t\t\"Abbrev\": \"ISO 8879:1986\",\n" + // + "\t\t\t\t\t\"GlossDef\": {\n" + // + "\t\t\t\t\t\t\"para\": \"A meta-markup language, used to create markup languages such as DocBook.\",\n" + // + "\t\t\t\t\t\t\"GlossSeeAlso\": [\n" + // + "\t\t\t\t\t\t\t\"GML\",\n" + // + "\t\t\t\t\t\t\t\"XML\"\n" + // + "\t\t\t\t\t\t]\n" + // + "\t\t\t\t\t},\n" + // + "\t\t\t\t\t\"GlossSee\": \"markup\"\n" + // + "\t\t\t\t}\n" + // + "\t\t\t}\n" + // + "\t\t}\n" + // + "\t}\n" + // + "}"); + assertThat( + flattenedMap3.keySet(), + containsInAnyOrder( + "glossary.title", + "glossary.GlossDiv.title", + "glossary.GlossDiv.GlossList.GlossEntry.ID", + "glossary.GlossDiv.GlossList.GlossEntry.SortAs", + "glossary.GlossDiv.GlossList.GlossEntry.GlossTerm", + "glossary.GlossDiv.GlossList.GlossEntry.Acronym", + "glossary.GlossDiv.GlossList.GlossEntry.Abbrev", + "glossary.GlossDiv.GlossList.GlossEntry.GlossDef.para", + "glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso[0]", + "glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso[1]", + "glossary.GlossDiv.GlossList.GlossEntry.GlossDef.GlossSeeAlso", + "glossary.GlossDiv.GlossList.GlossEntry.GlossDef", + "glossary.GlossDiv.GlossList.GlossEntry.GlossSee", + "glossary.GlossDiv.GlossList.GlossEntry", + "glossary.GlossDiv.GlossList", + "glossary.GlossDiv", + "glossary" + ) + ); + assertThat( + flattenedMap3.values(), + containsInAnyOrder( + "example glossary", + "S", + "SGML", + "SGML", + "Standard Generalized Markup Language", + "SGML", + "ISO 8879:1986", + "A meta-markup language, used to create markup languages such as DocBook.", + "GML", + "XML", + Arrays.asList("GML", "XML"), + Map.of( + "para", + "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso", + Arrays.asList("GML", "XML") + ), + "markup", + Map.of( + "ID", + "SGML", + "SortAs", + "SGML", + "GlossTerm", + "Standard Generalized Markup Language", + "Acronym", + "SGML", + "Abbrev", + "ISO 8879:1986", + "GlossDef", + Map.of( + "para", + "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso", + Arrays.asList("GML", "XML") + ), + "GlossSee", + "markup" + ), + Map.of( + "GlossEntry", + Map.of( + "ID", + "SGML", + "SortAs", + "SGML", + "GlossTerm", + "Standard Generalized Markup Language", + "Acronym", + "SGML", + "Abbrev", + "ISO 8879:1986", + "GlossDef", + Map.of( + "para", + "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso", + Arrays.asList("GML", "XML") + ), + "GlossSee", + "markup" + ) + ), + Map.of( + "title", + "S", + "GlossList", + Map.of( + "GlossEntry", + Map.of( + "ID", + "SGML", + "SortAs", + "SGML", + "GlossTerm", + "Standard Generalized Markup Language", + "Acronym", + "SGML", + "Abbrev", + "ISO 8879:1986", + "GlossDef", + Map.of( + "para", + "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso", + Arrays.asList("GML", "XML") + ), + "GlossSee", + "markup" + ) + ) + ), + Map.of( + "title", + "example glossary", + "GlossDiv", + Map.of( + "title", + "S", + "GlossList", + Map.of( + "GlossEntry", + Map.of( + "ID", + "SGML", + "SortAs", + "SGML", + "GlossTerm", + "Standard Generalized Markup Language", + "Acronym", + "SGML", + "Abbrev", + "ISO 8879:1986", + "GlossDef", + Map.of( + "para", + "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso", + Arrays.asList("GML", "XML") + ), + "GlossSee", + "markup" + ) + ) + ) + ) + ) + ); + Map flattenedMap4 = JsonFlattener.flattenAsMap("{\n" + // + "\t\"arrayOfObjects\": [\n" + // + "\t\ttrue,\n" + // + "\t\t{\n" + // + "\t\t\t\"x\": 1,\n" + // + "\t\t\t\"y\": 2,\n" + // + "\t\t\t\"z\": [\n" + // + "\t\t\t\t3,\n" + // + "\t\t\t\t4,\n" + // + "\t\t\t\t5\n" + // + "\t\t\t]\n" + // + "\t\t},\n" + // + "\t\t[\n" + // + "\t\t\t6,\n" + // + "\t\t\t7,\n" + // + "\t\t\t8\n" + // + "\t\t],\n" + // + "\t\t[\n" + // + "\t\t\t[\n" + // + "\t\t\t\t9,\n" + // + "\t\t\t\t10\n" + // + "\t\t\t],\n" + // + "\t\t\t11,\n" + // + "\t\t\t12\n" + // + "\t\t],\n" + // + "\t\tfalse\n" + // + "\t],\n" + // + "\t\"boolean\": true,\n" + // + "\t\"color\": \"#82b92c\",\n" + // + "\t\"null\": null,\n" + // + "\t\"number\": 123,\n" + // + "\t\"object\": {\n" + // + "\t\t\"a\": \"b\",\n" + // + "\t\t\"c\": \"d\",\n" + // + "\t\t\"e\": \"f\"\n" + // + "\t},\n" + // + "\t\"string\": \"Hello World\"\n" + // + "}"); + assertThat( + flattenedMap4.keySet(), + containsInAnyOrder( + "arrayOfObjects[0]", + "arrayOfObjects[1].x", + "arrayOfObjects[1].y", + "arrayOfObjects[1].z[0]", + "arrayOfObjects[1].z[1]", + "arrayOfObjects[1].z[2]", + "arrayOfObjects[1].z", + "arrayOfObjects[1]", + "arrayOfObjects[2][0]", + "arrayOfObjects[2][1]", + "arrayOfObjects[2][2]", + "arrayOfObjects[2]", + "arrayOfObjects[3][0][0]", + "arrayOfObjects[3][0][1]", + "arrayOfObjects[3][0]", + "arrayOfObjects[3][1]", + "arrayOfObjects[3][2]", + "arrayOfObjects[3]", + "arrayOfObjects[4]", + "arrayOfObjects", + "boolean", + "color", + "null", + "number", + "object.a", + "object.c", + "object.e", + "object", + "string" + ) + ); + assertThat( + flattenedMap4.values(), + containsInAnyOrder( + true, + 1, + 2, + 3, + 4, + 5, + Arrays.asList(3, 4, 5), + Map.of("x", 1, "y", 2, "z", Arrays.asList(3, 4, 5)), + 6, + 7, + 8, + Arrays.asList(6, 7, 8), + 9, + 10, + Arrays.asList(9, 10), + 11, + 12, + Arrays.asList(Arrays.asList(9, 10), 11, 12), + false, + Arrays.asList( + true, + Map.of("x", 1, "y", 2, "z", Arrays.asList(3, 4, 5)), + Arrays.asList(6, 7, 8), + Arrays.asList(Arrays.asList(9, 10), 11, 12), + false + ), + true, + "#82b92c", + null, + 123, + "b", + "d", + "f", + Map.of("a", "b", "c", "d", "e", "f"), + "Hello World" + ) + ); + } +}