Bug fixes and excluding non-serializable fields from Schema Attributes

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.0.1-RC8]
+
+- SchemaDetails: 
+  - Moved the nested `SchemaKey` attributes to the class level
+  - made the getReferenceId() uniform with `SchemaKey` implementation 
+- FieldUtils: Excluding non-serializable fields(static, transient, @JsonIgnore) for building the schema attributes
+- SchemaResource: Bug fix in SchemaValidation resource
+- Removed the unused `LeiaCompiledPath` class
+
 ## [0.0.1-RC7]
 
 - Introduced a `leia-common` module to host all the common utils classes

CONTRIBUTORS.md

@@ -0,0 +1,6 @@
+### Contributors
+
+* [Koushik R](https://github.com/koushikr)
+* [Abhishek](https://github.com/abhigun)
+* [Subham](https://github.com/nullsoni)
+

README.md

@@ -26,7 +26,7 @@ Leia is a governance and metadata framework aimed at meeting compliance requirem
 <dependency>
     <groupId>com.grookage.leia</groupId>
     <artifactId>leia-bom</artifactId>
-    <versio>0.0.1-RC7</version>
+    <version>latest</version>
 </dependency>
 ```
 

leia-bom/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>com.grookage.leia</groupId>
         <artifactId>leia</artifactId>
-        <version>0.0.1-RC7</version>
+        <version>0.0.1-RC8</version>
     </parent>
 
     <artifactId>leia-bom</artifactId>

leia-client-dropwizard/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>com.grookage.leia</groupId>
         <artifactId>leia-parent</artifactId>
-        <version>0.0.1-RC7</version>
+        <version>0.0.1-RC8</version>
         <relativePath>../leia-parent</relativePath>
     </parent>
 

leia-client/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>com.grookage.leia</groupId>
         <artifactId>leia-parent</artifactId>
-        <version>0.0.1-RC7</version>
+        <version>0.0.1-RC8</version>
         <relativePath>../leia-parent</relativePath>
     </parent>
 

leia-client/src/main/java/com/grookage/leia/client/AbstractSchemaClient.java

@@ -46,7 +46,7 @@ public List<SchemaDetails> getSchemaDetails(final Set<String> namespaces) {
             throw new IllegalStateException("The configuration object has returned null data. Something gone wrong with refresher");
         }
         return refresher.getData().stream()
-                .filter(each -> namespaces.contains(each.getSchemaKey().getNamespace())).toList();
+                .filter(each -> namespaces.contains(each.getNamespace())).toList();
     }
 
     public boolean valid(SchemaKey schemaKey) {

leia-client/src/main/java/com/grookage/leia/client/LeiaMessageProduceClient.java

@@ -88,7 +88,7 @@ private Map<SchemaKey, LeiaMessage> getMessages(SchemaKey schemaKey, byte[] sour
                 .build()
         );
         final var sourceSchemaDetails = super.getSchemaDetails()
-                .stream().filter(each -> each.getSchemaKey().equals(schemaKey))
+                .stream().filter(each -> each.match(schemaKey))
                 .findFirst().orElse(null);
 
         final var transformationTargets = null == sourceSchemaDetails ? null :
@@ -112,10 +112,11 @@ public void processMessages(final SchemaKey schemaKey,
     @Override
     public void start() {
         super.getSchemaDetails().forEach(schemaDetails -> {
-            final var validSchema = super.valid(schemaDetails.getSchemaKey());
+            final var schemaKey = schemaDetails.getSchemaKey();
+            final var validSchema = super.valid(schemaKey);
             if (!validSchema) {
                 log.error("The source schema doesn't seem to be valid for schemaKey {}. Please check the schema bindings provided",
-                        schemaDetails.getSchemaKey());
+                        schemaKey);
                 throw new IllegalStateException("Invalid source schema");
             }
             final var transformationTargets = schemaDetails.getTransformationTargets();

leia-common/pom.xml

@@ -23,7 +23,7 @@
     <parent>
         <groupId>com.grookage.leia</groupId>
         <artifactId>leia-parent</artifactId>
-        <version>0.0.1-RC7</version>
+        <version>0.0.1-RC8</version>
         <relativePath>../leia-parent</relativePath>
     </parent>
 

leia-common/src/main/java/com/grookage/leia/common/utils/FieldUtils.java

@@ -16,9 +16,11 @@
 
 package com.grookage.leia.common.utils;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.experimental.UtilityClass;
 
 import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -28,8 +30,18 @@ public class FieldUtils {
     public List<Field> getAllFields(final Class<?> type) {
         List<Field> fields = new ArrayList<>();
         for (Class<?> c = type; c != null; c = c.getSuperclass()) {
-            fields.addAll(Arrays.asList(c.getDeclaredFields()));
+            Arrays.stream(c.getDeclaredFields())
+                    .filter(field -> !isNonSerializable(field))
+                    .forEach(fields::add);
         }
         return fields;
     }
+
+    private boolean isNonSerializable(final Field field) {
+        if (field.isAnnotationPresent(JsonIgnore.class)) {
+            return true;
+        }
+        int modifiers = field.getModifiers();
+        return Modifier.isStatic(modifiers) || Modifier.isTransient(modifiers);
+    }
 }

leia-common/src/test/java/com/grookage/leia/common/builder/SchemaBuilderTest.java

@@ -11,18 +11,21 @@
 import com.grookage.leia.models.qualifiers.PIIQualifier;
 import com.grookage.leia.models.schema.SchemaType;
 import com.grookage.leia.models.schema.SchemaValidationType;
+import lombok.SneakyThrows;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.util.HashSet;
 import java.util.Set;
 
 class SchemaBuilderTest {
+    @SneakyThrows
     @Test
     void testSchemaRequest() {
         final var schemaCreateRequest = SchemaBuilder.buildSchemaRequest(TestRecord.class)
                 .orElse(null);
         Assertions.assertNotNull(schemaCreateRequest);
+        Assertions.assertEquals(5, schemaCreateRequest.getAttributes().size());
         final var schemaAttributes = SchemaBuilder.getSchemaAttributes(TestRecord.class);
         Assertions.assertEquals(TestRecord.NAME, schemaCreateRequest.getSchemaName());
         Assertions.assertEquals(TestRecord.NAMESPACE, schemaCreateRequest.getNamespace());

leia-common/src/test/java/com/grookage/leia/common/utils/FieldUtilsTest.java

@@ -0,0 +1,29 @@
+package com.grookage.leia.common.utils;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class FieldUtilsTest {
+    @Test
+    void testFieldUtils() {
+        final var fields = FieldUtils.getAllFields(TestData.class);
+        Assertions.assertFalse(fields.isEmpty());
+        Assertions.assertEquals(2, fields.size());
+        Assertions.assertTrue(fields.stream().noneMatch(field -> field.getName().equals("phoneNumber")));
+        Assertions.assertTrue(fields.stream().noneMatch(field -> field.getName().equals("CONSTANT")));
+        Assertions.assertTrue(fields.stream().noneMatch(field -> field.getName().equals("exclusion")));
+    }
+
+    static class BaseData {
+        @JsonIgnore
+        private String exclusion;
+        static final String CONSTANT = "CONSTANT";
+        String name;
+    }
+
+    static class TestData extends BaseData {
+        String email;
+        transient String phoneNumber;
+    }
+}

leia-common/src/test/resources/validNestedSchema.json

@@ -1,9 +1,7 @@
 {
-  "schemaKey": {
-    "namespace": "testNamespace",
-    "schemaName": "testSchema",
-    "version": "V1234"
-  },
+  "namespace": "testNamespace",
+  "schemaName": "testSchema",
+  "version": "V1234",
   "schemaState": "CREATED",
   "schemaType": "JSON",
   "schemaMeta": {

leia-common/src/test/resources/validSchema.json

@@ -1,9 +1,7 @@
 {
-  "schemaKey": {
-    "namespace": "testNamespace",
-    "schemaName": "testSchema",
-    "version": "V1234"
-  },
+  "namespace": "testNamespace",
+  "schemaName": "testSchema",
+  "version": "V1234",
   "schemaState": "CREATED",
   "schemaType": "JSON",
   "schemaMeta": {

leia-core/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>com.grookage.leia</groupId>
         <artifactId>leia-parent</artifactId>
-        <version>0.0.1-RC7</version>
+        <version>0.0.1-RC8</version>
         <relativePath>../leia-parent</relativePath>
     </parent>
 

leia-core/src/main/java/com/grookage/leia/core/ingestion/utils/SchemaUtils.java

@@ -34,12 +34,9 @@ public SchemaDetails toSchemaDetails(final CreateSchemaRequest createSchemaReque
                                          final String email,
                                          final Supplier<VersionIDGenerator> versionSupplier) {
         return SchemaDetails.builder()
-                .schemaKey(SchemaKey.builder()
-                        .namespace(createSchemaRequest.getNamespace())
-                        .schemaName(createSchemaRequest.getSchemaName())
-                        .version(versionSupplier.get().generateVersionId("V"))
-                        .build()
-                )
+                .namespace(createSchemaRequest.getNamespace())
+                .schemaName(createSchemaRequest.getSchemaName())
+                .version(versionSupplier.get().generateVersionId("V"))
                 .schemaState(SchemaState.CREATED)
                 .schemaType(createSchemaRequest.getSchemaType())
                 .description(createSchemaRequest.getDescription())

leia-core/src/test/java/com/grookage/leia/core/ingestion/utils/SchemaUtilsTest.java

@@ -43,11 +43,9 @@ public String generateVersionId(String prefix) {
                 "testEmail", () -> generator
         );
         Assertions.assertNotNull(schemaDetails);
-        final var schemaKey = schemaDetails.getSchemaKey();
-        Assertions.assertNotNull(schemaKey);
-        Assertions.assertEquals("testNamespace", schemaKey.getNamespace());
-        Assertions.assertEquals("testSchema", schemaKey.getSchemaName());
-        Assertions.assertEquals("V1234", schemaKey.getVersion());
+        Assertions.assertEquals("testNamespace", schemaDetails.getNamespace());
+        Assertions.assertEquals("testSchema", schemaDetails.getSchemaName());
+        Assertions.assertEquals("V1234", schemaDetails.getVersion());
         final var schemaMeta = schemaDetails.getSchemaMeta();
         Assertions.assertNotNull(schemaMeta);
         Assertions.assertEquals("testUser", schemaMeta.getCreatedBy());

leia-dropwizard-es/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>com.grookage.leia</groupId>
         <artifactId>leia-parent</artifactId>
-        <version>0.0.1-RC7</version>
+        <version>0.0.1-RC8</version>
         <relativePath>../leia-parent</relativePath>
     </parent>
 

leia-dropwizard-es/src/main/java/com/grookage/leia/es/dropwizard/LeiaElasticBundle.java

@@ -46,6 +46,7 @@ protected Supplier<SchemaRepository> getRepositorySupplier(T configuration) {
         return () -> elasticSchemaRepository;
     }
 
+    @Override
     protected List<LeiaHealthCheck> withHealthChecks(T configuration) {
         return List.of(new ElasticHealthCheck(elasticConfig, elasticsearchClient));
     }

leia-dropwizard/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>com.grookage.leia</groupId>
         <artifactId>leia-parent</artifactId>
-        <version>0.0.1-RC7</version>
+        <version>0.0.1-RC8</version>
         <relativePath>../leia-parent</relativePath>
     </parent>
 

leia-dropwizard/src/main/java/com/grookage/leia/dropwizard/bundle/resources/SchemaResource.java

@@ -18,13 +18,13 @@
 
 import com.codahale.metrics.annotation.ExceptionMetered;
 import com.codahale.metrics.annotation.Timed;
-import com.fasterxml.jackson.databind.JsonNode;
 import com.grookage.leia.common.validation.SchemaPayloadValidator;
 import com.grookage.leia.core.exception.LeiaErrorCode;
 import com.grookage.leia.core.exception.LeiaException;
 import com.grookage.leia.core.retrieval.SchemaRetriever;
 import com.grookage.leia.models.GenericResponse;
 import com.grookage.leia.models.request.NamespaceRequest;
+import com.grookage.leia.models.request.ValidateSchemaRequest;
 import com.grookage.leia.models.schema.SchemaDetails;
 import com.grookage.leia.models.schema.SchemaKey;
 import lombok.AllArgsConstructor;
@@ -86,11 +86,11 @@ public List<SchemaDetails> getAllSchemaDetails(@Valid final NamespaceRequest nam
     @Timed
     @ExceptionMetered
     @Path("/details/validate")
-    public GenericResponse<List<String>> validateSchema(@Valid SchemaKey schemaKey,
-                                                        @Valid JsonNode jsonNode) {
-        final var schemaDetails = schemaRetriever.getSchemaDetails(schemaKey)
+    public GenericResponse<List<String>> validateSchema(@Valid final ValidateSchemaRequest validateSchemaRequest) {
+        final var schemaDetails = schemaRetriever.getSchemaDetails(validateSchemaRequest.getSchemaKey())
                 .orElseThrow(() -> LeiaException.error(LeiaErrorCode.NO_SCHEMA_FOUND));
-        final var validationErrors = SchemaPayloadValidator.validate(jsonNode, schemaDetails.getValidationType(),
+        final var validationErrors = SchemaPayloadValidator.validate(validateSchemaRequest.getJsonNode(),
+                schemaDetails.getValidationType(),
                 schemaDetails.getAttributes());
         if (validationErrors.isEmpty()) {
             return GenericResponse.<List<String>>builder()

leia-elastic/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>com.grookage.leia</groupId>
         <artifactId>leia-parent</artifactId>
-        <version>0.0.1-RC7</version>
+        <version>0.0.1-RC8</version>
         <relativePath>../leia-parent</relativePath>
     </parent>
 

leia-models/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>com.grookage.leia</groupId>
         <artifactId>leia-parent</artifactId>
-        <version>0.0.1-RC7</version>
+        <version>0.0.1-RC8</version>
         <relativePath>../leia-parent</relativePath>
     </parent>
 

leia-models/src/main/java/com/grookage/leia/models/request/ValidateSchemaRequest.java

@@ -0,0 +1,25 @@
+package com.grookage.leia.models.request;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.grookage.leia.models.schema.SchemaKey;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import javax.validation.Valid;
+import javax.validation.constraints.NotNull;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ValidateSchemaRequest {
+    @Valid
+    @NotNull
+    private SchemaKey schemaKey;
+    @NotNull
+    private JsonNode jsonNode;
+}