Skip to content

Commit

Permalink
Merge branch 'master' into alhuang-merge-gate-sysadmin
Browse files Browse the repository at this point in the history
  • Loading branch information
sfc-gh-alhuang authored Nov 15, 2024
2 parents b153e41 + 8ba1e0c commit 10fbb62
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,9 @@ private static ParquetBufferValue getStructValue(
missingFields.add(type.getFieldName(i));
} else {
listVal.add(null);
subColumnFinder
.getSubColumns(type.getType(i).getId())
.forEach(subColumn -> statsMap.get(subColumn).incCurrentNullCount());
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import static org.assertj.core.api.Assertions.assertThat;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
Expand Down Expand Up @@ -58,6 +59,7 @@ public void testStructuredDataType() throws Exception {
"object(a int, b string, c boolean)", "{\"a\": 1, \"b\": \"test\", \"c\": true}");
assertStructuredDataType("map(string, int)", "{\"key1\": 1}");
assertStructuredDataType("array(int)", "[1, 2, 3]");
assertStructuredDataType("object(a int, b string, c boolean) not null", "{}");
assertStructuredDataType("array(string) not null", "[]");
assertStructuredDataType("map(string, int) not null", "{}");
assertMap(
Expand All @@ -68,6 +70,9 @@ public void testStructuredDataType() throws Exception {
}
});
assertStructuredDataType("array(string)", null);
assertStructuredDataType("map(string, int)", null);
assertStructuredDataType("object(a int, b string, c boolean)", null);
assertStructuredDataType("object(a int, b string, c boolean)", "{\"a\": null}");

/* Map with null key */
Assertions.assertThatThrownBy(
Expand Down Expand Up @@ -443,6 +448,8 @@ private void assertStructuredDataType(String dataType, String value) throws Exce
String tmp = res.getString(2);
JsonNode actualNode = tmp == null ? null : objectMapper.readTree(tmp);
JsonNode expectedNode = value == null ? null : objectMapper.readTree(value);
removeNullFields(actualNode);
removeNullFields(expectedNode);
assertThat(actualNode).isEqualTo(expectedNode);
}

Expand All @@ -464,4 +471,31 @@ private void assertMap(String dataType, Map<?, ?> value) throws Exception {
JsonNode expectedNode = value == null ? null : objectMapper.valueToTree(value);
assertThat(actualNode).isEqualTo(expectedNode);
}

/**
* Remove null fields from the JSON node. This is used to compare the JSON node ignoring null
* values. e.g. "{}" and "{a: null}".
*
* @param node JSON node
*/
private void removeNullFields(JsonNode node) {
if (node == null) {
return;
}
if (node.isObject()) {
List<String> keys = new ArrayList<>();
node.fields()
.forEachRemaining(
entry -> {
if (entry.getValue().isNull()) {
keys.add(entry.getKey());
} else {
removeNullFields(entry.getValue());
}
});
keys.forEach(((ObjectNode) (node))::remove);
} else if (node.isArray()) {
node.forEach(this::removeNullFields);
}
}
}

0 comments on commit 10fbb62

Please sign in to comment.