Skip to content

Commit

Permalink
ISPN-346 Fix JSON for maps
Browse files Browse the repository at this point in the history
  • Loading branch information
@tristantarrant
tristantarrant committed Jul 3, 2024
1 parent d3498d0 commit 55d26d4
Show file tree
Hide file tree
Showing 10 changed files with 304 additions and 36 deletions.
10 changes: 9 additions & 1 deletion core/src/main/java/org/infinispan/protostream/ProtobufParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import org.infinispan.protostream.descriptors.Descriptor;
import org.infinispan.protostream.descriptors.FieldDescriptor;
import org.infinispan.protostream.descriptors.MapDescriptor;
import org.infinispan.protostream.descriptors.Type;
import org.infinispan.protostream.descriptors.WireType;
import org.infinispan.protostream.impl.TagReaderImpl;
Expand Down Expand Up @@ -64,12 +65,19 @@ private void parseMessage(TagHandler tagHandler, Descriptor messageDescriptor, T
final int fieldNumber = WireType.getTagFieldNumber(tag);
final WireType wireType = WireType.fromTag(tag);
final FieldDescriptor fd = messageDescriptor != null ? messageDescriptor.findFieldByNumber(fieldNumber) : null;

switch (wireType) {
case LENGTH_DELIMITED: {
if (fd == null) {
byte[] value = in.readByteArray();
tagHandler.onTag(fieldNumber, null, value);
} else if (fd instanceof MapDescriptor md) {
int length = in.readUInt32();
int oldLimit = in.pushLimit(length);
tagHandler.onStartNested(fieldNumber, fd);
parseMessage(tagHandler, md.asDescriptor(), in);
tagHandler.onEndNested(fieldNumber, fd);
in.checkLastTagWas(0);
in.popLimit(oldLimit);
} else if (fd.getType() == Type.STRING) {
String value = in.readString();
tagHandler.onTag(fieldNumber, fd, value);
Expand Down
4 changes: 4 additions & 0 deletions core/src/main/java/org/infinispan/protostream/descriptors/FieldDescriptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,10 @@ public boolean isRepeated() {
return label == Label.REPEATED;
}

public boolean isMap() {
return false;
}

public JavaType getJavaType() {
return getType().getJavaType();
}
Expand Down
22 changes: 22 additions & 0 deletions core/src/main/java/org/infinispan/protostream/descriptors/MapDescriptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,34 @@
public class MapDescriptor extends FieldDescriptor {
private final String keyTypeName;
private final Type keyType;
private final Descriptor descriptor;

private MapDescriptor(Builder builder) {
super(builder);
keyTypeName = builder.keyTypeName;
keyType = Type.primitiveFromString(keyTypeName);
Descriptor.Builder b = new Descriptor.Builder().withName(name).withFullName(fullName);
FieldDescriptor.Builder kb = new FieldDescriptor.Builder().withNumber(1).withName("key").withTypeName(keyTypeName);
b.addField(kb);
FieldDescriptor.Builder vb = new FieldDescriptor.Builder().withNumber(2).withName("value").withTypeName(typeName);
b.addField(vb);
descriptor = b.build();
}

public Descriptor asDescriptor() {
return descriptor;
}

@Override
public boolean isRepeated() {
return true;
}

@Override
public boolean isMap() {
return true;
}

public Type getKeyType() {
return keyType;
}
Expand Down Expand Up @@ -44,6 +60,12 @@ public Label getLabel() {
return Label.OPTIONAL;
}

@Override
void setMessageType(Descriptor descriptor) {
super.setMessageType(descriptor);
this.descriptor.getFields().get(1).setMessageType(descriptor);
}

@Override
public String toString() {
return "MapDescriptor{" +
Expand Down
126 changes: 103 additions & 23 deletions core/src/main/java/org/infinispan/protostream/impl/JsonUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import org.infinispan.protostream.descriptors.FieldDescriptor;
import org.infinispan.protostream.descriptors.GenericDescriptor;
import org.infinispan.protostream.descriptors.Label;
import org.infinispan.protostream.descriptors.MapDescriptor;
import org.infinispan.protostream.descriptors.Type;

import com.fasterxml.jackson.core.JsonFactory;
Expand Down Expand Up @@ -230,11 +231,15 @@ private static void processObject(ImmutableSerializationContext ctx, JsonParser
break;
case START_OBJECT: {
FieldDescriptor fd = messageDescriptor.findFieldByName(currentField);
Descriptor messageType = fd.getMessageType();
if (messageType == null) {
throw new IllegalStateException("Field '" + currentField + "' is not an object");
if (fd.isMap()) {
processMap(ctx, (MapDescriptor) fd, parser, nestedWriter);
} else {
Descriptor messageType = fd.getMessageType();
if (messageType == null) {
throw new IllegalStateException("Field '" + currentField + "' is not an object");
}
processObject(ctx, parser, nestedWriter, messageType, fd.getNumber(), false);
}
processObject(ctx, parser, nestedWriter, messageType, fd.getNumber(), false);
break;
}
case FIELD_NAME:
Expand Down Expand Up @@ -280,6 +285,76 @@ private static void processObject(ImmutableSerializationContext ctx, JsonParser
writer.flush();
}

private static void processMap(ImmutableSerializationContext ctx, MapDescriptor md, JsonParser parser, TagWriter writer) throws IOException {
while (true) {
JsonToken token = parser.nextToken();
if (token == JsonToken.END_OBJECT) {
break;
}
if (token != JsonToken.FIELD_NAME) {
throw new IllegalStateException("Unexpected token");
}
ByteArrayOutputStream baos = new ByteArrayOutputStream(ProtobufUtil.DEFAULT_ARRAY_BUFFER_SIZE);
TagWriter nestedWriter = TagWriterImpl.newInstanceNoBuffer(ctx, baos);
String key = parser.getCurrentName();
switch (md.getKeyType()) {
case STRING -> nestedWriter.writeString(1, key);
case INT32 -> nestedWriter.writeInt32(1, Integer.parseInt(key));
case INT64 -> nestedWriter.writeInt64(1, Long.parseLong(key));
case FIXED32 -> nestedWriter.writeFixed32(1, Integer.parseInt(key));
case FIXED64 -> nestedWriter.writeFixed64(1, Long.parseLong(key));
case SINT32 -> nestedWriter.writeSInt32(1, Integer.parseInt(key));
case SINT64 -> nestedWriter.writeSInt64(1, Long.parseLong(key));
case SFIXED32 -> nestedWriter.writeSFixed32(1, Integer.parseInt(key));
case SFIXED64 -> nestedWriter.writeSFixed64(1, Long.parseLong(key));
case UINT32 -> nestedWriter.writeUInt32(1, Integer.parseInt(key));
case UINT64 -> nestedWriter.writeUInt64(1, Long.parseLong(key));
}
processMapValue(ctx, parser, nestedWriter, md);
writer.writeBytes(md.getNumber(), baos.toByteArray());
writer.flush();
}

}

private static void processMapValue(ImmutableSerializationContext ctx, JsonParser parser, TagWriter writer, MapDescriptor md) throws IOException {
JsonToken token = parser.nextToken();
if (token == null) {
return;
}
switch (token) {
case START_OBJECT: {
processObject(ctx, parser, writer, md.getMessageType(),2, false);
break;
}
case VALUE_STRING:
writer.writeString(2, parser.getValueAsString());
break;
case VALUE_NUMBER_INT:
switch (md.getType()) {
case INT32 -> writer.writeInt32(2, parser.getIntValue());
case INT64 -> writer.writeInt64(2, parser.getLongValue());
case FIXED32 -> writer.writeFixed32(2, parser.getIntValue());
case FIXED64 -> writer.writeFixed64(2, parser.getLongValue());
case SINT32 -> writer.writeSInt32(2, parser.getIntValue());
case SINT64 -> writer.writeSInt64(2, parser.getLongValue());
case SFIXED32 -> writer.writeSFixed32(2, parser.getIntValue());
case SFIXED64 -> writer.writeSFixed64(2, parser.getLongValue());
case UINT32 -> writer.writeUInt32(2, parser.getIntValue());
case UINT64 -> writer.writeUInt64(2, parser.getLongValue());
}
break;
case VALUE_NUMBER_FLOAT:
switch (md.getType()) {
case FLOAT -> writer.writeFloat(2, parser.getFloatValue());
case DOUBLE -> writer.writeDouble(2, parser.getDoubleValue());
}
break;
}
writer.flush();
}


private static void processPrimitive(JsonParser parser, TagWriter writer, Type fieldType) throws IOException {
while (true) {
JsonToken token = parser.nextToken();
Expand Down Expand Up @@ -546,15 +621,14 @@ public void onStartNested(int fieldNumber, FieldDescriptor fieldDescriptor) {
return;
}
startSlot(fieldDescriptor);

nestingLevel = new JsonNestingLevel(nestingLevel);

if (prettyPrint) {
indent();
nestingLevel.indent++;
}

jsonOut.append('{');
if (!fieldDescriptor.isMap()) {
jsonOut.append('{');
}
}

@Override
Expand All @@ -566,7 +640,9 @@ public void onEndNested(int fieldNumber, FieldDescriptor fieldDescriptor) {
nestingLevel.indent--;
indent();
}
jsonOut.append('}');
if (!fieldDescriptor.isMap()) {
jsonOut.append('}');
}
nestingLevel = nestingLevel.previous;
}

Expand All @@ -590,37 +666,40 @@ private void startSlot(FieldDescriptor fieldDescriptor) {
if (nestingLevel.repeatedFieldDescriptor != null && nestingLevel.repeatedFieldDescriptor != fieldDescriptor) {
endArraySlot();
}

boolean map = nestingLevel.previous != null && nestingLevel.previous.repeatedFieldDescriptor != null && nestingLevel.previous.repeatedFieldDescriptor.isMap();
if (nestingLevel.isFirstField) {
nestingLevel.isFirstField = false;
} else {
jsonOut.append(',');
jsonOut.append(map ? ':' : ',');
}
if (!fieldDescriptor.isRepeated() || nestingLevel.repeatedFieldDescriptor == null) {
if (prettyPrint) {
indent();
if (!map) {
if (!fieldDescriptor.isRepeated() || nestingLevel.repeatedFieldDescriptor == null) {
if (prettyPrint) {
indent();
}
if (fieldDescriptor.getLabel() == Label.ONE_OF) {
jsonOut.append('"').append(JSON_VALUE_FIELD).append("\":");
} else {
jsonOut.append('"').append(fieldDescriptor.getName()).append("\":");
}
}
if (fieldDescriptor.getLabel() == Label.ONE_OF) {
jsonOut.append('"').append(JSON_VALUE_FIELD).append("\":");
} else {
jsonOut.append('"').append(fieldDescriptor.getName()).append("\":");
if (prettyPrint) {
jsonOut.append(' ');
}
}
if (prettyPrint) {
jsonOut.append(' ');
}
if (fieldDescriptor.isRepeated() && nestingLevel.repeatedFieldDescriptor == null) {
nestingLevel.repeatedFieldDescriptor = fieldDescriptor;
jsonOut.append('[');
jsonOut.append(fieldDescriptor.isMap() ? '{' : '[');
}
}

private void endArraySlot() {
boolean map = nestingLevel.repeatedFieldDescriptor.isMap();
if (prettyPrint && nestingLevel.repeatedFieldDescriptor.getType() == Type.MESSAGE) {
indent();
}
nestingLevel.repeatedFieldDescriptor = null;
jsonOut.append(']');
jsonOut.append(map ? '}' : ']');
}
};

Expand Down Expand Up @@ -773,6 +852,7 @@ private static void expectField(String expectedFieldName, String actualFieldName
}

// todo [anistor] do we really need html escaping? so far I'm keeping it so we behave like previous implementation

/**
* Escapes a string literal in order to have a valid JSON representation. Optionally it can also escape some html chars.
*/
Expand Down
6 changes: 6 additions & 0 deletions integrationtests/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,12 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.google.testing.compile</groupId>
<artifactId>compile-testing</artifactId>
Expand Down
Loading

0 comments on commit 55d26d4

Please sign in to comment.