Skip to content
This repository has been archived by the owner on Apr 14, 2023. It is now read-only.

Commit

Permalink
Merge pull request #1393 from finos/1038-stdout-json-to-use-ndjson-fo…
Browse files Browse the repository at this point in the history
…rmat

1038 stdout json to use ndjson format
  • Loading branch information
cuthullu authored Oct 2, 2019
2 parents ec8d2db + 2b6e819 commit 684426e
Show file tree
Hide file tree
Showing 7 changed files with 105 additions and 17 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,11 @@ private void checkForAlphaGenerationDataTypes() {
description = "Defines whether to overwrite/replace existing output files")
boolean overwriteOutputFiles = false;

@CommandLine.Option(
names = {"--ndjson"},
description = "Defines whether JSON output is in NDJ (newline-delimited JSON) format- defaults true for stdOut")
private Boolean ndjson;

@CommandLine.Option(
names = { "--disable-schema-validation" },
description = "Disables schema validation")
Expand Down Expand Up @@ -152,6 +157,11 @@ public boolean useStdOut() {
return outputPath == null;
}

@Override
public boolean useNdJson() {
return (ndjson == null && this.useStdOut()) || ndjson;
}

@Override
public Path getOutputPath() {
return outputPath;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,11 @@ public boolean useStdOut() {
return false;
}

@Override
public boolean useNdJson() {
return false;
}

@Override
public OutputFormat getOutputFormat() {
return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,5 @@ public interface OutputConfigSource {
Path getOutputPath();
boolean overwriteOutputFiles();
boolean useStdOut();
boolean useNdJson();
}
Original file line number Diff line number Diff line change
Expand Up @@ -44,5 +44,9 @@ protected void configure() {
bind(boolean.class)
.annotatedWith(Names.named("config:canOverwriteOutputFiles"))
.toInstance(outputConfigSource.overwriteOutputFiles());

bind(boolean.class)
.annotatedWith(Names.named("config:useNdJson"))
.toInstance(outputConfigSource.useNdJson());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,13 @@

package com.scottlogic.deg.output.writer.json;

import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SequenceWriter;
import com.scottlogic.deg.common.output.GeneratedObject;
import com.scottlogic.deg.common.profile.Field;
import com.scottlogic.deg.common.profile.ProfileFields;
import com.scottlogic.deg.common.output.GeneratedObject;
import com.scottlogic.deg.output.writer.DataSetWriter;

import java.io.IOException;
import java.io.OutputStream;
import java.math.BigDecimal;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
Expand All @@ -40,24 +36,16 @@ class JsonDataSetWriter implements DataSetWriter {
private final SequenceWriter writer;
private final ProfileFields fields;

private JsonDataSetWriter(SequenceWriter writer, ProfileFields fields) {
JsonDataSetWriter(SequenceWriter writer, ProfileFields fields) {
this.writer = writer;
this.fields = fields;
}

static DataSetWriter open(OutputStream stream, ProfileFields fields) throws IOException {
ObjectWriter objectWriter = new ObjectMapper().writer(new DefaultPrettyPrinter());
SequenceWriter writer = objectWriter.writeValues(stream);
writer.init(true);

return new JsonDataSetWriter(writer, fields);
}

@Override
public void writeRow(GeneratedObject row) throws IOException {
Map<Field, Object> jsonObject = new HashMap<>();
fields.forEach(field -> jsonObject
.put(field , convertValue(row.getFormattedValue(field))));
.put(field, convertValue(row.getFormattedValue(field))));

writer.write(jsonObject);
}
Expand All @@ -78,7 +66,7 @@ private static Object convertValue(Object value) {
} else if (value instanceof String) {
return value;
} else if (value instanceof OffsetDateTime) {
return standardDateFormat.format((OffsetDateTime)value);
return standardDateFormat.format((OffsetDateTime) value);
} else {
return value.toString();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@

package com.scottlogic.deg.output.writer.json;

import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SequenceWriter;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import com.scottlogic.deg.common.profile.ProfileFields;
import com.scottlogic.deg.output.writer.DataSetWriter;
import com.scottlogic.deg.output.writer.OutputWriterFactory;
Expand All @@ -25,9 +31,20 @@
import java.util.Optional;

public class JsonOutputWriterFactory implements OutputWriterFactory {
private boolean useNdJson;
private static final String NEW_LINE_DELIMITER = "\n";
@Inject
public JsonOutputWriterFactory(@Named("config:useNdJson") boolean useNdJson) {
this.useNdJson = useNdJson;
}

@Override
public DataSetWriter createWriter(OutputStream stream, ProfileFields profileFields) throws IOException {
return JsonDataSetWriter.open(stream, profileFields);
ObjectWriter objectWriter = new ObjectMapper().writer(new DefaultPrettyPrinter(NEW_LINE_DELIMITER));
SequenceWriter writer = objectWriter.writeValues(stream);
writer.init(!useNdJson);

return new JsonDataSetWriter(writer, profileFields);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package com.scottlogic.deg.output.writer.json;

import com.scottlogic.deg.common.output.GeneratedObject;
import com.scottlogic.deg.common.profile.FieldBuilder;
import com.scottlogic.deg.common.profile.ProfileFields;
import com.scottlogic.deg.output.writer.DataSetWriter;
import org.hamcrest.Matcher;
import org.hamcrest.Matchers;
import org.junit.Assert;
import org.junit.jupiter.api.Test;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;

import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

class JsonOutputWriterFactoryTest {

@Test
void writer_withNDJSONTrue__shouldOutputNewLineDelimiterRows() throws IOException {
expectJson(
true,
Matchers.equalTo("{\n \"my_field\" : \"my_value\"\n}\n{\n \"my_field\" : \"my_value\"\n}"));
}

@Test
void writer_withNDJSONFalse__shouldOutputRowsWrappedInAnArray() throws IOException {
expectJson(
false,
Matchers.equalTo("[ {\n \"my_field\" : \"my_value\"\n}, {\n \"my_field\" : \"my_value\"\n} ]"));
}

private static void expectJson(boolean useNdJson, Matcher<String> matcher) throws IOException {
//Arrange
ProfileFields fields = new ProfileFields(Collections.singletonList(FieldBuilder.createField("my_field")));


// Act
GeneratedObject mockGeneratedObject = mock(GeneratedObject.class);
when(mockGeneratedObject.getFormattedValue(eq(fields.iterator().next()))).thenReturn("my_value");
String generateJson = generateJson(fields, mockGeneratedObject, useNdJson);

// Assert
Assert.assertThat(generateJson, matcher);
}

private static String generateJson(ProfileFields fields, GeneratedObject generatedObject, boolean useNdJson) throws IOException {
ByteArrayOutputStream stream = new ByteArrayOutputStream();

try (DataSetWriter writer = new JsonOutputWriterFactory(useNdJson).createWriter(stream, fields)) {
writer.writeRow(generatedObject);
writer.writeRow(generatedObject);
}

return stream
.toString(StandardCharsets.UTF_8.name())
.replace("\r\n", "\n"); // normalise line endings between e.g. Windows and Linux
}
}

0 comments on commit 684426e

Please sign in to comment.