From 8acfd17d540004d464804a989a0e3581e696b35a Mon Sep 17 00:00:00 2001 From: Thomas Hull Date: Tue, 1 Oct 2019 16:52:44 +0100 Subject: [PATCH] fix(#1038): stdout json to use ndjson format --- .../generate/GenerateCommandLine.java | 10 +++ .../utils/CucumberGenerationConfigSource.java | 5 ++ .../deg/output/guice/OutputConfigSource.java | 1 + .../deg/output/guice/OutputModule.java | 4 ++ .../output/writer/json/JsonDataSetWriter.java | 20 ++---- .../writer/json/JsonOutputWriterFactory.java | 19 +++++- .../json/JsonOutputWriterFactoryTest.java | 63 +++++++++++++++++++ 7 files changed, 105 insertions(+), 17 deletions(-) create mode 100644 output/src/test/java/com/scottlogic/deg/output/writer/json/JsonOutputWriterFactoryTest.java diff --git a/orchestrator/src/main/java/com/scottlogic/deg/orchestrator/generate/GenerateCommandLine.java b/orchestrator/src/main/java/com/scottlogic/deg/orchestrator/generate/GenerateCommandLine.java index 3a2b5c915..1cc398d8e 100644 --- a/orchestrator/src/main/java/com/scottlogic/deg/orchestrator/generate/GenerateCommandLine.java +++ b/orchestrator/src/main/java/com/scottlogic/deg/orchestrator/generate/GenerateCommandLine.java @@ -86,6 +86,11 @@ private void checkForAlphaGenerationDataTypes() { description = "Defines whether to overwrite/replace existing output files") boolean overwriteOutputFiles = false; + @CommandLine.Option( + names = {"--ndjson"}, + description = "Defines whether JSON output is in NDJ (newline-delimited JSON) format- defaults true for stdOut") + private Boolean ndjson; + @CommandLine.Option( names = { "--disable-schema-validation" }, description = "Disables schema validation") @@ -152,6 +157,11 @@ public boolean useStdOut() { return outputPath == null; } + @Override + public boolean useNdJson() { + return (ndjson == null && this.useStdOut()) || ndjson; + } + @Override public Path getOutputPath() { return outputPath; diff --git a/orchestrator/src/test/java/com/scottlogic/deg/orchestrator/cucumber/testframework/utils/CucumberGenerationConfigSource.java b/orchestrator/src/test/java/com/scottlogic/deg/orchestrator/cucumber/testframework/utils/CucumberGenerationConfigSource.java index d268c19f4..0d4389f26 100644 --- a/orchestrator/src/test/java/com/scottlogic/deg/orchestrator/cucumber/testframework/utils/CucumberGenerationConfigSource.java +++ b/orchestrator/src/test/java/com/scottlogic/deg/orchestrator/cucumber/testframework/utils/CucumberGenerationConfigSource.java @@ -87,6 +87,11 @@ public boolean useStdOut() { return false; } + @Override + public boolean useNdJson() { + return false; + } + @Override public OutputFormat getOutputFormat() { return null; diff --git a/output/src/main/java/com/scottlogic/deg/output/guice/OutputConfigSource.java b/output/src/main/java/com/scottlogic/deg/output/guice/OutputConfigSource.java index f0d82d159..6a0aa3dda 100644 --- a/output/src/main/java/com/scottlogic/deg/output/guice/OutputConfigSource.java +++ b/output/src/main/java/com/scottlogic/deg/output/guice/OutputConfigSource.java @@ -23,4 +23,5 @@ public interface OutputConfigSource { Path getOutputPath(); boolean overwriteOutputFiles(); boolean useStdOut(); + boolean useNdJson(); } diff --git a/output/src/main/java/com/scottlogic/deg/output/guice/OutputModule.java b/output/src/main/java/com/scottlogic/deg/output/guice/OutputModule.java index 6b3af712b..2c307f682 100644 --- a/output/src/main/java/com/scottlogic/deg/output/guice/OutputModule.java +++ b/output/src/main/java/com/scottlogic/deg/output/guice/OutputModule.java @@ -44,5 +44,9 @@ protected void configure() { bind(boolean.class) .annotatedWith(Names.named("config:canOverwriteOutputFiles")) .toInstance(outputConfigSource.overwriteOutputFiles()); + + bind(boolean.class) + .annotatedWith(Names.named("config:useNdJson")) + .toInstance(outputConfigSource.useNdJson()); } } diff --git a/output/src/main/java/com/scottlogic/deg/output/writer/json/JsonDataSetWriter.java b/output/src/main/java/com/scottlogic/deg/output/writer/json/JsonDataSetWriter.java index 266408963..20d729694 100644 --- a/output/src/main/java/com/scottlogic/deg/output/writer/json/JsonDataSetWriter.java +++ b/output/src/main/java/com/scottlogic/deg/output/writer/json/JsonDataSetWriter.java @@ -16,17 +16,13 @@ package com.scottlogic.deg.output.writer.json; -import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SequenceWriter; +import com.scottlogic.deg.common.output.GeneratedObject; import com.scottlogic.deg.common.profile.Field; import com.scottlogic.deg.common.profile.ProfileFields; -import com.scottlogic.deg.common.output.GeneratedObject; import com.scottlogic.deg.output.writer.DataSetWriter; import java.io.IOException; -import java.io.OutputStream; import java.math.BigDecimal; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; @@ -40,24 +36,16 @@ class JsonDataSetWriter implements DataSetWriter { private final SequenceWriter writer; private final ProfileFields fields; - private JsonDataSetWriter(SequenceWriter writer, ProfileFields fields) { + JsonDataSetWriter(SequenceWriter writer, ProfileFields fields) { this.writer = writer; this.fields = fields; } - static DataSetWriter open(OutputStream stream, ProfileFields fields) throws IOException { - ObjectWriter objectWriter = new ObjectMapper().writer(new DefaultPrettyPrinter()); - SequenceWriter writer = objectWriter.writeValues(stream); - writer.init(true); - - return new JsonDataSetWriter(writer, fields); - } - @Override public void writeRow(GeneratedObject row) throws IOException { Map jsonObject = new HashMap<>(); fields.forEach(field -> jsonObject - .put(field , convertValue(row.getFormattedValue(field)))); + .put(field, convertValue(row.getFormattedValue(field)))); writer.write(jsonObject); } @@ -78,7 +66,7 @@ private static Object convertValue(Object value) { } else if (value instanceof String) { return value; } else if (value instanceof OffsetDateTime) { - return standardDateFormat.format((OffsetDateTime)value); + return standardDateFormat.format((OffsetDateTime) value); } else { return value.toString(); } diff --git a/output/src/main/java/com/scottlogic/deg/output/writer/json/JsonOutputWriterFactory.java b/output/src/main/java/com/scottlogic/deg/output/writer/json/JsonOutputWriterFactory.java index b479365a0..f9d5ea474 100644 --- a/output/src/main/java/com/scottlogic/deg/output/writer/json/JsonOutputWriterFactory.java +++ b/output/src/main/java/com/scottlogic/deg/output/writer/json/JsonOutputWriterFactory.java @@ -16,6 +16,12 @@ package com.scottlogic.deg.output.writer.json; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SequenceWriter; +import com.google.inject.Inject; +import com.google.inject.name.Named; import com.scottlogic.deg.common.profile.ProfileFields; import com.scottlogic.deg.output.writer.DataSetWriter; import com.scottlogic.deg.output.writer.OutputWriterFactory; @@ -25,9 +31,20 @@ import java.util.Optional; public class JsonOutputWriterFactory implements OutputWriterFactory { + private boolean useNdJson; + private static final String NEW_LINE_DELIMITER = "\n"; + @Inject + public JsonOutputWriterFactory(@Named("config:useNdJson") boolean useNdJson) { + this.useNdJson = useNdJson; + } + @Override public DataSetWriter createWriter(OutputStream stream, ProfileFields profileFields) throws IOException { - return JsonDataSetWriter.open(stream, profileFields); + ObjectWriter objectWriter = new ObjectMapper().writer(new DefaultPrettyPrinter(NEW_LINE_DELIMITER)); + SequenceWriter writer = objectWriter.writeValues(stream); + writer.init(!useNdJson); + + return new JsonDataSetWriter(writer, profileFields); } @Override diff --git a/output/src/test/java/com/scottlogic/deg/output/writer/json/JsonOutputWriterFactoryTest.java b/output/src/test/java/com/scottlogic/deg/output/writer/json/JsonOutputWriterFactoryTest.java new file mode 100644 index 000000000..164363f00 --- /dev/null +++ b/output/src/test/java/com/scottlogic/deg/output/writer/json/JsonOutputWriterFactoryTest.java @@ -0,0 +1,63 @@ +package com.scottlogic.deg.output.writer.json; + +import com.scottlogic.deg.common.output.GeneratedObject; +import com.scottlogic.deg.common.profile.FieldBuilder; +import com.scottlogic.deg.common.profile.ProfileFields; +import com.scottlogic.deg.output.writer.DataSetWriter; +import org.hamcrest.Matcher; +import org.hamcrest.Matchers; +import org.junit.Assert; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; + +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class JsonOutputWriterFactoryTest { + + @Test + void writer_withNDJSONTrue__shouldOutputNewLineDelimiterRows() throws IOException { + expectJson( + true, + Matchers.equalTo("{\n \"my_field\" : \"my_value\"\n}\n{\n \"my_field\" : \"my_value\"\n}")); + } + + @Test + void writer_withNDJSONFalse__shouldOutputRowsWrappedInAnArray() throws IOException { + expectJson( + false, + Matchers.equalTo("[ {\n \"my_field\" : \"my_value\"\n}, {\n \"my_field\" : \"my_value\"\n} ]")); + } + + private static void expectJson(boolean useNdJson, Matcher matcher) throws IOException { + //Arrange + ProfileFields fields = new ProfileFields(Collections.singletonList(FieldBuilder.createField("my_field"))); + + + // Act + GeneratedObject mockGeneratedObject = mock(GeneratedObject.class); + when(mockGeneratedObject.getFormattedValue(eq(fields.iterator().next()))).thenReturn("my_value"); + String generateJson = generateJson(fields, mockGeneratedObject, useNdJson); + + // Assert + Assert.assertThat(generateJson, matcher); + } + + private static String generateJson(ProfileFields fields, GeneratedObject generatedObject, boolean useNdJson) throws IOException { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + try (DataSetWriter writer = new JsonOutputWriterFactory(useNdJson).createWriter(stream, fields)) { + writer.writeRow(generatedObject); + writer.writeRow(generatedObject); + } + + return stream + .toString(StandardCharsets.UTF_8.name()) + .replace("\r\n", "\n"); // normalise line endings between e.g. Windows and Linux + } +} \ No newline at end of file