Updates to JSON schema and Data Prepper documentation. (#5019)

Support @JsonValue for determining enumeration values in the JSON Schema. Provide a JSON schema type of string for EventKey objects. Documentation wording improvements to the mutate event and mutate string processors.

Signed-off-by: David Venable <[email protected]>

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/DataType.java

@@ -95,7 +95,7 @@ public String getTypeName() {
  }
 
  @JsonCreator
- static DataType fromTypeName(final String option) {
+ public static DataType fromTypeName(final String option) {
  return TYPES_MAP.get(option);
  }
 

data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/DataTypeTest.java

@@ -6,14 +6,21 @@
 package org.opensearch.dataprepper.model.event;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.ArgumentsProvider;
+import org.junit.jupiter.params.provider.ArgumentsSource;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.EnumSource;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.emptyString;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.params.provider.Arguments.arguments;
 
 import java.math.BigDecimal;
 import java.util.ArrayList;
@@ -39,26 +46,55 @@ void test_isSameType(Object object, String type, boolean expectedResult) {
  assertThat(DataType.isSameType(object, type), equalTo(expectedResult));
  }
 
+ @ParameterizedTest
+ @EnumSource(DataType.class)
+ void getTypeName_returns_non_empty_string_for_all_types(final DataType dataType) {
+ assertThat(dataType.getTypeName(), notNullValue());
+ assertThat(dataType.getTypeName(), not(emptyString()));
+ }
+
+ @ParameterizedTest
+ @ArgumentsSource(DataTypeToKnownString.class)
+ void getTypeName_returns_expected_name(final DataType dataType, final String expectedString) {
+ assertThat(dataType.getTypeName(), equalTo(expectedString));
+ }
+
  private static Stream<Arguments> getSameTypeTestData() {
  int[] testArray = {1,2};
  List<Integer> testList = new ArrayList<>();
  return Stream.of(
- Arguments.of(2, "integer", true),
- Arguments.of("testString", "string", true),
- Arguments.of(2L, "long", true),
- Arguments.of(2.0, "double", true),
- Arguments.of(BigDecimal.valueOf(2.34567), "big_decimal", true),
- Arguments.of(true, "boolean", true),
- Arguments.of(Map.of("k","v"), "map", true),
- Arguments.of(testArray, "array", true),
- Arguments.of(testList, "array", true),
- Arguments.of(2.0, "integer", false),
- Arguments.of(2, "string", false),
- Arguments.of("testString", "long", false),
- Arguments.of("testString", "double", false),
- Arguments.of(2, "boolean", false),
- Arguments.of(2L, "map", false),
- Arguments.of(2, "array", false)
+ arguments(2, "integer", true),
+ arguments("testString", "string", true),
+ arguments(2L, "long", true),
+ arguments(2.0, "double", true),
+ arguments(BigDecimal.valueOf(2.34567), "big_decimal", true),
+ arguments(true, "boolean", true),
+ arguments(Map.of("k","v"), "map", true),
+ arguments(testArray, "array", true),
+ arguments(testList, "array", true),
+ arguments(2.0, "integer", false),
+ arguments(2, "string", false),
+ arguments("testString", "long", false),
+ arguments("testString", "double", false),
+ arguments(2, "boolean", false),
+ arguments(2L, "map", false),
+ arguments(2, "array", false)
  );
  }
+
+ static class DataTypeToKnownString implements ArgumentsProvider {
+ @Override
+ public Stream<? extends Arguments> provideArguments(final ExtensionContext extensionContext) {
+ return Stream.of(
+ arguments(DataType.STRING, "string"),
+ arguments(DataType.BOOLEAN, "boolean"),
+ arguments(DataType.INTEGER, "integer"),
+ arguments(DataType.LONG, "long"),
+ arguments(DataType.DOUBLE, "double"),
+ arguments(DataType.BIG_DECIMAL, "big_decimal"),
+ arguments(DataType.MAP, "map"),
+ arguments(DataType.ARRAY, "array")
+ );
+ }
+ }
 }

data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java

@@ -23,6 +23,7 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static com.github.victools.jsonschema.module.jackson.JacksonOption.FLATTENED_ENUMS_FROM_JSONVALUE;
 import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_ORDER;
 import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_REQUIRED;
 
@@ -52,7 +53,7 @@ public static void main(String[] args) {
  @Override
  public void run() {
  final List<Module> modules = List.of(
- new CustomJacksonModule(RESPECT_JSONPROPERTY_REQUIRED, RESPECT_JSONPROPERTY_ORDER),
+ new CustomJacksonModule(RESPECT_JSONPROPERTY_REQUIRED, RESPECT_JSONPROPERTY_ORDER, FLATTENED_ENUMS_FROM_JSONVALUE),
  new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED,
  JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS)
  );

data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java

@@ -1,5 +1,7 @@
 package org.opensearch.dataprepper.schemas;
 
+import com.fasterxml.classmate.TypeBindings;
+import com.fasterxml.classmate.types.ResolvedObjectType;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -12,12 +14,14 @@
 import com.github.victools.jsonschema.generator.SchemaGeneratorConfigPart;
 import com.github.victools.jsonschema.generator.SchemaGeneratorGeneralConfigPart;
 import com.github.victools.jsonschema.generator.SchemaVersion;
+import org.opensearch.dataprepper.model.event.EventKey;
 import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
 import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin;
 import org.opensearch.dataprepper.plugin.PluginProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -44,6 +48,7 @@ public ObjectNode convertIntoJsonSchema(
  overrideTargetTypeWithUsesDataPrepperPlugin(scopeSchemaGeneratorConfigPart);
  resolveDefaultValueFromJsonProperty(scopeSchemaGeneratorConfigPart);
  overrideDataPrepperPluginTypeAttribute(configBuilder.forTypesInGeneral(), schemaVersion, optionPreset);
+ resolveDataPrepperTypes(scopeSchemaGeneratorConfigPart);
 
  final SchemaGeneratorConfig config = configBuilder.build();
  final SchemaGenerator generator = new SchemaGenerator(config);
@@ -103,4 +108,13 @@ private void resolveDefaultValueFromJsonProperty(
  return annotation == null || annotation.defaultValue().isEmpty() ? null : annotation.defaultValue();
  });
  }
+
+ private void resolveDataPrepperTypes(final SchemaGeneratorConfigPart<FieldScope> scopeSchemaGeneratorConfigPart) {
+ scopeSchemaGeneratorConfigPart.withTargetTypeOverridesResolver(field -> {
+ if(field.getType().getErasedType().equals(EventKey.class)) {
+ return Collections.singletonList(ResolvedObjectType.create(String.class, TypeBindings.emptyBindings(), null, null));
+ }
+ return Collections.singletonList(field.getType());
+ });
+ }
 }

data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java

@@ -10,6 +10,7 @@
 import com.github.victools.jsonschema.generator.OptionPreset;
 import com.github.victools.jsonschema.generator.SchemaVersion;
 import org.junit.jupiter.api.Test;
+import org.opensearch.dataprepper.model.event.EventKey;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
@@ -22,6 +23,7 @@
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
 
 @ExtendWith(MockitoExtension.class)
@@ -64,6 +66,18 @@ void testConvertIntoJsonSchemaWithCustomJacksonModule() throws JsonProcessingExc
  assertThat(propertiesNode.has("custom_test_attribute"), is(true));
  }
 
+ @Test
+ void testConvertIntoJsonSchemaWithEventKey() throws JsonProcessingException {
+ final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest(Collections.emptyList(), pluginProvider);
+ final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema(
+ SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfig.class);
+ final JsonNode propertiesNode = jsonSchemaNode.at("/properties");
+ assertThat(propertiesNode, instanceOf(ObjectNode.class));
+ assertThat(propertiesNode.has("testAttributeEventKey"), is(equalTo(true)));
+ assertThat(propertiesNode.get("testAttributeEventKey"), is(notNullValue()));
+ assertThat(propertiesNode.get("testAttributeEventKey").get("type"), is(equalTo(TextNode.valueOf("string"))));
+ }
+
  @JsonClassDescription("test config")
  static class TestConfig {
  private String testAttributeWithGetter;
@@ -77,5 +91,7 @@ static class TestConfig {
  public String getTestAttributeWithGetter() {
  return testAttributeWithGetter;
  }
+
+ private EventKey testAttributeEventKey;
  }
 }

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java

@@ -18,7 +18,7 @@
 import java.util.stream.Stream;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `add_entries` processor adds entries to an event.")
+@JsonClassDescription("The <code>add_entries</code> processor adds entries to an event.")
 public class AddEntryProcessorConfig {
  public static class Entry {
 
@@ -29,7 +29,7 @@ public static class Entry {
 
  @JsonProperty("metadata_key")
  @JsonPropertyDescription("The key for the new metadata attribute. The argument must be a literal string key " +
- "and not a JSON Pointer. Either one string key or <code>metadata_key</code> is required.")
+ "and not a JSON Pointer. Either one of <code>key</code> or <code>metadata_key</code> is required.")
  private String metadataKey;
 
  @JsonPropertyDescription("The value of the new entry to be added, which can be used with any of the " +

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java

@@ -15,30 +15,30 @@
 import java.util.Optional;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `convert_entry_type` processor converts a value type associated with the specified key in " +
- "a event to the specified type. It is a casting processor that changes the types of some fields in events.")
+@JsonClassDescription("The <code>convert_entry_type</code> processor converts a value associated with the specified key in " +
+ "a event to the specified type. It is a casting processor that changes the types of specified fields in events.")
 public class ConvertEntryTypeProcessorConfig implements ConverterArguments {
  @JsonProperty("key")
  @JsonPropertyDescription("Key whose value needs to be converted to a different type.")
  private String key;
 
  @JsonProperty("keys")
- @JsonPropertyDescription("List of keys whose value needs to be converted to a different type.")
+ @JsonPropertyDescription("List of keys whose values needs to be converted to a different type.")
  private List<String> keys;
 
  @JsonProperty("type")
- @JsonPropertyDescription("Target type for the key-value pair. Possible values are integer, long, double, big_decimal, string, and boolean. Default value is integer.")
+ @JsonPropertyDescription("Target type for the values. Default value is <code>integer.</code>")
  private TargetType type = TargetType.INTEGER;
 
  /**
  * Optional scale value used only in the case of BigDecimal converter
  */
  @JsonProperty("scale")
- @JsonPropertyDescription("Modifies the scale of the big_decimal when converting to a big_decimal. The default value is 0.")
+ @JsonPropertyDescription("Modifies the scale of the <code>big_decimal</code> when converting to a <code>big_decimal</code>. The default value is 0.")
  private int scale = 0;
 
  @JsonProperty("convert_when")
- @JsonPropertyDescription("Specifies a condition using a <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">Data Prepper expression</a> for performing the convert_entry_type operation. If specified, the convert_entry_type operation runs only when the expression evaluates to true.")
+ @JsonPropertyDescription("Specifies a condition using a <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> for performing the <code>convert_entry_type</code> operation. If specified, the <code>convert_entry_type</code> operation runs only when the expression evaluates to true. Example: <code>/mykey != \"---\"</code>")
  private String convertWhen;
 
  @JsonProperty("null_values")

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java

@@ -17,8 +17,7 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `copy_values` processor copies values within an event and is a [mutate event]" +
- "(https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/processors/mutate-event/) processor.")
+@JsonClassDescription("The <code>copy_values</code> processor copies values within an event to other fields within the event.")
 public class CopyValueProcessorConfig {
  public static class Entry {
  @NotEmpty

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorConfig.java

@@ -18,20 +18,20 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `delete_entries` processor deletes entries, such as key-value pairs, from an event. " +
- "You can define the keys you want to delete in the `with-keys` field following `delete_entries` in the YAML " +
- "configuration file. Those keys and their values are deleted.")
+@JsonClassDescription("The <code>delete_entries</code> processor deletes fields from events. " +
+ "You can define the keys you want to delete in the <code>with_keys</code> configuration." +
+ "Those keys and their values are deleted from events.")
 public class DeleteEntryProcessorConfig {
  @NotEmpty
  @NotNull
  @JsonProperty("with_keys")
  @EventKeyConfiguration(EventKeyFactory.EventAction.DELETE)
- @JsonPropertyDescription("An array of keys for the entries to be deleted.")
+ @JsonPropertyDescription("A list of keys to be deleted.")
  private List<@NotNull @NotEmpty EventKey> withKeys;
 
  @JsonProperty("delete_when")
  @JsonPropertyDescription("Specifies under what condition the <code>delete_entries</code> processor should perform deletion. " +
- "Default is no condition.")
+ "By default, keys are always deleted. Example: <code>/mykey == \"---\"</code>")
  private String deleteWhen;
 
  public List<EventKey> getWithKeys() {

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java

@@ -19,8 +19,8 @@
 import java.util.stream.Collectors;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `list_to_map` processor converts a list of objects from an event, " +
- "where each object contains a `key` field, into a map of target keys.")
+@JsonClassDescription("The <code>list_to_map</code> processor converts a list of objects from an event, " +
+ "where each object contains a <code>key</code> field, into a map of target keys.")
 public class ListToMapProcessorConfig {
  enum FlattenedElement {
  FIRST("first"),
@@ -89,9 +89,9 @@ static FlattenedElement fromOptionValue(final String option) {
  private FlattenedElement flattenedElement = FlattenedElement.FIRST;
 
  @JsonProperty("list_to_map_when")
- @JsonPropertyDescription("A Data Prepper <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
+ @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
  "such as <code>/some-key == \"test\"'</code>, that will be evaluated to determine whether the processor will be " +
- "run on the event. Default is <code>null</code>. All events will be processed unless otherwise stated.")
+ "run on the event. By default, all events will be processed unless otherwise stated.")
  private String listToMapWhen;
 
  @JsonProperty("tags_on_failure")

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorConfig.java

@@ -16,7 +16,7 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `map_to_list` processor converts a map of key-value pairs to a list of objects. " +
+@JsonClassDescription("The <code>map_to_list</code> processor converts a map of key-value pairs to a list of objects. " +
  "Each object contains the key and value in separate fields.")
 public class MapToListProcessorConfig {
  private static final String DEFAULT_KEY_NAME = "key";
@@ -45,9 +45,9 @@ public class MapToListProcessorConfig {
  private String valueName = DEFAULT_VALUE_NAME;
 
  @JsonProperty("map_to_list_when")
- @JsonPropertyDescription("A Data Prepper <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
+ @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
  "such as <code>/some-key == \"test\"'</code>, that will be evaluated to determine whether the processor will " +
- "be run on the event. Default is <code>null</code>. All events will be processed unless otherwise stated.")
+ "be run on the event. By default, all events will be processed unless otherwise stated.")
  private String mapToListWhen;
 
  @JsonProperty("exclude_keys")

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java

@@ -19,7 +19,7 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `rename_keys` processor renames keys in an event.")
+@JsonClassDescription("The <code>rename_keys</code> processor renames keys in an event.")
 public class RenameKeyProcessorConfig {
  public static class Entry {
  @NotEmpty
@@ -41,9 +41,9 @@ public static class Entry {
  private boolean overwriteIfToKeyExists = false;
 
  @JsonProperty("rename_when")
- @JsonPropertyDescription("A Data Prepper <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
+ @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
  "such as <code>/some-key == \"test\"'</code>, that will be evaluated to determine whether the processor will be " +
- "run on the event. Default is <code>null</code>. All events will be processed unless otherwise stated.")
+ "run on the event. By default, all events will be processed unless otherwise stated.")
  private String renameWhen;
 
  public EventKey getFromKey() {

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/SelectEntriesProcessorConfig.java

@@ -15,7 +15,7 @@
 import java.util.List;
 
 @JsonPropertyOrder
-@JsonClassDescription("The `select_entries` processor selects entries from a Data Prepper event.")
+@JsonClassDescription("The <code>select_entries</code> processor selects entries from an event.")
 public class SelectEntriesProcessorConfig {
  @NotEmpty
  @NotNull
@@ -24,7 +24,7 @@ public class SelectEntriesProcessorConfig {
  private List<String> includeKeys;
 
  @JsonProperty("select_when")
- @JsonPropertyDescription("A Data Prepper <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
+ @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
  "such as <code>/some-key == \"test\"'</code>, that will be evaluated to determine whether the processor will be " +
  "run on the event. Default is <code>null</code>. All events will be processed unless otherwise stated.")
  private String selectWhen;