Add examples and reorder some processor configurations in schemas (#5216

) * update schemas according to review Signed-off-by: Katherine Shen <[email protected]> * fix build Signed-off-by: Katherine Shen <[email protected]> --------- Signed-off-by: Katherine Shen <[email protected]>
opensearch-project · Nov 27, 2024 · 76b06ef · 76b06ef
1 parent 539d599
commit 76b06ef
Show file tree

Hide file tree

Showing 15 changed files with 63 additions and 60 deletions.
diff --git a/...or/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessorConfig.java b/...or/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessorConfig.java
@@ -37,7 +37,7 @@ public class CsvProcessorConfig {
     @JsonPropertyDescription("The character separating each column. Default value is <code>,</code>.")
     private String delimiter = DEFAULT_DELIMITER;
 
-    @JsonProperty("delete_header")
+    @JsonProperty(value = "delete_header", defaultValue = "true")
     @JsonPropertyDescription("If specified, the event header (<code>column_names_source_key</code>) is deleted after the event " +
             "is parsed. If there is no event header, no action is taken. Default value is true.")
     private Boolean deleteHeader = DEFAULT_DELETE_HEADERS;
@@ -68,6 +68,10 @@ public class CsvProcessorConfig {
     })
     private List<String> columnNames;
 
+    @JsonPropertyDescription("If true, the configured source field will be deleted after the CSV data is parsed into separate fields.")
+    @JsonProperty
+    private boolean deleteSource = false;
+
     @JsonProperty("csv_when")
     @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> such as <code>/some_key == \"test\"</code>. " +
             "If specified, the <code>csv</code> processor will only run on events when the expression evaluates to true. ")
@@ -76,10 +80,6 @@ public class CsvProcessorConfig {
     })
     private String csvWhen;
 
-    @JsonPropertyDescription("If true, the configured source field will be deleted after the CSV data is parsed into separate fields.")
-    @JsonProperty
-    private boolean deleteSource = false;
-
     /**
      * The field of the Event that contains the CSV data to be processed.
      *

diff --git a/.../src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java b/.../src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java
@@ -198,6 +198,9 @@ public static boolean isValidPattern(final String pattern) {
     @JsonProperty("date_when")
     @JsonPropertyDescription("Specifies under what condition the <code>date</code> processor should perform matching. " +
             "Default is no condition.")
+    @ExampleValues({
+        @Example(value = "/some_key == null", description = "Only runs the date processor on the Event if some_key is null or doesn't exist.")
+    })
     private String dateWhen;
 
     @JsonIgnore

diff --git a/...va/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessorConfig.java b/...va/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessorConfig.java
@@ -33,6 +33,9 @@ public class DecompressProcessorConfig {
     @JsonPropertyDescription("The type of decompression to use for the keys in the event. Only <code>gzip</code> is supported.")
     @JsonProperty("type")
     @NotNull
+    @ExampleValues({
+            @Example(value = "gzip", description = "GZIP decompression.")
+    })
     private DecompressionType decompressionType;
 
     @JsonPropertyDescription("A list of strings with which to tag events when the processor fails to decompress the keys inside an event. Defaults to <code>_decompression_failure</code>.")

diff --git a/.../src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfig.java b/.../src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfig.java
@@ -114,11 +114,14 @@ public class GrokProcessorConfig {
 
     @JsonProperty(TAGS_ON_TIMEOUT)
     @JsonPropertyDescription("The tags to add to the event metadata if the grok match times out.")
-    @ExampleValues({
-        @Example(value = "_timeout", description = "Events are tagged with this string if grok match times out.")
-    })
     private List<String> tagsOnTimeout = Collections.emptyList();
 
+    @JsonProperty(INCLUDE_PERFORMANCE_METADATA)
+    @JsonPropertyDescription("A boolean value to determine whether to include performance metadata into event metadata. " +
+            "If set to true, the events coming out of grok will have new fields such as <code>_total_grok_patterns_attempted</code> and <code>_total_grok_processing_time</code>." +
+            "You can use this metadata to perform performance testing and tuning of your grok patterns. By default, it is not included.")
+    private boolean includePerformanceMetadata = false;
+
     @JsonProperty(GROK_WHEN)
     @ExampleValues({
         @Example(value = "/type == \"ipv4\"", description = "When the IP type is IPV4, the processor will perform matching.")
@@ -127,12 +130,6 @@ public class GrokProcessorConfig {
             "If specified, the <code>grok</code> processor will only run on events when the expression evaluates to true. ")
     private String grokWhen;
 
-    @JsonProperty(INCLUDE_PERFORMANCE_METADATA)
-    @JsonPropertyDescription("A boolean value to determine whether to include performance metadata into event metadata. " +
-            "If set to true, the events coming out of grok will have new fields such as <code>_total_grok_patterns_attempted</code> and <code>_total_grok_processing_time</code>." +
-            "You can use this metadata to perform performance testing and tuning of your grok patterns. By default, it is not included.")
-    private boolean includePerformanceMetadata = false;
-
     public boolean isBreakOnMatch() {
         return breakOnMatch;
     }

diff --git a/...ava/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java b/...ava/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java
@@ -66,7 +66,7 @@ public class AddEntryProcessorConfig {
     public static class Entry {
         @JsonPropertyDescription("The key of the new entry to be added. Some examples of keys include <code>my_key</code>, " +
                 "<code>myKey</code>, and <code>object/sub_Key</code>. The key can also be a format expression, for example, <code>${/key1}</code> to " +
-                "use the value of field <code>key1</code> as the key.")
+                "use the value of field <code>key1</code> as the key. Either one of <code>key</code> or <code>metadata_key</code> is required.")
         @AlsoRequired(values = {
                 @AlsoRequired.Required(name=METADATA_KEY_KEY, allowedValues = {"null"})
         })

diff --git a/...opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java b/...opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java
@@ -38,14 +38,14 @@ public class ConvertEntryTypeProcessorConfig implements ConverterArguments {
     static final String KEYS_KEY = "keys";
 
     @JsonProperty(KEY_KEY)
-    @JsonPropertyDescription("Key whose value needs to be converted to a different type.")
+    @JsonPropertyDescription("Key whose value needs to be converted to a different type. Cannot be declared at the same time as <code>keys</code>.")
     @AlsoRequired(values = {
             @AlsoRequired.Required(name = KEYS_KEY, allowedValues = {"null"})
     })
     private String key;
 
     @JsonProperty(KEYS_KEY)
-    @JsonPropertyDescription("List of keys whose values needs to be converted to a different type.")
+    @JsonPropertyDescription("List of keys whose values needs to be converted to a different type. Cannot be declared at the same time as <code>key</code>.")
     @AlsoRequired(values = {
             @AlsoRequired.Required(name = KEY_KEY, allowedValues = {"null"})
     })

diff --git a/...va/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java b/...va/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java
@@ -86,14 +86,14 @@ public Entry() {
     private List<Entry> entries;
 
     @JsonProperty(FROM_LIST_KEY)
-    @JsonPropertyDescription("The key of the list of objects to be copied.")
+    @JsonPropertyDescription("The key of the list of objects to be copied. <code>to_list</code> must also be defined.")
     @AlsoRequired(values = {
             @AlsoRequired.Required(name = TO_LIST_KEY)
     })
     private String fromList;
 
     @JsonProperty(TO_LIST_KEY)
-    @JsonPropertyDescription("The key of the new list to be added.")
+    @JsonPropertyDescription("The key of the new list to be added. <code>from_list</code> must also be defined.")
     @AlsoRequired(values = {
             @AlsoRequired.Required(name = FROM_LIST_KEY)
     })

diff --git a/...va/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java b/...va/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessorConfig.java
@@ -102,7 +102,7 @@ public String getOptionValue() {
     private boolean extractValue = false;
 
     @NotNull
-    @JsonProperty("flatten")
+    @JsonProperty(value = "flatten", defaultValue = "false")
     @JsonPropertyDescription("When <code>true</code>, values in the generated map output flatten into single items based on " +
             "the <code>flattened_element</code>. Otherwise, objects mapped to values from the generated map appear as lists. " +
             "Default is <code>false</code>.")
@@ -121,6 +121,9 @@ public String getOptionValue() {
     @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
             "such as <code>/some-key == \"test\"</code>, that will be evaluated to determine whether the processor will be " +
             "run on the event. By default, all events will be processed unless otherwise stated.")
+    @ExampleValues({
+        @Example(value = "/some-key == \"test\"", description = "The operation will run when the value of the key is 'test'.")
+    })
     private String listToMapWhen;
 
     public String getSource() {

diff --git a/...va/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorConfig.java b/...va/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorConfig.java
@@ -65,16 +65,10 @@ public class MapToListProcessorConfig {
     @JsonProperty("exclude_keys")
     @JsonPropertyDescription("The keys in the source map that will be excluded from processing. Default is an " +
             "empty list (<code>[]</code>).")
-    @ExampleValues({
-        @Example(value = "[\"key1\"]", description = "When the key is 'key1', the processor will not include this key-value pair in the list and will leave it in the map.")
-    })
     private List<String> excludeKeys = DEFAULT_EXCLUDE_KEYS;
 
     @JsonProperty("tags_on_failure")
     @JsonPropertyDescription("A list of tags to add to the event metadata when the event fails to process.")
-    @ExampleValues({
-        @Example(value = "[\"_failure\"]", description = "{\"tags\": [\"_failure\"]} will be added to the event’s metadata in the event of a processing failure.")
-    })
     private List<String> tagsOnFailure;
 
     @JsonProperty("map_to_list_when")

diff --git a/...va/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java b/...va/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java
@@ -43,6 +43,9 @@ public static class Entry {
         @JsonProperty(defaultValue = FROM_KEY_REGEX)
         @JsonPropertyDescription("The regex pattern of the key of the entry to be renamed. " +
                 "This field cannot be defined along with <code>from_key</code>.")
+        @ExampleValues(
+                @Example(value = "regex", description = "Generic regex string.")
+        )
         @AlsoRequired(values = {
                 @AlsoRequired.Required(name = FROM_KEY, allowedValues = {"null"})
         })

diff --git a/.../org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java b/.../org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java
@@ -47,7 +47,7 @@ public class ObfuscationProcessorConfig {
     @JsonProperty("patterns")
     @JsonPropertyDescription("A list of regex patterns that allow you to obfuscate specific parts of a field. Only parts that match the regex pattern will obfuscate. When not provided, the processor obfuscates the whole field.")
     @ExampleValues({
-        @Example(value = "[A-Za-z0-9+_.-]+@([\\w-]+\\.)+[\\w-]{2,4}", description = "This pattern represents an email address that will be obfuscated in the given field.")
+        @Example(value = "regex", description = "Generic regex pattern.")
     })
     private List<String> patterns;
 
@@ -57,21 +57,21 @@ public class ObfuscationProcessorConfig {
     private PluginModel action;
 
     @JsonProperty("single_word_only")
-    @JsonPropertyDescription("When set to <code>true</code>, a word boundary <code>\b</code> is added to the pattern, " +
+    @JsonPropertyDescription("When set to <code>true</code>, a word boundary <code>\b</code>is added to the pattern, " +
             "which causes obfuscation to be applied only to words that are standalone in the input text. " +
             "By default, it is false, meaning obfuscation patterns are applied to all occurrences.")
     private boolean singleWordOnly = false;
 
     @JsonProperty("tags_on_match_failure")
     @JsonPropertyDescription("The tag to add to an event if the <code>obfuscate</code> processor fails to match the pattern.")
-    @ExampleValues({
-        @Example(value = "[\"_failure\"]", description = "{\"tags\": [\"_failure\"]} will be added to the event if the processor fails to match the pattern.")
-    })
     private List<String> tagsOnMatchFailure;
 
     @JsonProperty("obfuscate_when")
     @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> such as <code>/is_testing_data == true</code>. " +
             "If specified, the <code>obfuscate</code> processor will only run on events when the expression evaluates to true. ")
+    @ExampleValues({
+            @Example(value = "/some_key != null", description = "Only runs the obfuscate processor on the Event if the existing key some_key is not null.")
+    })
     private String obfuscateWhen;
 
     public ObfuscationProcessorConfig() {

diff --git a/...ava/org/opensearch/dataprepper/plugins/processor/obfuscation/action/MaskActionConfig.java b/...ava/org/opensearch/dataprepper/plugins/processor/obfuscation/action/MaskActionConfig.java
@@ -21,7 +21,7 @@ public class MaskActionConfig {
 
     @JsonProperty(value = "mask_character", defaultValue = DEFAULT_MASK_CHARACTER)
     @Pattern(regexp = "[*#!%&@]", message = "Valid characters are *, #, $, %, &, ! and @")
-    @JsonPropertyDescription("The character to use to mask text. By default, this is <code>*</code>. Valid characters are *, #, $, %, &, ! and @")
+    @JsonPropertyDescription("The character to use to mask text. By default, this is <code>*</code>. Valid characters are *, #, $, %, &, ! and @.")
     private String maskCharacter = DEFAULT_MASK_CHARACTER;
 
     @JsonProperty(value = "mask_character_length", defaultValue = "" + DEFAULT_MASK_LENGTH)

diff --git a/.../java/org/opensearch/dataprepper/plugins/processor/parse/ion/ParseIonProcessorConfig.java b/.../java/org/opensearch/dataprepper/plugins/processor/parse/ion/ParseIonProcessorConfig.java
@@ -38,6 +38,12 @@ public class ParseIonProcessorConfig implements CommonParseConfig {
     @JsonPropertyDescription("The destination field of the structured object from the parsed ION. Defaults to the root of the event. Cannot be an empty string, <code>/</code>, or any whitespace-only string because these are not valid event fields.")
     private String destination;
 
+    @JsonProperty(value = "depth", defaultValue = "0")
+    @Min(0)
+    @Max(10)
+    @JsonPropertyDescription("Indicates the depth at which the nested values of the event are not parsed any more. Default is 0, which means all levels of nested values are parsed. If the depth is 1, only the top level keys are parsed and all its nested values are represented as strings")
+    private int depth = 0;
+
     @JsonProperty("pointer")
     @JsonPropertyDescription("A JSON pointer to the field to be parsed. There is no pointer by default, meaning the entire source is parsed. The pointer can access JSON array indexes as well. " +
             "If the JSON pointer is invalid then the entire source data is parsed into the outgoing event. If the key that is pointed to already exists in the event and the destination is the root, then the pointer uses the entire path of the key.")
@@ -58,25 +64,19 @@ public class ParseIonProcessorConfig implements CommonParseConfig {
     @JsonPropertyDescription("A list of strings specifying the tags to be set in the event that the processor fails or an unknown exception occurs during parsing.")
     private List<String> tagsOnFailure;
 
-    @JsonProperty("parse_when")
-    @JsonPropertyDescription("A Data Prepper [conditional expression](https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/), such as <code>/some-key == \"test\"</code>, that will be evaluated to determine whether the processor will be run on the event.")
-    @ExampleValues({
-        @Example(value = "/some_key == null", description = "Only runs parsing on the Event if some_key is null or doesn't exist.")
-    })
-    private String parseWhen;
-
     @JsonProperty(value = "handle_failed_events", defaultValue = "skip")
     @JsonPropertyDescription("Determines how to handle events with ION processing errors. Options include 'skip', " +
             "which will log the error and send the event downstream to the next processor, and 'skip_silently', " +
             "which will send the Event downstream to the next processor without logging the error. ")
     @NotNull
     private HandleFailedEventsOption handleFailedEventsOption = HandleFailedEventsOption.SKIP;
 
-    @JsonProperty(value = "depth", defaultValue = "0")
-    @Min(0)
-    @Max(10)
-    @JsonPropertyDescription("Indicates the depth at which the nested values of the event are not parsed any more. Default is 0, which means all levels of nested values are parsed. If the depth is 1, only the top level keys are parsed and all its nested values are represented as strings")
-    private int depth = 0;
+    @JsonProperty("parse_when")
+    @JsonPropertyDescription("A Data Prepper [conditional expression](https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/), such as <code>/some-key == \"test\"</code>, that will be evaluated to determine whether the processor will be run on the event.")
+    @ExampleValues({
+        @Example(value = "/some_key == null", description = "Only runs parsing on the Event if some_key is null or doesn't exist.")
+    })
+    private String parseWhen;
 
     @Override
     public String getSource() {

diff --git a/...ava/org/opensearch/dataprepper/plugins/processor/parse/json/ParseJsonProcessorConfig.java b/...ava/org/opensearch/dataprepper/plugins/processor/parse/json/ParseJsonProcessorConfig.java
@@ -64,14 +64,6 @@ public class ParseJsonProcessorConfig implements CommonParseConfig {
     @JsonPropertyDescription("A list of strings specifying the tags to be set in the event when the processor fails or an unknown exception occurs during parsing.")
     private List<String> tagsOnFailure;
 
-    @JsonProperty("parse_when")
-    @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> such as <code>/some_key == \"test\"</code>. " +
-            "If specified, the <code>parse_json</code> processor will only run on events when the expression evaluates to true. ")
-    @ExampleValues({
-        @Example(value = "/some_key == null", description = "Only runs parsing on the Event if some_key is null or doesn't exist.")
-    })
-    private String parseWhen;
-
     @JsonProperty("handle_failed_events")
     @JsonPropertyDescription("Determines how to handle events with JSON processing errors. Options include 'skip', " +
             "which will log the error and send the event downstream to the next processor, and 'skip_silently', " +
@@ -80,6 +72,14 @@ public class ParseJsonProcessorConfig implements CommonParseConfig {
     @NotNull
     private HandleFailedEventsOption handleFailedEventsOption = HandleFailedEventsOption.SKIP;
 
+    @JsonProperty("parse_when")
+    @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> such as <code>/some_key == \"test\"</code>. " +
+            "If specified, the <code>parse_json</code> processor will only run on events when the expression evaluates to true. ")
+    @ExampleValues({
+        @Example(value = "/some_key == null", description = "Only runs parsing on the Event if some_key is null or doesn't exist.")
+    })
+    private String parseWhen;
+
     @Override
     public String getSource() {
         return source;

diff --git a/.../java/org/opensearch/dataprepper/plugins/processor/parse/xml/ParseXmlProcessorConfig.java b/.../java/org/opensearch/dataprepper/plugins/processor/parse/xml/ParseXmlProcessorConfig.java
@@ -47,14 +47,6 @@ public class ParseXmlProcessorConfig implements CommonParseConfig {
     @JsonPropertyDescription("If true, the configured <code>source</code> field will be deleted after the XML data is parsed into separate fields.")
     private boolean deleteSource = false;
 
-    @JsonProperty("parse_when")
-    @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> such as <code>/some_key == \"test\"</code>. " +
-            "If specified, the <code>parse_xml</code> processor will only run on events when the expression evaluates to true. ")
-    @ExampleValues({
-        @Example(value = "/some_key == null", description = "Only runs parsing on the Event if some_key is null or doesn't exist.")
-    })
-    private String parseWhen;
-
     @JsonProperty("tags_on_failure")
     @JsonPropertyDescription("A list of strings specifying the tags to be set in the event when the processor fails or an unknown exception occurs during parsing.")
     private List<String> tagsOnFailure;
@@ -67,6 +59,14 @@ public class ParseXmlProcessorConfig implements CommonParseConfig {
     @NotNull
     private HandleFailedEventsOption handleFailedEventsOption = HandleFailedEventsOption.SKIP;
 
+    @JsonProperty("parse_when")
+    @JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> such as <code>/some_key == \"test\"</code>. " +
+            "If specified, the <code>parse_xml</code> processor will only run on events when the expression evaluates to true. ")
+    @ExampleValues({
+        @Example(value = "/some_key == null", description = "Only runs parsing on the Event if some_key is null or doesn't exist.")
+    })
+    private String parseWhen;
+
     @Override
     public String getSource() {
         return source;