forked from opensearch-project/data-prepper
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
9 changed files
with
300 additions
and
3 deletions.
There are no files selected for viewing
52 changes: 52 additions & 0 deletions
52
...rg/opensearch/dataprepper/plugins/processor/aggregate/actions/GroupByAggregateAction.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| package org.opensearch.dataprepper.plugins.processor.aggregate.actions; | ||
|
|
||
| import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; | ||
| import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; | ||
| import org.opensearch.dataprepper.model.event.Event; | ||
| import org.opensearch.dataprepper.plugins.hasher.IdentificationKeysHasher; | ||
| import org.opensearch.dataprepper.plugins.processor.aggregate.AggregateAction; | ||
| import org.opensearch.dataprepper.plugins.processor.aggregate.AggregateActionInput; | ||
| import org.opensearch.dataprepper.plugins.processor.aggregate.AggregateActionOutput; | ||
| import org.opensearch.dataprepper.plugins.processor.aggregate.AggregateActionResponse; | ||
| import org.opensearch.dataprepper.plugins.processor.aggregate.GroupState; | ||
|
|
||
| import java.time.Instant; | ||
| import java.util.ArrayList; | ||
| import java.util.Arrays; | ||
| import java.util.List; | ||
| import java.util.Objects; | ||
| import java.util.Random; | ||
|
|
||
| @DataPrepperPlugin(name = "groupby", pluginType = AggregateAction.class, pluginConfigurationType = GroupByAggregateActionConfig.class) | ||
| public class GroupByAggregateAction implements AggregateAction { | ||
| static final String LAST_RECEIVED_TIME_KEY = "last_received_time"; | ||
| static final String SHOULD_CONCLUDE_CHECK_SET_KEY = "should_conclude_check_set"; | ||
| static final String EVENTS_KEY = "events"; | ||
| static final String ERROR_STATUS_KEY = "error_status"; | ||
| private final Random random; | ||
| private final IdentificationKeysHasher identificationKeysHasher; | ||
|
|
||
| @DataPrepperPluginConstructor | ||
| public GroupByAggregateAction(final GroupByAggregateActionConfig groupByAggregateActionConfig) { | ||
| this.identificationKeysHasher = new IdentificationKeysHasher(groupByAggregateActionConfig.getIdentificationKeys()); | ||
| this.random = new Random(); | ||
| } | ||
|
|
||
| @Override | ||
| public AggregateActionResponse handleEvent(final Event event, final AggregateActionInput aggregateActionInput) { | ||
| final GroupState groupState = aggregateActionInput.getGroupState(); | ||
| List<Event> events = (List)groupState.getOrDefault(EVENTS_KEY, new ArrayList<>()); | ||
| events.add(event); | ||
| final IdentificationKeysHasher.IdentificationKeysMap identificationKeysMap = identificationKeysHasher.createIdentificationKeysMapFromEvent(event); | ||
| event.getMetadata().setAttribute("partition_key", identificationKeysMap.hashCode()); | ||
| groupState.put(EVENTS_KEY, events); | ||
| groupState.put(LAST_RECEIVED_TIME_KEY, Instant.now()); | ||
| return AggregateActionResponse.nullEventResponse(); | ||
| } | ||
|
|
||
| @Override | ||
| public AggregateActionOutput concludeGroup(final AggregateActionInput aggregateActionInput) { | ||
| GroupState groupState = aggregateActionInput.getGroupState(); | ||
| return new AggregateActionOutput((List)groupState.getOrDefault(EVENTS_KEY, List.of())); | ||
| } | ||
| } |
16 changes: 16 additions & 0 deletions
16
...nsearch/dataprepper/plugins/processor/aggregate/actions/GroupByAggregateActionConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| package org.opensearch.dataprepper.plugins.processor.aggregate.actions; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonProperty; | ||
| import jakarta.validation.constraints.NotEmpty; | ||
|
|
||
| import java.util.List; | ||
|
|
||
| public class GroupByAggregateActionConfig { | ||
| @JsonProperty("identification_keys") | ||
| @NotEmpty | ||
| private List<String> identificationKeys; | ||
|
|
||
| public List<String> getIdentificationKeys() { | ||
| return identificationKeys; | ||
| } | ||
| } |
117 changes: 117 additions & 0 deletions
117
...a/org/opensearch/dataprepper/plugins/processor/aggregate/actions/JoinAggregateAction.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| package org.opensearch.dataprepper.plugins.processor.aggregate.actions; | ||
|
|
||
| import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; | ||
| import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; | ||
| import org.opensearch.dataprepper.model.event.Event; | ||
| import org.opensearch.dataprepper.model.event.EventType; | ||
| import org.opensearch.dataprepper.model.event.JacksonEvent; | ||
| import org.opensearch.dataprepper.plugins.hasher.IdentificationKeysHasher; | ||
| import org.opensearch.dataprepper.plugins.processor.aggregate.AggregateAction; | ||
| import org.opensearch.dataprepper.plugins.processor.aggregate.AggregateActionInput; | ||
| import org.opensearch.dataprepper.plugins.processor.aggregate.AggregateActionOutput; | ||
| import org.opensearch.dataprepper.plugins.processor.aggregate.AggregateActionResponse; | ||
| import org.opensearch.dataprepper.plugins.processor.aggregate.GroupState; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.Collection; | ||
| import java.util.Collections; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Random; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| @DataPrepperPlugin(name = "join", pluginType = AggregateAction.class, pluginConfigurationType = JoinAggregateActionConfig.class) | ||
| public class JoinAggregateAction implements AggregateAction { | ||
| private static final String JOINED_KEY = "JOINED_KEY"; | ||
| private static final String STREAMS_KEY = "STREAMS_KEY"; | ||
| private final IdentificationKeysHasher identificationKeysHasher; | ||
| private final Random random; | ||
| private final JoinAggregateActionConfig joinAggregateActionConfig; | ||
|
|
||
| @DataPrepperPluginConstructor | ||
| public JoinAggregateAction(final JoinAggregateActionConfig joinAggregateActionConfig) { | ||
| this.identificationKeysHasher = new IdentificationKeysHasher(joinAggregateActionConfig.getIdentificationKeys()); | ||
| this.joinAggregateActionConfig = joinAggregateActionConfig; | ||
| this.random = new Random(); | ||
| } | ||
|
|
||
| @Override | ||
| public AggregateActionResponse handleEvent(final Event event, final AggregateActionInput aggregateActionInput) { | ||
| final GroupState groupState = aggregateActionInput.getGroupState(); | ||
|
|
||
| String stream = event.get("stream", String.class); | ||
| Map<String, List<Object>> streamMap = (Map<String, List<Object>>) groupState.getOrDefault(stream, new HashMap<>()); | ||
|
|
||
| IdentificationKeysHasher.IdentificationKeysMap keysMap = identificationKeysHasher.createIdentificationKeysMapFromEvent(event); | ||
| streamMap.computeIfAbsent(keysMap.toString(), k -> new ArrayList<>()).add(event); | ||
|
|
||
| groupState.put(stream, streamMap); | ||
|
|
||
| return AggregateActionResponse.nullEventResponse(); | ||
| } | ||
|
|
||
| @Override | ||
| public AggregateActionOutput concludeGroup(final AggregateActionInput aggregateActionInput) { | ||
| GroupState groupState = aggregateActionInput.getGroupState(); | ||
|
|
||
| List<Map<String, List<Object>>> listOfAllStreams = new ArrayList<>(); | ||
| for (String stream: joinAggregateActionConfig.getStreams()) { | ||
| Map<String, List<Object>> streamMap = (Map<String, List<Object>>) groupState.getOrDefault(stream, new HashMap<>()); | ||
| if (streamMap.isEmpty()) { | ||
| return new AggregateActionOutput(List.of()); | ||
| } | ||
| listOfAllStreams.add(streamMap); | ||
| } | ||
|
|
||
|
|
||
| List<Event> events = new ArrayList<>(); | ||
| // Iterate over all streams | ||
| Map<String, List<List<Object>>> aggregatedMap = new HashMap<>(); | ||
| for (Map<String, List<Object>> map : listOfAllStreams) { | ||
| for (Map.Entry<String, List<Object>> entry : map.entrySet()) { | ||
| aggregatedMap.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(entry.getValue()); | ||
| } | ||
| } | ||
|
|
||
| Map<String, List<Object>> mergedJoinedMap = aggregatedMap.entrySet().stream() | ||
| .collect(Collectors.toMap( | ||
| Map.Entry::getKey, | ||
| entry -> mergeAndJoin(entry.getValue()) | ||
| )); | ||
|
|
||
| for (Map.Entry<String, List<Object>> key: mergedJoinedMap.entrySet()) { | ||
| List<Object> records = key.getValue(); | ||
| records.forEach(record -> events.add((Event) record)); | ||
| } | ||
|
|
||
| return new AggregateActionOutput(events); | ||
| } | ||
|
|
||
| private static List<Object> mergeAndJoin(List<List<Object>> lists) { | ||
| if (lists.isEmpty()) { | ||
| return Collections.emptyList(); | ||
| } | ||
|
|
||
| // Start with the first list | ||
| List<Object> result = new ArrayList<>(lists.get(0)); | ||
|
|
||
| // Compute the Cartesian product for all lists | ||
| for (int i = 1; i < lists.size(); i++) { | ||
| List<Object> currentList = lists.get(i); | ||
| result = result.stream() | ||
| .flatMap(v1 -> currentList.stream() | ||
| .map(v2 -> { | ||
| Event e1 = (Event) v1; | ||
| Event e2 = (Event) v2; | ||
| Map<String, Object> m1 = e1.toMap(); | ||
| Map<String, Object> m2 = e2.toMap(); | ||
| m1.putAll(m2); | ||
| return JacksonEvent.builder().withEventType(EventType.DOCUMENT.toString()).withData(m1).build(); | ||
| })) | ||
| .collect(Collectors.toList()); | ||
| } | ||
|
|
||
| return result; | ||
| } | ||
| } |
32 changes: 32 additions & 0 deletions
32
...opensearch/dataprepper/plugins/processor/aggregate/actions/JoinAggregateActionConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| package org.opensearch.dataprepper.plugins.processor.aggregate.actions; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonProperty; | ||
| import jakarta.validation.constraints.NotEmpty; | ||
|
|
||
| import java.util.List; | ||
|
|
||
| public class JoinAggregateActionConfig { | ||
| @JsonProperty("identification_keys") | ||
| @NotEmpty | ||
| private List<String> identificationKeys; | ||
|
|
||
| @JsonProperty("streams") | ||
| @NotEmpty | ||
| private List<String> streams; | ||
|
|
||
| @JsonProperty("primary_key") | ||
| @NotEmpty | ||
| private String primary_key; | ||
|
|
||
| public List<String> getIdentificationKeys() { | ||
| return identificationKeys; | ||
| } | ||
|
|
||
| public List<String> getStreams() { | ||
| return streams; | ||
| } | ||
|
|
||
| public String getPrimary_key() { | ||
| return primary_key; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.