Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Start unit tests for the OpenSearch sink testing the document_version… #3599

Merged
merged 1 commit into from
Nov 10, 2023
Merged

Start unit tests for the OpenSearch sink testing the document_version… #3599

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...-prepper-api/src/main/java/org/opensearch/dataprepper/expression/ExpressionEvaluator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,5 @@ default Boolean evaluateConditional(final String statement, final Event context)

Boolean isValidExpressionStatement(final String statement);

Boolean isValidFormatExpressions(final String format);
Boolean isValidFormatExpression(final String format);
}
2 changes: 1 addition & 1 deletion ...pper-api/src/test/java/org/opensearch/dataprepper/expression/ExpressionEvaluatorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public Boolean isValidExpressionStatement(final String statement) {
}

@Override
public Boolean isValidFormatExpressions(String format) {
public Boolean isValidFormatExpression(String format) {
return true;
}
}
Expand Down
2 changes: 1 addition & 1 deletion ...ssion/src/main/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public Boolean isValidExpressionStatement(final String statement) {
}

@Override
public Boolean isValidFormatExpressions(final String format) {
public Boolean isValidFormatExpression(final String format) {
int fromIndex = 0;
int position = 0;
while ((position = format.indexOf("${", fromIndex)) != -1) {
Expand Down
4 changes: 2 additions & 2 deletions ...n/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluatorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,14 +132,14 @@ void isValidExpressionStatement_returns_false_when_parse_throws() {
"abc-${invalid, false"
})
void isValidFormatExpressionsReturnsCorrectResult(final String format, final Boolean expectedResult) {
assertThat(statementEvaluator.isValidFormatExpressions(format), equalTo(expectedResult));
assertThat(statementEvaluator.isValidFormatExpression(format), equalTo(expectedResult));
}

@ParameterizedTest
@ValueSource(strings = {"abc-${anyS(=tring}"})
void isValidFormatExpressionsReturnsFalseWhenIsValidKeyAndValidExpressionIsFalse(final String format) {
doThrow(RuntimeException.class).when(parser).parse(anyString());
assertThat(statementEvaluator.isValidFormatExpressions(format), equalTo(false));
assertThat(statementEvaluator.isValidFormatExpression(format), equalTo(false));
}

}
Expand Down
4 changes: 2 additions & 2 deletions ...grationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -647,7 +647,7 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException,
Event event = (Event)testRecords.get(0).getData();
event.getMetadata().setAttribute("action", "create");
final String actionFormatExpression = "${getMetadata(\"action\")}";
when(expressionEvaluator.isValidFormatExpressions(actionFormatExpression)).thenReturn(true);
when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true);
when(expressionEvaluator.isValidExpressionStatement("getMetadata(\"action\")")).thenReturn(true);
when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getMetadata().getAttribute("action"));
pluginSetting.getSettings().put(IndexConfiguration.ACTION, actionFormatExpression);
Expand Down Expand Up @@ -680,7 +680,7 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce
Event event = (Event)testRecords.get(0).getData();
event.getMetadata().setAttribute("action", "unknown");
final String actionFormatExpression = "${getMetadata(\"action\")}";
when(expressionEvaluator.isValidFormatExpressions(actionFormatExpression)).thenReturn(true);
when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true);
when(expressionEvaluator.isValidExpressionStatement("getMetadata(\"action\")")).thenReturn(true);
when(expressionEvaluator.evaluate("getMetadata(\"action\")", event)).thenReturn(event.getMetadata().getAttribute("action"));
pluginSetting.getSettings().put(IndexConfiguration.ACTION, actionFormatExpression);
Expand Down
13 changes: 9 additions & 4 deletions ...arch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSink.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,12 +371,17 @@ public void doOutput(final Collection<Record<Event>> records) {
versionExpressionEvaluationResult = event.formatString(versionExpression, expressionEvaluator);
version = Long.valueOf(event.formatString(versionExpression, expressionEvaluator));
} catch (final NumberFormatException e) {
LOG.warn("Unable to convert the result of evaluating document_version '{}' to Long for an Event. The evaluation result '{}' must be a valid Long type", versionExpression, versionExpressionEvaluationResult);
logFailureForDlqObjects(List.of(createDlqObjectFromEvent(event, indexName, e.getMessage())), e);
final String errorMessage = String.format(
"Unable to convert the result of evaluating document_version '%s' to Long for an Event. The evaluation result '%s' must be a valid Long type", versionExpression, versionExpressionEvaluationResult
);
LOG.error(errorMessage);
logFailureForDlqObjects(List.of(createDlqObjectFromEvent(event, indexName, errorMessage)), e);
dynamicDocumentVersionDroppedEvents.increment();
} catch (final RuntimeException e) {
LOG.error("There was an exception when evaluating the document_version '{}'. Check the dlq if configured to see details about the affected Event: {}", versionExpression, e.getMessage());
logFailureForDlqObjects(List.of(createDlqObjectFromEvent(event, indexName, e.getMessage())), e);
final String errorMessage = String.format(
"There was an exception when evaluating the document_version '%s': %s", versionExpression, e.getMessage());
LOG.error(errorMessage + " Check the dlq if configured to see more details about the affected Event");
logFailureForDlqObjects(List.of(createDlqObjectFromEvent(event, indexName, errorMessage)), e);
dynamicDocumentVersionDroppedEvents.increment();
}
}
Expand Down
6 changes: 3 additions & 3 deletions ...ain/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetti
final String versionType = pluginSetting.getStringOrDefault(DOCUMENT_VERSION_TYPE, null);

builder = builder.withVersionExpression(versionExpression);
if (versionExpression != null && (!expressionEvaluator.isValidFormatExpressions(versionExpression))) {
if (versionExpression != null && (!expressionEvaluator.isValidFormatExpression(versionExpression))) {
throw new InvalidPluginConfigurationException("document_version {} is not a valid format expression.");
}

Expand Down Expand Up @@ -546,7 +546,7 @@ public Builder withIsmPolicyFile(final String ismPolicyFile) {

public Builder withAction(final String action, final ExpressionEvaluator expressionEvaluator) {
checkArgument((EnumUtils.isValidEnumIgnoreCase(OpenSearchBulkActions.class, action) ||
(action.contains("${") && expressionEvaluator.isValidFormatExpressions(action))), "action \"" + action + "\" is invalid. action must be one of the following: " + Arrays.stream(OpenSearchBulkActions.values()).collect(Collectors.toList()));
(action.contains("${") && expressionEvaluator.isValidFormatExpression(action))), "action \"" + action + "\" is invalid. action must be one of the following: " + Arrays.stream(OpenSearchBulkActions.values()).collect(Collectors.toList()));
this.action = action;
return this;
}
Expand All @@ -556,7 +556,7 @@ public Builder withActions(final List<Map<String, Object>> actions, final Expres
String action = (String)actionMap.get("type");
if (action != null) {
checkArgument((EnumUtils.isValidEnumIgnoreCase(OpenSearchBulkActions.class, action) ||
(action.contains("${") && expressionEvaluator.isValidFormatExpressions(action))), "action \"" + action + "\". action must be one of the following: " + Arrays.stream(OpenSearchBulkActions.values()).collect(Collectors.toList()));
(action.contains("${") && expressionEvaluator.isValidFormatExpression(action))), "action \"" + action + "\". action must be one of the following: " + Arrays.stream(OpenSearchBulkActions.values()).collect(Collectors.toList()));
}
}
this.actions = actions;
Expand Down
2 changes: 1 addition & 1 deletion .../org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkConfigurationTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ public void testReadESConfigWithBulkActionCreateExpression() {
pluginSetting.setPipelineName(PIPELINE_NAME);

expressionEvaluator = mock(ExpressionEvaluator.class);
when(expressionEvaluator.isValidFormatExpressions(actionFormatExpression)).thenReturn(true);
when(expressionEvaluator.isValidFormatExpression(actionFormatExpression)).thenReturn(true);
final OpenSearchSinkConfiguration openSearchSinkConfiguration =
OpenSearchSinkConfiguration.readESConfig(pluginSetting, expressionEvaluator);

Expand Down
Loading