Skip to content

Commit

Permalink
Integration test changes
Browse files Browse the repository at this point in the history 
Signed-off-by: Maxwell Brown <[email protected]>
  • Loading branch information
@Galactus22625
Galactus22625 committed Dec 23, 2024
1 parent 21535f6 commit d42bb12
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 14 deletions.
38 changes: 25 additions & 13 deletions ...grationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -826,10 +826,11 @@ public void testOutputCustomIndex() throws IOException, InterruptedException {
final String testTemplateFile = Objects.requireNonNull(
getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile();
final String testIdField = "someId";
final String testDocumentID = "${/someId}";
final String testId = "foo";
final List<Record<Event>> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId)));
Map<String, Object> metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile);
metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField);
metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID);
final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata);
final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true);
sink.output(testRecords);
Expand All @@ -853,10 +854,11 @@ public void testOpenSearchBulkActionsCreate() throws IOException, InterruptedExc
final String testTemplateFile = Objects.requireNonNull(
getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile();
final String testIdField = "someId";
final String testDocumentID = "${/someId}";
final String testId = "foo";
final List<Record<Event>> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId)));
Map<String, Object> metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile);
metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField);
metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID);
metadata.put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString());
final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata);
final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true);
Expand All @@ -881,10 +883,11 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException,
final String testTemplateFile = Objects.requireNonNull(
getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile();
final String testIdField = "someId";
final String testDocumentID = "${/someId}";
final String testId = "foo";
final List<Record<Event>> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId)));
Map<String, Object> metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile);
metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField);
metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID);
Event event = (Event) testRecords.get(0).getData();
event.getMetadata().setAttribute("action", "create");
final String actionFormatExpression = "${getMetadata(\"action\")}";
Expand Down Expand Up @@ -915,10 +918,11 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce
final String testTemplateFile = Objects.requireNonNull(
getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile();
final String testIdField = "someId";
final String testDocumentID = "${/someId}";
final String testId = "foo";
final List<Record<Event>> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId)));
Map<String, Object> metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile);
metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField);
metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID);
Event event = (Event) testRecords.get(0).getData();
event.getMetadata().setAttribute("action", "unknown");
final String actionFormatExpression = "${getMetadata(\"action\")}";
Expand All @@ -941,11 +945,12 @@ public void testBulkActionCreateWithActions() throws IOException, InterruptedExc
final String testTemplateFile = Objects.requireNonNull(
getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile();
final String testIdField = "someId";
final String testDocumentID = "${/someId}";
final String testId = "foo";
final List<Record<Event>> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId)));

Map<String, Object> metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile);
metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField);
metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID);
List<Map<String, Object>> aList = new ArrayList<>();
Map<String, Object> aMap = new HashMap<>();
aMap.put("type", OpenSearchBulkActions.CREATE.toString());
Expand Down Expand Up @@ -975,11 +980,12 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc
getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile();

final String testIdField = "someId";
final String testDocumentID = "${/someId}";
final String testId = "foo";
List<Record<Event>> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1")));

Map<String, Object> metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile);
metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField);
metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID);
List<Map<String, Object>> aList = new ArrayList<>();
Map<String, Object> aMap = new HashMap<>();
aMap.put("type", OpenSearchBulkActions.CREATE.toString());
Expand Down Expand Up @@ -1025,6 +1031,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr
getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile();

final String testIdField = "someId";
final String testDocumentID = "${/someId}";
final String testId = "foo";
final String documentRootKey = "root_key";

Expand All @@ -1039,7 +1046,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr
Map<String, Object> metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile);

metadata.put(IndexConfiguration.DOCUMENT_ROOT_KEY, documentRootKey);
metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField);
metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID);
List<Map<String, Object>> aList = new ArrayList<>();
Map<String, Object> actionMap = new HashMap<>();
actionMap.put("type", OpenSearchBulkActions.CREATE.toString());
Expand Down Expand Up @@ -1093,6 +1100,7 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int
getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile();

final String testIdField = "someId";
final String testDocumentID = "${/someId}";
final String testId = "foo";
List<Record<Event>> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "key", "value")));

Expand All @@ -1102,7 +1110,7 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int
aList.add(actionMap);

Map<String, Object> metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile);
metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField);
metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID);
metadata.put(IndexConfiguration.ACTIONS, aList);
final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata);
OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true);
Expand All @@ -1125,11 +1133,12 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc
getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile();

final String testIdField = "someId";
final String testDocumentID = "${/someId}";
final String testId = "foo";
List<Record<Event>> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1")));

Map<String, Object> metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile);
metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField);
metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID);
List<Map<String, Object>> aList = new ArrayList<>();
Map<String, Object> aMap = new HashMap<>();
aMap.put("type", OpenSearchBulkActions.CREATE.toString());
Expand Down Expand Up @@ -1177,10 +1186,11 @@ public void testBulkActionUpsertWithoutCreate() throws IOException, InterruptedE
getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile();

final String testIdField = "someId";
final String testDocumentID = "${/someId}";
final String testId = "foo";
List<Record<Event>> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson3(testIdField, testId, "name", "value1", "newKey", "newValue")));
Map<String, Object> metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile);
metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField);
metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID);
List<Map<String, Object>> aList = new ArrayList<>();
Map<String, Object> aMap = new HashMap<>();
aMap.put("type", OpenSearchBulkActions.UPSERT.toString());
Expand Down Expand Up @@ -1213,11 +1223,12 @@ public void testBulkActionDeleteWithActions() throws IOException, InterruptedExc
getClass().getClassLoader().getResource(TEST_TEMPLATE_BULK_FILE)).getFile();

final String testIdField = "someId";
final String testDocumentID = "${/someId}";
final String testId = "foo";
List<Record<Event>> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId)));

Map<String, Object> metadata = initializeConfigurationMetadata(null, testIndexAlias, testTemplateFile);
metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField);
metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID);
List<Map<String, Object>> aList = new ArrayList<>();
Map<String, Object> aMap = new HashMap<>();
aMap.put("type", OpenSearchBulkActions.DELETE.toString());
Expand Down Expand Up @@ -1315,7 +1326,7 @@ public void testEventOutputWithSpecialAndExtremeValues(final Object testValue) t
}

@ParameterizedTest
@ValueSource(strings = {"info/ids/id", "id"})
@ValueSource(strings = {"${/info/ids/id}", "${/id}"})
public void testOpenSearchDocumentId(final String testDocumentIdField) throws IOException, InterruptedException {
final String expectedId = UUID.randomUUID().toString();
final String testIndexAlias = "test_index";
Expand Down Expand Up @@ -1640,6 +1651,7 @@ public void testOutputManagementDisabled() throws IOException, InterruptedExcept
securityAccessor.createUser(username, password, roleName);

final String testIdField = "someId";
final String testDocumentID = "${/someId}";
final String testId = "foo";

final List<Record<Event>> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId)));
Expand All @@ -1648,7 +1660,7 @@ public void testOutputManagementDisabled() throws IOException, InterruptedExcept
metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.MANAGEMENT_DISABLED.getValue());
metadata.put(USERNAME, username);
metadata.put(PASSWORD, password);
metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField);
metadata.put(IndexConfiguration.DOCUMENT_ID, testDocumentID);
final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata);
final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true);

Expand Down
2 changes: 1 addition & 1 deletion ...ain/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public class IndexConfiguration {
public static final String MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION = "max_local_compressions_for_estimation";
public static final String FLUSH_TIMEOUT = "flush_timeout";
public static final String DOCUMENT_ID = "document_id";
public static final String ROUTING_FIELD = "routing_field";
public static final String ROUTING_FIELD = "routing";
public static final String ROUTING = "routing";
public static final String PIPELINE = "pipeline";
public static final String ISM_POLICY_FILE = "ism_policy_file";
Expand Down

0 comments on commit d42bb12

Please sign in to comment.