From 5f9b9f6ded47c696f31a327b0ba104b56e90079a Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 18 Dec 2024 17:07:25 -0800 Subject: [PATCH] document_id_field to document_id since deprecated Signed-off-by: Maxwell Brown --- .../sink/opensearch/OpenSearchSinkIT.java | 26 +++++++++---------- .../opensearch/index/IndexConfiguration.java | 1 - 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java index 3bcdccd0ba..14597bde95 100644 --- a/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java +++ b/data-prepper-plugins/opensearch/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/opensearch/OpenSearchSinkIT.java @@ -824,7 +824,7 @@ public void testOutputCustomIndex() throws IOException, InterruptedException { final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); final List> retSources = getSearchResponseDocSources(testIndexAlias); @@ -850,7 +850,7 @@ public void testOpenSearchBulkActionsCreate() throws IOException, InterruptedExc final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); pluginSetting.getSettings().put(IndexConfiguration.ACTION, OpenSearchBulkActions.CREATE.toString()); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -877,7 +877,7 @@ public void testOpenSearchBulkActionsCreateWithExpression() throws IOException, final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "create"); final String actionFormatExpression = "${getMetadata(\"action\")}"; @@ -910,7 +910,7 @@ public void testOpenSearchBulkActionsCreateWithInvalidExpression() throws IOExce final String testId = "foo"; final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); Event event = (Event) testRecords.get(0).getData(); event.getMetadata().setAttribute("action", "unknown"); final String actionFormatExpression = "${getMetadata(\"action\")}"; @@ -936,7 +936,7 @@ public void testBulkActionCreateWithActions() throws IOException, InterruptedExc final List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -969,7 +969,7 @@ public void testBulkActionUpdateWithActions() throws IOException, InterruptedExc List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1027,7 +1027,7 @@ public void testBulkActionUpdateWithDocumentRootKey() throws IOException, Interr final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ROOT_KEY, documentRootKey); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map actionMap = new HashMap<>(); actionMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1088,7 +1088,7 @@ public void testBulkActionUpsertWithActionsAndNoCreate() throws IOException, Int aList.add(actionMap); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); pluginSetting.getSettings().put(IndexConfiguration.ACTIONS, aList); OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); @@ -1114,7 +1114,7 @@ public void testBulkActionUpsertWithActions() throws IOException, InterruptedExc List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson2(testIdField, testId, "name", "value1"))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.CREATE.toString()); @@ -1162,7 +1162,7 @@ public void testBulkActionUpsertWithoutCreate() throws IOException, InterruptedE final String testId = "foo"; List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson3(testIdField, testId, "name", "value1", "newKey", "newValue"))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.UPSERT.toString()); @@ -1198,7 +1198,7 @@ public void testBulkActionDeleteWithActions() throws IOException, InterruptedExc List> testRecords = Collections.singletonList(jsonStringToRecord(generateCustomRecordJson(testIdField, testId))); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, testTemplateFile); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testIdField); List> aList = new ArrayList<>(); Map aMap = new HashMap<>(); aMap.put("type", OpenSearchBulkActions.DELETE.toString()); @@ -1308,7 +1308,7 @@ public void testOpenSearchDocumentId(final String testDocumentIdField) throws IO final List> testRecords = Collections.singletonList(new Record<>(testEvent)); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfig(null, testIndexAlias, null); - pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testDocumentIdField); + pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID, testDocumentIdField); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); sink.output(testRecords); @@ -1622,7 +1622,7 @@ public void testOutputManagementDisabled() throws IOException, InterruptedExcept final Map metadata = initializeConfigurationMetadata(null, testIndexAlias, null); metadata.put(IndexConfiguration.INDEX_TYPE, IndexType.MANAGEMENT_DISABLED.getValue()); metadata.put(AUTHENTICATION, Map.of(USERNAME, username, PASSWORD, password)); - metadata.put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField); + metadata.put(IndexConfiguration.DOCUMENT_ID, testIdField); final OpenSearchSinkConfig openSearchSinkConfig = generateOpenSearchSinkConfigByMetadata(metadata); final OpenSearchSink sink = createObjectUnderTest(openSearchSinkConfig, true); diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index 172f17c4b4..ebd1240655 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -54,7 +54,6 @@ public class IndexConfiguration { public static final String ESTIMATE_BULK_SIZE_USING_COMPRESSION = "estimate_bulk_size_using_compression"; public static final String MAX_LOCAL_COMPRESSIONS_FOR_ESTIMATION = "max_local_compressions_for_estimation"; public static final String FLUSH_TIMEOUT = "flush_timeout"; - public static final String DOCUMENT_ID_FIELD = "document_id_field"; public static final String DOCUMENT_ID = "document_id"; public static final String ROUTING_FIELD = "routing_field"; public static final String ROUTING = "routing";